Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 11 Sep 2015 11:35:00 -0400
changeset 262584 cd7b4b4b747485d59c9fb07cd31e17afac87f70b
parent 262562 7abc5253e8cf95a869a114604fdecdcf569c8d2e
child 262585 60ed4fafa41e63dc1855bd64830a9c00bcea0b83
push id15198
push userkwierso@gmail.com
push dateWed, 16 Sep 2015 00:21:31 +0000
treeherderfx-team@dbfddd6efb5d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted.mielczarek
bugs1198179
milestone43.0a1
Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek
build/upload.py
testing/mozharness/configs/builds/build_pool_specifics.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/builds/linux64_clobber.yml
testing/taskcluster/tasks/builds/opt_macosx64.yml
--- a/build/upload.py
+++ b/build/upload.py
@@ -10,16 +10,20 @@
 # UPLOAD_USER    : username on that host
 #  and one of the following:
 # UPLOAD_PATH    : path on that host to put the files in
 # UPLOAD_TO_TEMP : upload files to a new temporary directory
 #
 # If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out
 # the properties file.
 #
+# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH.
+# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no
+# properties are written out.
+#
 # And will use the following optional environment variables if set:
 # UPLOAD_SSH_KEY : path to a ssh private key to use
 # UPLOAD_PORT    : port to use for ssh
 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
 #                  UPLOAD_PATH and the full paths of all files uploaded will
 #                  be appended to the commandline.
 #
 # All files to be uploaded should be passed as commandline arguments to this
@@ -27,16 +31,17 @@
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 import re
 import json
 import errno
 import hashlib
+import shutil
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
 
 def OptionalEnvironmentVariable(v):
     """Return the value of the environment variable named v, or None
     if it's unset (or empty)."""
     if v in os.environ and os.environ[v] != "":
@@ -101,17 +106,17 @@ def DoSCPFile(file, remote_path, user, h
     cmdline.extend([WindowsPathToMsysPath(file),
                     "%s@%s:%s" % (user, host, remote_path)])
     with redo.retrying(check_call, sleeptime=10) as f:
         f(cmdline)
         return
 
     raise Exception("Command %s returned non-zero exit code" % cmdline)
 
-def GetRemotePath(path, local_file, base_path):
+def GetBaseRelativePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
@@ -209,17 +214,17 @@ def UploadFiles(user, host, path, files,
     remote_files = []
     properties = {}
     try:
         for file in files:
             file = os.path.abspath(file)
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
-            remote_path = GetRemotePath(path, file, base_path)
+            remote_path = GetBaseRelativePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
@@ -231,16 +236,35 @@ def UploadFiles(user, host, path, files,
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
         print "Upload complete"
     return properties
 
+def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
+    """Copy each file in the list of files to `path`.  The `base_path` argument is treated
+    as it is by UploadFiles."""
+    if not path.endswith("/"):
+        path += "/"
+    if base_path is not None:
+        base_path = os.path.abspath(base_path)
+    for file in files:
+        file = os.path.abspath(file)
+        if not os.path.isfile(file):
+            raise IOError("File not found: %s" % file)
+        # first ensure that path exists remotely
+        target_path = GetBaseRelativePath(path, file, base_path)
+        if not os.path.exists(target_path):
+            os.makedirs(target_path)
+        if verbose:
+            print "Copying " + file + " to " + target_path
+        shutil.copy(file, target_path)
+
 def WriteProperties(files, properties_file, url_properties, package):
     properties = url_properties
     for file in files:
         if file.endswith('.complete.mar'):
             properties.update(GetMarProperties(file))
     with open(properties_file, 'w') as outfile:
         properties['packageFilename'] = package
         properties['uploadFiles'] = [os.path.abspath(f) for f in files]
@@ -275,21 +299,35 @@ if __name__ == '__main__':
                       help="Name of the main package.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print "You must specify at least one file to upload"
         sys.exit(1)
     if not options.properties_file:
         print "You must specify a --properties-file"
         sys.exit(1)
+
+    if host == "localhost":
+        if upload_to_temp_dir:
+            print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost"
+            sys.exit(1)
+        if post_upload_command:
+            # POST_UPLOAD_COMMAND is difficult to extract from the mozharness
+            # scripts, so just ignore it until it's no longer used anywhere
+            print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost"
+
     try:
-        url_properties = UploadFiles(user, host, path, args, base_path=options.base_path,
-                                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
-                                     post_upload_command=post_upload_command,
-                                     package=options.package,
-                                     verbose=True)
-        WriteProperties(args, options.properties_file, url_properties, options.package)
+        if host == "localhost":
+            CopyFilesLocally(path, args, base_path=options.base_path,
+                             package=options.package,
+                             verbose=True)
+        else:
+
+            url_properties = UploadFiles(user, host, path, args,
+                                         base_path=options.base_path, port=port, ssh_key=key,
+                                         upload_to_temp_dir=upload_to_temp_dir,
+                                         post_upload_command=post_upload_command,
+                                         package=options.package, verbose=True)
+
+            WriteProperties(args, options.properties_file, url_properties, options.package)
     except IOError, (strerror):
         print strerror
         sys.exit(1)
-    except Exception, (err):
-        print err
-        sys.exit(2)
--- a/testing/mozharness/configs/builds/build_pool_specifics.py
+++ b/testing/mozharness/configs/builds/build_pool_specifics.py
@@ -33,14 +33,18 @@ config = {
         'symbol_server_host': "symbolpush.mozilla.org",
         'stage_server': 'stage.mozilla.org',
         "sendchange_masters": ["buildbot-master81.build.mozilla.org:9301"],
         'taskcluster_index': 'index',
     },
     "taskcluster": {
         'graph_server': 'graphs.mozilla.org',
         'symbol_server_host': "symbolpush.mozilla.org",
-        'stage_server': 'stage.mozilla.org',
+        'stage_server': 'ignored',
         # use the relengapi proxy to talk to tooltool
         "tooltool_servers": ['http://relengapi/tooltool/'],
         "tooltool_url": 'http://relengapi/tooltool/',
+        'upload_env': {
+            'UPLOAD_HOST': 'localhost',
+            'UPLOAD_PATH': '/home/worker/artifacts',
+        },
     },
 }
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -821,25 +821,28 @@ or run without that action (ie: --no-{ac
 
     def query_mach_build_env(self, multiLocale=None):
         c = self.config
         if multiLocale is None and self.query_is_nightly():
             multiLocale = c.get('multi_locale', False)
         mach_env = {}
         if c.get('upload_env'):
             mach_env.update(c['upload_env'])
-            mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
-                'stage_server': c['stage_server']
-            }
-            mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
-                'stage_username': c['stage_username']
-            }
-            mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
-                'stage_ssh_key': c['stage_ssh_key']
-            }
+            if 'UPLOAD_HOST' in mach_env:
+                mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
+                    'stage_server': c['stage_server']
+                }
+            if 'UPLOAD_USER' in mach_env:
+                mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
+                    'stage_username': c['stage_username']
+                }
+            if 'UPLOAD_SSH_KEY' in mach_env:
+                mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
+                    'stage_ssh_key': c['stage_ssh_key']
+                }
 
         if self.query_is_nightly():
             mach_env['LATEST_MAR_DIR'] = c['latest_mar_dir'] % {
                 'branch': self.branch
             }
 
         # _query_post_upload_cmd returns a list (a cmd list), for env sake here
         # let's make it a string
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -20,35 +20,27 @@ echo "running as" $(id)
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
-# files to be "uploaded" (moved to ~/artifacts) from obj-firefox/dist
-: DIST_UPLOADS                  ${DIST_UPLOADS:=""}
-# files which will be be prefixed with target before being sent to artifacts
-# e.g. DIST_TARGET_UPLOADS="a.zip" runs mv v2.0.a.zip mv artifacts/target.a.zip
-: DIST_TARGET_UPLOADS           ${DIST_TARGET_UPLOADS:=""}
-
 set -v
 
-# Don't run the upload step; this is passed through mozharness to mach.  Once
-# the mozharness scripts are not run in Buildbot anymore, this can be moved to
-# Mozharness (or the upload step removed from mach entirely)
-export MOZ_AUTOMATION_UPLOAD=0
-
 export MOZ_CRASHREPORTER_NO_REPORT=1
 export MOZ_OBJDIR=obj-firefox
 export MOZ_SYMBOLS_EXTRA_BUILDID=linux64
-export POST_SYMBOL_UPLOAD_CMD=/usr/local/bin/post-symbol-upload.py
 export TINDERBOX_OUTPUT=1
 
+# use "simple" package names so that they can be hard-coded in the task's
+# extras.locations
+export MOZ_SIMPLE_PACKAGE_NAME=target
+
 # Ensure that in tree libraries can be found
 export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
 
 # test required parameters are supplied
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then exit 1; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then exit 1; fi
 
 cleanup() {
@@ -131,25 +123,8 @@ python2.7 $WORKSPACE/build/src/testing/$
   --no-update \
   --no-upload-files \
   --no-sendchange \
   --log-level=debug \
   --work-dir=$WORKSPACE/build \
   --no-action=generate-build-stats \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}
-
-mkdir -p /home/worker/artifacts
-
-# upload auxiliary files
-cd $WORKSPACE/build/src/obj-firefox/dist
-
-for file in $DIST_UPLOADS
-do
-    mv $file $HOME/artifacts/$file
-done
-
-# Discard version numbers from packaged files, they just make it hard to write
-# the right filename in the task payload where artifacts are declared
-for file in $DIST_TARGET_UPLOADS
-do
-    mv *.$file $HOME/artifacts/target.$file
-done
--- a/testing/taskcluster/tasks/builds/linux64_clobber.yml
+++ b/testing/taskcluster/tasks/builds/linux64_clobber.yml
@@ -25,29 +25,27 @@ task:
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_base_linux_64_builds.py balrog/production.py'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
       NEED_XVFB: true
-      DIST_UPLOADS: 'jsshell-linux-x86_64.zip'
-      DIST_TARGET_UPLOADS: 'x-test.linux-x86_64.tar.bz2 linux-x86_64.tar.bz2 linux-x86_64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
 
   extra:
     treeherderEnv:
       - production
       - staging
     treeherder:
       machine:
         # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
         platform: linux64
     # Rather then enforcing particular conventions we require that all build
     # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
+    # files are located, relative to the task's artifacts URL
     locations:
-      build: 'public/build/target.linux-x86_64.tar.bz2'
+      build: 'public/build/target.tar.bz2'
       tests: 'public/build/target.tests.zip'
--- a/testing/taskcluster/tasks/builds/opt_macosx64.yml
+++ b/testing/taskcluster/tasks/builds/opt_macosx64.yml
@@ -34,18 +34,16 @@ task:
       MOZHARNESS_CONFIG: 'builds/releng_base_mac_64_cross_builds.py balrog/production.py'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       #TODO: bug 1164617 - remove Docker image hacks
       LIBRARY_PATH: ""
       CPLUS_INCLUDE_PATH: ""
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
-      DIST_UPLOADS: 'jsshell-mac64.zip'
-      DIST_TARGET_UPLOADS: 'mac64.dmg mac64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
 
   extra:
     treeherderEnv:
       - production