Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 11 Sep 2015 11:35:00 -0400
changeset 262597 cd7b4b4b747485d59c9fb07cd31e17afac87f70b
parent 262556 7abc5253e8cf95a869a114604fdecdcf569c8d2e
child 262598 60ed4fafa41e63dc1855bd64830a9c00bcea0b83
push id29378
push userkwierso@gmail.com
push dateWed, 16 Sep 2015 00:16:30 +0000
treeherdermozilla-central@df10a3f6060f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1198179
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1198179: in taskcluster, have 'make upload' just copy; r=ted.mielczarek
build/upload.py
testing/mozharness/configs/builds/build_pool_specifics.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/builds/linux64_clobber.yml
testing/taskcluster/tasks/builds/opt_macosx64.yml
--- a/build/upload.py
+++ b/build/upload.py
@@ -10,16 +10,20 @@
 # UPLOAD_USER    : username on that host
 #  and one of the following:
 # UPLOAD_PATH    : path on that host to put the files in
 # UPLOAD_TO_TEMP : upload files to a new temporary directory
 #
 # If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out
 # the properties file.
 #
+# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH.
+# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no
+# properties are written out.
+#
 # And will use the following optional environment variables if set:
 # UPLOAD_SSH_KEY : path to a ssh private key to use
 # UPLOAD_PORT    : port to use for ssh
 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
 #                  UPLOAD_PATH and the full paths of all files uploaded will
 #                  be appended to the commandline.
 #
 # All files to be uploaded should be passed as commandline arguments to this
@@ -27,16 +31,17 @@
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 import re
 import json
 import errno
 import hashlib
+import shutil
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
 
 def OptionalEnvironmentVariable(v):
     """Return the value of the environment variable named v, or None
     if it's unset (or empty)."""
     if v in os.environ and os.environ[v] != "":
@@ -101,17 +106,17 @@ def DoSCPFile(file, remote_path, user, h
     cmdline.extend([WindowsPathToMsysPath(file),
                     "%s@%s:%s" % (user, host, remote_path)])
     with redo.retrying(check_call, sleeptime=10) as f:
         f(cmdline)
         return
 
     raise Exception("Command %s returned non-zero exit code" % cmdline)
 
-def GetRemotePath(path, local_file, base_path):
+def GetBaseRelativePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
@@ -209,17 +214,17 @@ def UploadFiles(user, host, path, files,
     remote_files = []
     properties = {}
     try:
         for file in files:
             file = os.path.abspath(file)
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
-            remote_path = GetRemotePath(path, file, base_path)
+            remote_path = GetBaseRelativePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
@@ -231,16 +236,35 @@ def UploadFiles(user, host, path, files,
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
         print "Upload complete"
     return properties
 
+def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
+    """Copy each file in the list of files to `path`.  The `base_path` argument is treated
+    as it is by UploadFiles."""
+    if not path.endswith("/"):
+        path += "/"
+    if base_path is not None:
+        base_path = os.path.abspath(base_path)
+    for file in files:
+        file = os.path.abspath(file)
+        if not os.path.isfile(file):
+            raise IOError("File not found: %s" % file)
+        # first ensure that path exists remotely
+        target_path = GetBaseRelativePath(path, file, base_path)
+        if not os.path.exists(target_path):
+            os.makedirs(target_path)
+        if verbose:
+            print "Copying " + file + " to " + target_path
+        shutil.copy(file, target_path)
+
 def WriteProperties(files, properties_file, url_properties, package):
     properties = url_properties
     for file in files:
         if file.endswith('.complete.mar'):
             properties.update(GetMarProperties(file))
     with open(properties_file, 'w') as outfile:
         properties['packageFilename'] = package
         properties['uploadFiles'] = [os.path.abspath(f) for f in files]
@@ -275,21 +299,35 @@ if __name__ == '__main__':
                       help="Name of the main package.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print "You must specify at least one file to upload"
         sys.exit(1)
     if not options.properties_file:
         print "You must specify a --properties-file"
         sys.exit(1)
+
+    if host == "localhost":
+        if upload_to_temp_dir:
+            print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost"
+            sys.exit(1)
+        if post_upload_command:
+            # POST_UPLOAD_COMMAND is difficult to extract from the mozharness
+            # scripts, so just ignore it until it's no longer used anywhere
+            print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost"
+
     try:
-        url_properties = UploadFiles(user, host, path, args, base_path=options.base_path,
-                                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
-                                     post_upload_command=post_upload_command,
-                                     package=options.package,
-                                     verbose=True)
-        WriteProperties(args, options.properties_file, url_properties, options.package)
+        if host == "localhost":
+            CopyFilesLocally(path, args, base_path=options.base_path,
+                             package=options.package,
+                             verbose=True)
+        else:
+
+            url_properties = UploadFiles(user, host, path, args,
+                                         base_path=options.base_path, port=port, ssh_key=key,
+                                         upload_to_temp_dir=upload_to_temp_dir,
+                                         post_upload_command=post_upload_command,
+                                         package=options.package, verbose=True)
+
+            WriteProperties(args, options.properties_file, url_properties, options.package)
     except IOError, (strerror):
         print strerror
         sys.exit(1)
-    except Exception, (err):
-        print err
-        sys.exit(2)
--- a/testing/mozharness/configs/builds/build_pool_specifics.py
+++ b/testing/mozharness/configs/builds/build_pool_specifics.py
@@ -33,14 +33,18 @@ config = {
         'symbol_server_host': "symbolpush.mozilla.org",
         'stage_server': 'stage.mozilla.org',
         "sendchange_masters": ["buildbot-master81.build.mozilla.org:9301"],
         'taskcluster_index': 'index',
     },
     "taskcluster": {
         'graph_server': 'graphs.mozilla.org',
         'symbol_server_host': "symbolpush.mozilla.org",
-        'stage_server': 'stage.mozilla.org',
+        'stage_server': 'ignored',
         # use the relengapi proxy to talk to tooltool
         "tooltool_servers": ['http://relengapi/tooltool/'],
         "tooltool_url": 'http://relengapi/tooltool/',
+        'upload_env': {
+            'UPLOAD_HOST': 'localhost',
+            'UPLOAD_PATH': '/home/worker/artifacts',
+        },
     },
 }
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -821,25 +821,28 @@ or run without that action (ie: --no-{ac
 
     def query_mach_build_env(self, multiLocale=None):
         c = self.config
         if multiLocale is None and self.query_is_nightly():
             multiLocale = c.get('multi_locale', False)
         mach_env = {}
         if c.get('upload_env'):
             mach_env.update(c['upload_env'])
-            mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
-                'stage_server': c['stage_server']
-            }
-            mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
-                'stage_username': c['stage_username']
-            }
-            mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
-                'stage_ssh_key': c['stage_ssh_key']
-            }
+            if 'UPLOAD_HOST' in mach_env:
+                mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
+                    'stage_server': c['stage_server']
+                }
+            if 'UPLOAD_USER' in mach_env:
+                mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
+                    'stage_username': c['stage_username']
+                }
+            if 'UPLOAD_SSH_KEY' in mach_env:
+                mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
+                    'stage_ssh_key': c['stage_ssh_key']
+                }
 
         if self.query_is_nightly():
             mach_env['LATEST_MAR_DIR'] = c['latest_mar_dir'] % {
                 'branch': self.branch
             }
 
         # _query_post_upload_cmd returns a list (a cmd list), for env sake here
         # let's make it a string
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -20,35 +20,27 @@ echo "running as" $(id)
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
-# files to be "uploaded" (moved to ~/artifacts) from obj-firefox/dist
-: DIST_UPLOADS                  ${DIST_UPLOADS:=""}
-# files which will be be prefixed with target before being sent to artifacts
-# e.g. DIST_TARGET_UPLOADS="a.zip" runs mv v2.0.a.zip mv artifacts/target.a.zip
-: DIST_TARGET_UPLOADS           ${DIST_TARGET_UPLOADS:=""}
-
 set -v
 
-# Don't run the upload step; this is passed through mozharness to mach.  Once
-# the mozharness scripts are not run in Buildbot anymore, this can be moved to
-# Mozharness (or the upload step removed from mach entirely)
-export MOZ_AUTOMATION_UPLOAD=0
-
 export MOZ_CRASHREPORTER_NO_REPORT=1
 export MOZ_OBJDIR=obj-firefox
 export MOZ_SYMBOLS_EXTRA_BUILDID=linux64
-export POST_SYMBOL_UPLOAD_CMD=/usr/local/bin/post-symbol-upload.py
 export TINDERBOX_OUTPUT=1
 
+# use "simple" package names so that they can be hard-coded in the task's
+# extras.locations
+export MOZ_SIMPLE_PACKAGE_NAME=target
+
 # Ensure that in tree libraries can be found
 export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
 
 # test required parameters are supplied
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then exit 1; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then exit 1; fi
 
 cleanup() {
@@ -131,25 +123,8 @@ python2.7 $WORKSPACE/build/src/testing/$
   --no-update \
   --no-upload-files \
   --no-sendchange \
   --log-level=debug \
   --work-dir=$WORKSPACE/build \
   --no-action=generate-build-stats \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}
-
-mkdir -p /home/worker/artifacts
-
-# upload auxiliary files
-cd $WORKSPACE/build/src/obj-firefox/dist
-
-for file in $DIST_UPLOADS
-do
-    mv $file $HOME/artifacts/$file
-done
-
-# Discard version numbers from packaged files, they just make it hard to write
-# the right filename in the task payload where artifacts are declared
-for file in $DIST_TARGET_UPLOADS
-do
-    mv *.$file $HOME/artifacts/target.$file
-done
--- a/testing/taskcluster/tasks/builds/linux64_clobber.yml
+++ b/testing/taskcluster/tasks/builds/linux64_clobber.yml
@@ -25,29 +25,27 @@ task:
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_base_linux_64_builds.py balrog/production.py'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
       NEED_XVFB: true
-      DIST_UPLOADS: 'jsshell-linux-x86_64.zip'
-      DIST_TARGET_UPLOADS: 'x-test.linux-x86_64.tar.bz2 linux-x86_64.tar.bz2 linux-x86_64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
 
   extra:
     treeherderEnv:
       - production
       - staging
     treeherder:
       machine:
         # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
         platform: linux64
     # Rather then enforcing particular conventions we require that all build
     # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
+    # files are located, relative to the task's artifacts URL
     locations:
-      build: 'public/build/target.linux-x86_64.tar.bz2'
+      build: 'public/build/target.tar.bz2'
       tests: 'public/build/target.tests.zip'
--- a/testing/taskcluster/tasks/builds/opt_macosx64.yml
+++ b/testing/taskcluster/tasks/builds/opt_macosx64.yml
@@ -34,18 +34,16 @@ task:
       MOZHARNESS_CONFIG: 'builds/releng_base_mac_64_cross_builds.py balrog/production.py'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       #TODO: bug 1164617 - remove Docker image hacks
       LIBRARY_PATH: ""
       CPLUS_INCLUDE_PATH: ""
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
-      DIST_UPLOADS: 'jsshell-mac64.zip'
-      DIST_TARGET_UPLOADS: 'mac64.dmg mac64.json tests.zip crashreporter-symbols.zip'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
 
   extra:
     treeherderEnv:
       - production