Bug 1455127 - Remove functionality for uploading to a server; r=ted
authorGregory Szorc <gps@mozilla.com>
Thu, 19 Apr 2018 09:30:31 -0700
changeset 468023 64ec147eb854e6beefd2efae5e465de99bbfe97d
parent 468022 f7f47849e748a4509be6005d060be7e09f537495
child 468024 ac6e19238b99b775fbe888ac56d58fdabfa6495b
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1455127
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1455127 - Remove functionality for uploading to a server; r=ted With the transition to Taskcluster, "uploads" are artifacts in a local directory. So we don't need to support uploading to a remote server using SCP. This commit removes all the code to support uploading to a remote server. And since property files were only written out for the remote case, all that code can be deleted as well. Since UPLOAD_HOST no longer means anything, we no longer set it in mozharness configs. MozReview-Commit-ID: 66gkM8erkGk
build/upload.py
testing/mozharness/configs/builds/build_pool_specifics.py
testing/mozharness/configs/builds/taskcluster_base_windows.py
testing/mozharness/configs/single_locale/tc_android-api-16.py
testing/mozharness/configs/single_locale/tc_linux32.py
testing/mozharness/configs/single_locale/tc_linux64.py
testing/mozharness/configs/single_locale/tc_macosx64.py
testing/mozharness/configs/single_locale/tc_win32.py
testing/mozharness/configs/single_locale/tc_win64.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -1,57 +1,29 @@
 #!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # When run directly, this script expects the following environment variables
 # to be set:
-# UPLOAD_HOST    : host to upload files to
-# UPLOAD_USER    : username on that host
-#  and one of the following:
 # UPLOAD_PATH    : path on that host to put the files in
-# UPLOAD_TO_TEMP : upload files to a new temporary directory
-#
-# If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out
-# the properties file.
 #
-# If UPLOAD_HOST is "localhost", then files are simply copied to UPLOAD_PATH.
-# In this case, UPLOAD_TO_TEMP and POST_UPLOAD_CMD are not supported, and no
-# properties are written out.
-#
-# And will use the following optional environment variables if set:
-# UPLOAD_SSH_KEY : path to a ssh private key to use
-# UPLOAD_PORT    : port to use for ssh
-# POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
-#                  UPLOAD_PATH and the full paths of all files uploaded will
-#                  be appended to the commandline.
+# Files are simply copied to UPLOAD_PATH.
 #
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys
 import os
-import re
-import json
-import errno
-import hashlib
 import shutil
 from optparse import OptionParser
-from subprocess import (
-    check_call,
-    check_output,
-    STDOUT,
-    CalledProcessError,
-)
-import concurrent.futures as futures
-import redo
 
 
 def OptionalEnvironmentVariable(v):
     """Return the value of the environment variable named v, or None
     if it's unset (or empty)."""
     if v in os.environ and os.environ[v] != "":
         return os.environ[v]
     return None
@@ -68,244 +40,30 @@ def FixupMsysPath(path):
         if 'SHELL' in os.environ:
             sh = os.environ['SHELL']
             msys = sh[:sh.find('/bin')]
             if path.startswith(msys):
                 path = path[len(msys):]
     return path
 
 
-def WindowsPathToMsysPath(path):
-    """Translate a Windows pathname to an MSYS pathname.
-    Necessary because we call out to ssh/scp, which are MSYS binaries
-    and expect MSYS paths."""
-    # If we're not on Windows, or if we already have an MSYS path (starting
-    # with '/' instead of 'c:' or something), then just return.
-    if sys.platform != 'win32' or path.startswith('/'):
-        return path
-    (drive, path) = os.path.splitdrive(os.path.abspath(path))
-    return "/" + drive[0] + path.replace('\\', '/')
-
-
-def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
-    """Given optional port and ssh key values, append valid OpenSSH
-    commandline arguments to the list cmdline if the values are not None."""
-    if port is not None:
-        cmdline.append("-P%d" % port)
-    if ssh_key is not None:
-        # Don't interpret ~ paths - ssh can handle that on its own
-        if not ssh_key.startswith('~'):
-            ssh_key = WindowsPathToMsysPath(ssh_key)
-        cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
-    # In case of an issue here we don't want to hang on a password prompt.
-    cmdline.extend(["-o", "BatchMode=yes"])
-
-
-def DoSSHCommand(command, user, host, port=None, ssh_key=None):
-    """Execute command on user@host using ssh. Optionally use
-    port and ssh_key, if provided."""
-    cmdline = ["ssh"]
-    AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
-    cmdline.extend(["%s@%s" % (user, host), command])
-
-    with redo.retrying(check_output, sleeptime=10) as f:
-        try:
-            output = f(cmdline, stderr=STDOUT).strip()
-        except CalledProcessError as e:
-            print("failed ssh command output:")
-            print('=' * 20)
-            print(e.output)
-            print('=' * 20)
-            raise
-        return output
-
-    raise Exception("Command %s returned non-zero exit code" % cmdline)
-
-
-def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None,
-              log=False):
-    """Upload file to user@host:remote_path using scp. Optionally use
-    port and ssh_key, if provided."""
-    if log:
-        print('Uploading %s' % file)
-    cmdline = ["scp"]
-    AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
-    cmdline.extend([WindowsPathToMsysPath(file),
-                    "%s@%s:%s" % (user, host, remote_path)])
-    with redo.retrying(check_call, sleeptime=10) as f:
-        f(cmdline)
-        return
-
-    raise Exception("Command %s returned non-zero exit code" % cmdline)
-
-
 def GetBaseRelativePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
-        # Hack to work around OSX uploading the i386 SDK from i386/dist. Both
-        # the i386 SDK and x86-64 SDK end up in the same directory this way.
-        if base_path.endswith('/x86_64/dist'):
-            return GetBaseRelativePath(path, local_file, base_path.replace('/x86_64/', '/i386/'))
         return path
+
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
     dir = dir[len(base_path) + 1:].replace('\\', '/')
     return path + dir
 
 
-def GetFileHashAndSize(filename):
-    sha512Hash = 'UNKNOWN'
-    size = 'UNKNOWN'
-
-    try:
-        # open in binary mode to make sure we get consistent results
-        # across all platforms
-        with open(filename, "rb") as f:
-            shaObj = hashlib.sha512(f.read())
-            sha512Hash = shaObj.hexdigest()
-
-        size = os.path.getsize(filename)
-    except Exception:
-        raise Exception("Unable to get filesize/hash from file: %s" % filename)
-
-    return (sha512Hash, size)
-
-
-def GetMarProperties(filename):
-    if not os.path.exists(filename):
-        return {}
-    (mar_hash, mar_size) = GetFileHashAndSize(filename)
-    return {
-        'completeMarFilename': os.path.basename(filename),
-        'completeMarSize': mar_size,
-        'completeMarHash': mar_hash,
-    }
-
-
-def GetUrlProperties(output, package):
-    # let's create a switch case using name-spaces/dict
-    # rather than a long if/else with duplicate code
-    property_conditions = [
-        # key: property name, value: condition
-        ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
-         m.endswith('crashreporter-symbols-full.zip')),
-        ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
-        ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
-        ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
-        ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
-        ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
-        ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
-        ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
-        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
-        ('packageUrl', lambda m: m.endswith(package)),
-    ]
-    url_re = re.compile(
-        r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
-    properties = {}
-
-    try:
-        for line in output.splitlines():
-            m = url_re.match(line.strip())
-            if m:
-                m = m.group(1)
-                for prop, condition in property_conditions:
-                    if condition(m):
-                        properties.update({prop: m})
-                        break
-    except IOError as e:
-        if e.errno != errno.ENOENT:
-            raise
-        properties = {prop: 'UNKNOWN' for prop, condition
-                      in property_conditions}
-    return properties
-
-
-def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None,
-                upload_to_temp_dir=False, post_upload_command=None, package=None):
-    """Upload each file in the list files to user@host:path. Optionally pass
-    port and ssh_key to the ssh commands. If base_path is not None, upload
-    files including their path relative to base_path. If upload_to_temp_dir is
-    True files will be uploaded to a temporary directory on the remote server.
-    Generally, you should have a post upload command specified in these cases
-    that can move them around to their correct location(s).
-    If post_upload_command is not None, execute that command on the remote host
-    after uploading all files, passing it the upload path, and the full paths to
-    all files uploaded.
-    If verbose is True, print status updates while working."""
-    if not host or not user:
-        return {}
-    if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
-        print("One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be defined.")
-        sys.exit(1)
-
-    if upload_to_temp_dir:
-        path = DoSSHCommand("mktemp -d", user, host,
-                            port=port, ssh_key=ssh_key)
-    if not path.endswith("/"):
-        path += "/"
-    if base_path is not None:
-        base_path = os.path.abspath(base_path)
-    remote_files = []
-    properties = {}
-
-    def get_remote_path(p):
-        return GetBaseRelativePath(path, os.path.abspath(p), base_path)
-
-    try:
-        # Do a pass to find remote directories so we don't perform excessive
-        # scp calls.
-        remote_paths = set()
-        for file in files:
-            if not os.path.isfile(file):
-                raise IOError("File not found: %s" % file)
-
-            remote_paths.add(get_remote_path(file))
-
-        # If we wanted to, we could reduce the remote paths if they are a parent
-        # of any entry.
-        for p in sorted(remote_paths):
-            DoSSHCommand("mkdir -p " + p, user, host,
-                         port=port, ssh_key=ssh_key)
-
-        with futures.ThreadPoolExecutor(4) as e:
-            fs = []
-            # Since we're uploading in parallel, the largest file should take
-            # the longest to upload. So start it first.
-            for file in sorted(files, key=os.path.getsize, reverse=True):
-                remote_path = get_remote_path(file)
-                fs.append(e.submit(DoSCPFile, file, remote_path, user, host,
-                                   port=port, ssh_key=ssh_key, log=verbose))
-                remote_files.append(remote_path + '/' + os.path.basename(file))
-
-            # We need to call result() on the future otherwise exceptions could
-            # get swallowed.
-            for f in futures.as_completed(fs):
-                f.result()
-
-        if post_upload_command is not None:
-            if verbose:
-                print("Running post-upload command: " + post_upload_command)
-            file_list = '"' + '" "'.join(remote_files) + '"'
-            output = DoSSHCommand('%s "%s" %s' % (
-                post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
-            # We print since mozharness may parse URLs from the output stream.
-            print(output)
-            properties = GetUrlProperties(output, package)
-    finally:
-        if upload_to_temp_dir:
-            DoSSHCommand("rm -rf %s" % path, user, host, port=port,
-                         ssh_key=ssh_key)
-    if verbose:
-        print("Upload complete")
-    return properties
-
-
 def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
     """Copy each file in the list of files to `path`.  The `base_path` argument is treated
     as it is by UploadFiles."""
     if not path.endswith("/"):
         path += "/"
     if base_path is not None:
         base_path = os.path.abspath(base_path)
     for file in files:
@@ -316,43 +74,22 @@ def CopyFilesLocally(path, files, verbos
         target_path = GetBaseRelativePath(path, file, base_path)
         if not os.path.exists(target_path):
             os.makedirs(target_path)
         if verbose:
             print("Copying " + file + " to " + target_path)
         shutil.copy(file, target_path)
 
 
-def WriteProperties(files, properties_file, url_properties, package):
-    properties = url_properties
-    for file in files:
-        if file.endswith('.complete.mar'):
-            properties.update(GetMarProperties(file))
-    with open(properties_file, 'w') as outfile:
-        properties['packageFilename'] = package
-        properties['uploadFiles'] = [os.path.abspath(f) for f in files]
-        json.dump(properties, outfile, indent=4)
-
-
 if __name__ == '__main__':
-    host = OptionalEnvironmentVariable('UPLOAD_HOST')
-    user = OptionalEnvironmentVariable('UPLOAD_USER')
     path = OptionalEnvironmentVariable('UPLOAD_PATH')
-    upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
-    port = OptionalEnvironmentVariable('UPLOAD_PORT')
-    if port is not None:
-        port = int(port)
-    key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
-    post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
 
     if sys.platform == 'win32':
         if path is not None:
             path = FixupMsysPath(path)
-        if post_upload_command is not None:
-            post_upload_command = FixupMsysPath(post_upload_command)
 
     parser = OptionParser(usage="usage: %prog [options] <files>")
     parser.add_option("-b", "--base-path",
                       action="store",
                       help="Preserve file paths relative to this path when uploading. "
                       "If unset, all files will be uploaded directly to UPLOAD_PATH.")
     parser.add_option("--properties-file",
                       action="store",
@@ -363,35 +100,15 @@ if __name__ == '__main__':
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print("You must specify at least one file to upload")
         sys.exit(1)
     if not options.properties_file:
         print("You must specify a --properties-file")
         sys.exit(1)
 
-    if host == "localhost":
-        if upload_to_temp_dir:
-            print("Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost")
-            sys.exit(1)
-        if post_upload_command:
-            # POST_UPLOAD_COMMAND is difficult to extract from the mozharness
-            # scripts, so just ignore it until it's no longer used anywhere
-            print("Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost")
-
     try:
-        if host == "localhost":
-            CopyFilesLocally(path, args, base_path=options.base_path,
-                             package=options.package,
-                             verbose=True)
-        else:
-
-            url_properties = UploadFiles(user, host, path, args,
-                                         base_path=options.base_path, port=port, ssh_key=key,
-                                         upload_to_temp_dir=upload_to_temp_dir,
-                                         post_upload_command=post_upload_command,
-                                         package=options.package, verbose=True)
-
-            WriteProperties(args, options.properties_file,
-                            url_properties, options.package)
+        CopyFilesLocally(path, args, base_path=options.base_path,
+                         package=options.package,
+                         verbose=True)
     except IOError as strerror:
         print(strerror)
         sys.exit(1)
--- a/testing/mozharness/configs/builds/build_pool_specifics.py
+++ b/testing/mozharness/configs/builds/build_pool_specifics.py
@@ -2,13 +2,12 @@
 # fx build factories are ported, we might deal with this differently
 
 config = {
     "taskcluster": {
         # use the relengapi proxy to talk to tooltool
         "tooltool_servers": ['http://relengapi/tooltool/'],
         "tooltool_url": 'http://relengapi/tooltool/',
         'upload_env': {
-            'UPLOAD_HOST': 'localhost',
             'UPLOAD_PATH': '/builds/worker/artifacts',
         },
     },
 }
--- a/testing/mozharness/configs/builds/taskcluster_base_windows.py
+++ b/testing/mozharness/configs/builds/taskcluster_base_windows.py
@@ -20,12 +20,11 @@ config = {
         'MOZ_CRASHREPORTER_NO_REPORT': '1',
         'MOZ_OBJDIR': '%(abs_obj_dir)s',
         'TINDERBOX_OUTPUT': '1',
         'TOOLTOOL_CACHE': 'c:/builds/tooltool_cache',
         'TOOLTOOL_HOME': '/c/builds',
         'MSYSTEM': 'MINGW32',
     },
     'upload_env': {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
     },
 }
--- a/testing/mozharness/configs/single_locale/tc_android-api-16.py
+++ b/testing/mozharness/configs/single_locale/tc_android-api-16.py
@@ -5,17 +5,16 @@ config = {
     "mozconfig": "src/mobile/android/config/mozconfigs/android-api-16/l10n-nightly",
     "tooltool_config": {
         "manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
         "output_dir": "%(abs_work_dir)s/src",
     },
     "tooltool_servers": ['http://relengapi/tooltool/'],
 
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': '/builds/worker/artifacts/',
     },
     'secret_files': [
         {'filename': '/builds/gapi.data',
          'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
          'min_scm_level': 1},
         {'filename': '/builds/mozilla-fennec-geoloc-api.key',
          'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key',
--- a/testing/mozharness/configs/single_locale/tc_linux32.py
+++ b/testing/mozharness/configs/single_locale/tc_linux32.py
@@ -10,12 +10,11 @@ config = {
         "MOZ_OBJDIR": "obj-firefox",
         "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "L10NBASEDIR": "../../l10n",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': '/builds/worker/artifacts/',
     },
 }
--- a/testing/mozharness/configs/single_locale/tc_linux64.py
+++ b/testing/mozharness/configs/single_locale/tc_linux64.py
@@ -10,12 +10,11 @@ config = {
         "MOZ_OBJDIR": "obj-firefox",
         "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "L10NBASEDIR": "../../l10n",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': '/builds/worker/artifacts/',
     },
 }
--- a/testing/mozharness/configs/single_locale/tc_macosx64.py
+++ b/testing/mozharness/configs/single_locale/tc_macosx64.py
@@ -10,15 +10,14 @@ config = {
         "MOZ_OBJDIR": "obj-firefox",
         "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "L10NBASEDIR": "../../l10n",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': '/builds/worker/artifacts/',
     },
 
     "tooltool_url": 'http://relengapi/tooltool/',
 }
 
--- a/testing/mozharness/configs/single_locale/tc_win32.py
+++ b/testing/mozharness/configs/single_locale/tc_win32.py
@@ -17,15 +17,14 @@ config = {
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "L10NBASEDIR": "../../l10n",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE', 'c:/builds/tooltool_cache'),
         'EN_US_PACKAGE_NAME': 'target.zip',
         'EN_US_PKG_INST_BASENAME': 'target.installer',
     },
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
     },
 
     "tooltool_url": 'https://tooltool.mozilla-releng.net/',
     'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
 }
--- a/testing/mozharness/configs/single_locale/tc_win64.py
+++ b/testing/mozharness/configs/single_locale/tc_win64.py
@@ -17,16 +17,15 @@ config = {
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "L10NBASEDIR": "../../l10n",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE', 'c:/builds/tooltool_cache'),
         'EN_US_PACKAGE_NAME': 'target.zip',
         'EN_US_PKG_INST_BASENAME': 'target.installer',
     },
     "upload_env": {
-        'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
     },
 
     "tooltool_url": 'https://tooltool.mozilla-releng.net/',
     'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
 }