Backed out 8 changesets (bug 1512285, bug 1512188) for fetch bustages CLOSED TREE
authorBogdan Tara <btara@mozilla.com>
Sat, 29 Dec 2018 07:56:56 +0200
changeset 509218 d506757ef6a3eb411066b13e38f0e5bb5e18f403
parent 509217 5e7aa7d98012c6a21268232515f7ae2a090ee375
child 509219 dd13046c9317521aca5c2e3459df99419d3949f3
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1512285, 1512188
milestone66.0a1
backs out5e7aa7d98012c6a21268232515f7ae2a090ee375
55874a956ae19f7052f95e5d2bb5bf8b8da174e3
ac6e2cd6eb32af57cf40b5f917ae7874804dc325
f68df074eac1a35790f30bde6bdcbf5760d05f19
f01d2662582fdecb02cf0dedbaa0a616c7964c99
35d74dd16706248fc969a90429615942111cc7c6
9e95b9bf1201d14379e750d0156a63dd32488662
ac4a49eb95f50cbcdf20adf8d5e01d127162749e
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 8 changesets (bug 1512285, bug 1512188) for fetch bustages CLOSED TREE Backed out changeset 5e7aa7d98012 (bug 1512188) Backed out changeset 55874a956ae1 (bug 1512188) Backed out changeset ac6e2cd6eb32 (bug 1512188) Backed out changeset f68df074eac1 (bug 1512188) Backed out changeset f01d2662582f (bug 1512188) Backed out changeset 35d74dd16706 (bug 1512285) Backed out changeset 9e95b9bf1201 (bug 1512188) Backed out changeset ac4a49eb95f5 (bug 1512188)
taskcluster/docker/image_builder/build-image.sh
taskcluster/scripts/run-task
taskcluster/taskgraph/transforms/job/hazard.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/docker/image_builder/build-image.sh
+++ b/taskcluster/docker/image_builder/build-image.sh
@@ -33,18 +33,18 @@ if [ -n "$DOCKER_IMAGE_PARENT" ]; then
     LOAD_COMMAND="\
       /builds/worker/checkouts/gecko/mach taskcluster-load-image \
       --task-id \"$DOCKER_IMAGE_PARENT_TASK\" \
       -t \"$DOCKER_IMAGE_PARENT\" && "
 fi
 
 # Build image
 run-task \
-  --gecko-checkout "/builds/worker/checkouts/gecko" \
-  --gecko-sparse-profile build/sparse-profiles/docker-image \
+  --vcs-checkout "/builds/worker/checkouts/gecko" \
+  --sparse-profile build/sparse-profiles/docker-image \
   -- \
   sh -x -c "$LOAD_COMMAND \
   /builds/worker/checkouts/gecko/mach taskcluster-build-image \
   -t \"$IMAGE_NAME:$HASH\" \
   \"$IMAGE_NAME\""
 
 # Create artifact folder (note that this must occur after run-task)
 mkdir -p /builds/worker/workspace/artifacts
--- a/taskcluster/scripts/run-task
+++ b/taskcluster/scripts/run-task
@@ -497,83 +497,16 @@ def fetch_artifacts():
         print(FETCH_CONTENT_NOT_FOUND)
         sys.exit(1)
 
     cmd = [fetch_content, 'task-artifacts']
     subprocess.run(cmd, check=True, env=os.environ)
     print_line(b'fetches', b'finished fetching artifacts\n')
 
 
-def add_vcs_arguments(parser, project, name):
-    """Adds arguments to ArgumentParser to control VCS options for a project."""
-
-    parser.add_argument('--%s-checkout' % project,
-                        help='Directory where %s checkout should be created' %
-                             name)
-    parser.add_argument('--%s-sparse-profile' % project,
-                        help='Path to sparse profile for %s checkout' % name)
-
-
-def collect_vcs_options(args, project):
-    checkout = getattr(args, '%s_checkout' % project)
-    sparse_profile = getattr(args, '%s_sparse_profile' % project)
-
-    env_prefix = project.upper()
-
-    base_repo = os.environ.get('%s_BASE_REPOSITORY' % env_prefix)
-    head_repo = os.environ.get('%s_HEAD_REPOSITORY' % env_prefix)
-    revision = os.environ.get('%s_HEAD_REV' % env_prefix)
-    branch = os.environ.get('%s_HEAD_REF' % env_prefix)
-
-    store_path = os.environ['HG_STORE_PATH']
-
-    # Expand ~ in some paths.
-    if checkout:
-        checkout = os.path.expanduser(checkout)
-    store_path = os.path.expanduser(store_path)
-
-    # Some callers set the base repository to mozilla-central for historical
-    # reasons. Switch to mozilla-unified because robustcheckout works best
-    # with it.
-    if base_repo == 'https://hg.mozilla.org/mozilla-central':
-        base_repo = 'https://hg.mozilla.org/mozilla-unified'
-
-    return {
-        'store-path': store_path,
-        'project': project,
-        'env-prefix': env_prefix,
-        'checkout': checkout,
-        'sparse-profile': sparse_profile,
-        'base-repo': base_repo,
-        'head-repo': head_repo,
-        'revision': revision,
-        'branch': branch,
-    }
-
-
-def vcs_checkout_from_args(args, project):
-    options = collect_vcs_options(args, project)
-
-    if not options['checkout']:
-        if options['branch'] and not options['revision']:
-            print('task should be defined in terms of non-symbolic revision')
-            sys.exit(1)
-        return
-
-    os.environ['%s_HEAD_REV' % options['env-prefix']] = vcs_checkout(
-        options['head-repo'],
-        options['checkout'],
-        options['store-path'],
-        base_repo=options['base-repo'],
-        revision=options['revision'],
-        fetch_hgfingerprint=args.fetch_hgfingerprint,
-        branch=options['branch'],
-        sparse_profile=options['sparse-profile'])
-
-
 def main(args):
     print_line(b'setup', b'run-task started in %s\n' % os.getcwd().encode('utf-8'))
     running_as_root = IS_POSIX and os.getuid() == 0
 
     # Arguments up to '--' are ours. After are for the main task
     # to be executed.
     try:
         i = args.index('--')
@@ -581,26 +514,38 @@ def main(args):
         task_args = args[i + 1:]
     except ValueError:
         our_args = args
         task_args = []
 
     parser = argparse.ArgumentParser()
     parser.add_argument('--user', default='worker', help='user to run as')
     parser.add_argument('--group', default='worker', help='group to run as')
-
-    add_vcs_arguments(parser, 'gecko', 'Firefox')
-    add_vcs_arguments(parser, 'comm', 'Comm')
-
+    parser.add_argument('--vcs-checkout',
+                        help='Directory where Gecko checkout should be created')
+    parser.add_argument('--sparse-profile',
+                        help='Path to sparse checkout profile to use')
+    parser.add_argument('--comm-checkout',
+                        help='Directory where Comm checkout should be created')
+    parser.add_argument('--tools-checkout',
+                        help='Directory where build/tools checkout should be created')
     parser.add_argument('--fetch-hgfingerprint', action='store_true',
                         help='Fetch the latest hgfingerprint from the secrets store, '
                         'using the taskclsuerProxy')
 
     args = parser.parse_args(our_args)
 
+    # expand ~ in some paths
+    if args.vcs_checkout:
+        args.vcs_checkout = os.path.expanduser(args.vcs_checkout)
+    if args.tools_checkout:
+        args.tools_checkout = os.path.expanduser(args.tools_checkout)
+    if 'HG_STORE_PATH' in os.environ:
+        os.environ['HG_STORE_PATH'] = os.path.expanduser(os.environ['HG_STORE_PATH'])
+
     uid = gid = gids = None
     if IS_POSIX and running_as_root:
         user, group, gids = get_posix_user_group(args.user, args.group)
         uid = user.pw_uid
         gid = group.gr_gid
 
     if running_as_root and os.path.exists("/dev/kvm"):
         # Ensure kvm permissions for worker, required for Android x86
@@ -720,35 +665,90 @@ def main(args):
             os.makedirs(store_path)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         if running_as_root:
             os.chown(store_path, uid, gid)
 
-    prepare_checkout_dir(args.gecko_checkout)
-    if args.gecko_checkout or args.comm_checkout:
+    prepare_checkout_dir(args.vcs_checkout)
+    prepare_checkout_dir(args.tools_checkout)
+    if args.vcs_checkout or args.tools_checkout or args.comm_checkout:
         prepare_hg_store_path()
 
     if IS_POSIX and running_as_root:
         # Drop permissions to requested user.
         # This code is modeled after what `sudo` was observed to do in a Docker
         # container. We do not bother calling setrlimit() because containers have
         # their own limits.
         print_line(b'setup', b'running as %s:%s\n' % (
             args.user.encode('utf-8'), args.group.encode('utf-8')))
 
         os.setgroups(gids)
         os.umask(0o22)
         os.setresgid(gid, gid, gid)
         os.setresuid(uid, uid, uid)
 
-    vcs_checkout_from_args(args, 'gecko')
-    vcs_checkout_from_args(args, 'comm')
+    # Checkout the repository, setting the GECKO_HEAD_REV to the current
+    # revision hash. Revision hashes have priority over symbolic revisions. We
+    # disallow running tasks with symbolic revisions unless they have been
+    # resolved by a checkout.
+    if args.vcs_checkout:
+        base_repo = os.environ.get('GECKO_BASE_REPOSITORY')
+        # Some callers set the base repository to mozilla-central for historical
+        # reasons. Switch to mozilla-unified because robustcheckout works best
+        # with it.
+        if base_repo == 'https://hg.mozilla.org/mozilla-central':
+            base_repo = 'https://hg.mozilla.org/mozilla-unified'
+
+        os.environ['GECKO_HEAD_REV'] = vcs_checkout(
+            os.environ['GECKO_HEAD_REPOSITORY'],
+            args.vcs_checkout,
+            os.environ['HG_STORE_PATH'],
+            fetch_hgfingerprint=args.fetch_hgfingerprint,
+            base_repo=base_repo,
+            revision=os.environ.get('GECKO_HEAD_REV'),
+            branch=os.environ.get('GECKO_HEAD_REF'),
+            sparse_profile=args.sparse_profile)
+
+    elif not os.environ.get('GECKO_HEAD_REV') and \
+            os.environ.get('GECKO_HEAD_REF'):
+        print('task should be defined in terms of non-symbolic revision')
+        return 1
+
+    if args.tools_checkout:
+        vcs_checkout('https://hg.mozilla.org/build/tools',
+                     args.tools_checkout,
+                     os.environ['HG_STORE_PATH'],
+                     fetch_hgfingerprint=args.fetch_hgfingerprint,
+                     # Always check out the latest commit on default branch.
+                     # This is non-deterministic!
+                     branch='default')
+
+    # Checkout the repository, setting the COMM_HEAD_REV to the current
+    # revision hash. Revision hashes have priority over symbolic revisions. We
+    # disallow running tasks with symbolic revisions unless they have been
+    # resolved by a checkout.
+    if args.comm_checkout:
+        base_repo = os.environ.get('COMM_BASE_REPOSITORY')
+
+        os.environ['COMM_HEAD_REV'] = vcs_checkout(
+            os.environ['COMM_HEAD_REPOSITORY'],
+            args.comm_checkout,
+            os.environ['HG_STORE_PATH'],
+            fetch_hgfingerprint=args.fetch_hgfingerprint,
+            base_repo=base_repo,
+            revision=os.environ.get('COMM_HEAD_REV'),
+            branch=os.environ.get('COMM_HEAD_REF'))
+
+    elif not os.environ.get('COMM_HEAD_REV') and \
+            os.environ.get('COMM_HEAD_REF'):
+        print('task should be defined in terms of non-symbolic revision')
+        return 1
 
     try:
         if 'GECKO_PATH' in os.environ:
             os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH'])
 
         if 'MOZ_FETCHES' in os.environ:
             fetch_artifacts()
 
--- a/taskcluster/taskgraph/transforms/job/hazard.py
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -64,12 +64,12 @@ def docker_worker_hazard(config, job, ta
         env['MOZCONFIG'] = run['mozconfig']
 
     # build-haz-linux.sh needs this otherwise it assumes the checkout is in
     # the workspace.
     env['GECKO_DIR'] = '{workdir}/checkouts/gecko'.format(**run)
 
     worker['command'] = [
         '{workdir}/bin/run-task'.format(**run),
-        '--gecko-checkout', '{workdir}/checkouts/gecko'.format(**run),
+        '--vcs-checkout', '{workdir}/checkouts/gecko'.format(**run),
         '--',
         '/bin/bash', '-c', run['command']
     ]
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -214,17 +214,18 @@ def mozharness_on_docker_worker_setup(co
 
     # Retry if mozharness returns TBPL_RETRY
     worker['retry-exit-status'] = [4]
 
     docker_worker_setup_secrets(config, job, taskdesc)
 
     command = [
         '{workdir}/bin/run-task'.format(**run),
-        '--gecko-checkout', env['GECKO_PATH'],
+        '--vcs-checkout', env['GECKO_PATH'],
+        '--tools-checkout', '{workdir}/workspace/build/tools'.format(**run),
     ]
     if run['comm-checkout']:
         command.append('--comm-checkout={workdir}/workspace/build/src/comm'.format(**run))
 
     command += [
         '--',
         '{workdir}/workspace/build/src/{script}'.format(
             workdir=run['workdir'],
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -135,17 +135,17 @@ def mozharness_test_on_docker(config, jo
 
     # Support vcs checkouts regardless of whether the task runs from
     # source or not in case it is needed on an interactive loaner.
     support_vcs_checkout(config, job, taskdesc)
 
     # If we have a source checkout, run mozharness from it instead of
     # downloading a zip file with the same content.
     if test['checkout']:
-        command.extend(['--gecko-checkout', '{workdir}/checkouts/gecko'.format(**run)])
+        command.extend(['--vcs-checkout', '{workdir}/checkouts/gecko'.format(**run)])
         env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(**run)
     else:
         env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
 
     command.extend([
         '--',
         '{workdir}/bin/test-linux.sh'.format(**run),
     ])
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -41,20 +41,20 @@ run_task_schema = Schema({
 })
 
 
 def common_setup(config, job, taskdesc, command):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc,
                              sparse=bool(run['sparse-profile']))
-        command.append('--gecko-checkout={}'.format(taskdesc['worker']['env']['GECKO_PATH']))
+        command.append('--vcs-checkout={}'.format(taskdesc['worker']['env']['GECKO_PATH']))
 
     if run['sparse-profile']:
-        command.append('--gecko-sparse-profile=build/sparse-profiles/%s' %
+        command.append('--sparse-profile=build/sparse-profiles/%s' %
                        run['sparse-profile'])
 
     taskdesc['worker'].setdefault('env', {})['MOZ_SCM_LEVEL'] = config.params['level']
 
 
 worker_defaults = {
     'cache-dotcache': False,
     'checkout': True,
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -68,17 +68,17 @@ def docker_worker_spidermonkey(config, j
         script = "build-sm-package.sh"
     elif run['using'] == 'spidermonkey-mozjs-crate':
         script = "build-sm-mozjs-crate.sh"
     elif run['using'] == 'spidermonkey-rust-bindings':
         script = "build-sm-rust-bindings.sh"
 
     worker['command'] = [
         '{workdir}/bin/run-task'.format(**run),
-        '--gecko-checkout', '{workdir}/workspace/build/src'.format(**run),
+        '--vcs-checkout', '{workdir}/workspace/build/src'.format(**run),
         '--',
         '/bin/bash',
         '-c',
         'cd {workdir} && workspace/build/src/taskcluster/scripts/builder/{script}'.format(
             workdir=run['workdir'], script=script)
     ]
 
 
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -147,22 +147,22 @@ def docker_worker_toolchain(config, job,
         wrapper = ''
 
     args = run.get('arguments', '')
     if args:
         args = ' ' + shell_quote(*args)
 
     sparse_profile = []
     if run.get('sparse-profile'):
-        sparse_profile = ['--gecko-sparse-profile=build/sparse-profiles/{}'
+        sparse_profile = ['--sparse-profile=build/sparse-profiles/{}'
                           .format(run['sparse-profile'])]
 
     worker['command'] = [
         '{workdir}/bin/run-task'.format(**run),
-        '--gecko-checkout={}'.format(gecko_path),
+        '--vcs-checkout={}'.format(gecko_path),
     ] + sparse_profile + [
         '--',
         'bash',
         '-c',
         'cd {} && '
         '{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
             run['workdir'], wrapper, run['script'], args)
     ]
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1937,22 +1937,21 @@ def check_run_task_caches(config, tasks)
         if run_task:
             for arg in command[1:]:
                 if not isinstance(arg, basestring):
                     continue
 
                 if arg == '--':
                     break
 
-                if arg.startswith('--gecko-sparse-profile'):
+                if arg.startswith('--sparse-profile'):
                     if '=' not in arg:
                         raise Exception(
-                            '{} is specifying `--gecko-sparse-profile` to run-task '
-                            'as two arguments. Unable to determine if the sparse '
-                            'profile exists.'.format(
+                            '{} is specifying `--sparse-profile` to run-task as two arguments. '
+                            'Unable to determine if the sparse profile exists.'.format(
                                 task['label']))
                     _, sparse_profile = arg.split('=', 1)
                     if not os.path.exists(os.path.join(GECKO, sparse_profile)):
                         raise Exception(
                             '{} is using non-existant sparse profile {}.'.format(
                                 task['label'], sparse_profile))
                     require_sparse_cache = True
                     break