Bug 1185643: use in-tree mozharness; r=wander r=mrrrgn
☠☠ backed out by 2a2f1d5a647b ☠ ☠
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 27 Jul 2015 15:21:15 -0400
changeset 288270 e3c15b9f89b3c331805b4f1a83f62805f32bbb9a
parent 288269 873bab8f2d7bdfe337120d9fc839be4afe8be160
child 288271 27732f823726d44659740f89ac3275bf441f8a55
push id934
push userraliiev@mozilla.com
push dateMon, 26 Oct 2015 12:58:05 +0000
treeherdermozilla-release@05704e35c1d0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswander, mrrrgn
bugs1185643
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1185643: use in-tree mozharness; r=wander r=mrrrgn This required moving the custom mozharness configs out of the builder image (which puts them in ~, where mozharness no longer looks) and moves them into mozharness itself.
testing/docker/desktop-build/Dockerfile
testing/docker/desktop-build/bin/checkout-sources.sh
testing/docker/desktop-build/configs/disable_signing.py
testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py
testing/docker/desktop32-build/bin/checkout-sources.sh
testing/mozharness/configs/disable_signing.py
testing/mozharness/configs/platform_supports_post_upload_to_latest.py
testing/taskcluster/mach_commands.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/build.yml
--- a/testing/docker/desktop-build/Dockerfile
+++ b/testing/docker/desktop-build/Dockerfile
@@ -1,19 +1,16 @@
 FROM          quay.io/mozilla/ubuntu-build:0.0.3
 MAINTAINER    Morgan Reece Phillips <winter2718@gmail.com>
 
 # Add build scripts; these are the entry points from the taskcluster worker, and
 # operate on environment variables
 ADD             bin /home/worker/bin
 RUN             chmod +x /home/worker/bin/*
 
-# Add custom mozharness configs
-ADD             configs /home/worker/configs
-
 ##
 # COMPILER HACKS
 ##
 ADD             hacks.sh /tmp/
 RUN             /tmp/hacks.sh
 
 # So that the compilers can find necessary libraries
 # gcc
--- a/testing/docker/desktop-build/bin/checkout-sources.sh
+++ b/testing/docker/desktop-build/bin/checkout-sources.sh
@@ -5,59 +5,43 @@ set -x -e
 # Inputs, with defaults
 
 # mozharness builds use three repositories: gecko (source), mozharness (build
 # scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
 # revision is not in the standard repo for the codebase, specify *_BASE_REPO as
 # the canonical repo to clone and *_HEAD_REPO as the repo containing the
 # desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
 # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.  For compatibility, we also accept MOZHARNESS_{REV,REF}
+# as *_HEAD_REV.
 
 : GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
 : GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
 : GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
 
-: MOZHARNESS_REPOSITORY         ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
-: MOZHARNESS_BASE_REPOSITORY    ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_HEAD_REPOSITORY    ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_REV                ${MOZHARNESS_REV:=production}
-: MOZHARNESS_REF                ${MOZHARNESS_REF:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REV           ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REF           ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}}
-: MOZHARNESS_DISABLE            ${MOZHARNESS_DISABLE:=false}
-
 : TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
 : TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
 : TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
 : TOOLS_DISABLE                 ${TOOLS_DISABLE:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
-# check out mozharness
-if [ ! "$MOZHARNESS_DISABLE" = "true" ]
-then
-    tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF
-fi
-
 # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
 if [ ! "$TOOLS_DISABLE" = true ]
 then
     tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
 
     if [ ! -d build ]; then
         mkdir -p build
         ln -s $WORKSPACE/build/tools build/tools
     fi
 fi
 
-# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared)
 tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
--- a/testing/docker/desktop32-build/bin/checkout-sources.sh
+++ b/testing/docker/desktop32-build/bin/checkout-sources.sh
@@ -5,50 +5,34 @@ set -x -e
 # Inputs, with defaults
 
 # mozharness builds use three repositories: gecko (source), mozharness (build
 # scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
 # revision is not in the standard repo for the codebase, specify *_BASE_REPO as
 # the canonical repo to clone and *_HEAD_REPO as the repo containing the
 # desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
 # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.  For compatibility, we also accept MOZHARNESS_{REV,REF}
+# as *_HEAD_REV.
 
 : GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
 : GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
 : GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
 
-: MOZHARNESS_REPOSITORY         ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
-: MOZHARNESS_BASE_REPOSITORY    ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_HEAD_REPOSITORY    ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_REV                ${MOZHARNESS_REV:=production}
-: MOZHARNESS_REF                ${MOZHARNESS_REF:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REV           ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REF           ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}}
-
 : TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
 : TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
 : TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
 
-: MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
-: MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
-: MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
-
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
-# check out mozharness
-tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF
-
 # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
 tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
 if [ ! -d build ]; then
     mkdir -p build
     ln -s $WORKSPACE/build/tools build/tools
 fi
 
-# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared)
 tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
rename from testing/docker/desktop-build/configs/disable_signing.py
rename to testing/mozharness/configs/disable_signing.py
rename from testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py
rename to testing/mozharness/configs/platform_supports_post_upload_to_latest.py
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -16,17 +16,16 @@ from mach.decorators import (
     CommandProvider,
     Command,
 )
 
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
 DOCKER_ROOT = os.path.join(ROOT, '..', 'docker')
-MOZHARNESS_CONFIG = os.path.join(GECKO, 'testing', 'mozharness', 'mozharness.json')
 
 # XXX: If/when we have the taskcluster queue use construct url instead
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 
 TREEHERDER_ROUTE_PREFIX = 'tc-treeherder-stage'
@@ -35,20 +34,16 @@ TREEHERDER_ROUTES = {
     'production': 'tc-treeherder'
 }
 
 DEFAULT_TRY = 'try: -b do -p all -u all'
 DEFAULT_JOB_PATH = os.path.join(
     ROOT, 'tasks', 'branches', 'base_jobs.yml'
 )
 
-def load_mozharness_info():
-    with open(MOZHARNESS_CONFIG) as content:
-        return json.load(content)
-
 def docker_image(name):
     ''' Determine the docker tag/revision from an in tree docker file '''
     repository_path = os.path.join(DOCKER_ROOT, name, 'REGISTRY')
     repository = REGISTRY
 
     version = open(os.path.join(DOCKER_ROOT, name, 'VERSION')).read().strip()
 
     if os.path.isfile(repository_path):
@@ -258,34 +253,30 @@ class Graph(object):
 
         templates = Templates(ROOT)
         job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
         job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
 
         jobs = templates.load(job_path, {})
 
         job_graph = parse_commit(message, jobs)
-        mozharness = load_mozharness_info()
 
         # Template parameters used when expanding the graph
         parameters = dict(gaia_info().items() + {
             'project': project,
             'pushlog_id': params.get('pushlog_id', 0),
             'docker_image': docker_image,
             'base_repository': params['base_repository'] or \
                 params['head_repository'],
             'head_repository': params['head_repository'],
             'head_ref': params['head_ref'] or params['head_rev'],
             'head_rev': params['head_rev'],
             'owner': params['owner'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
-            'mozharness_repository': mozharness['repo'],
-            'mozharness_rev': mozharness['revision'],
-            'mozharness_ref':mozharness.get('reference', mozharness['revision']),
             'revision_hash': params['revision_hash']
         }.items())
 
         treeherder_route = '{}.{}'.format(
             params['project'],
             params.get('revision_hash', '')
         )
 
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -113,17 +113,17 @@ export TOOLTOOL_CACHE
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
 # Mozharness would ordinarily do the checkouts itself, but they are disabled
 # here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
 
-./${MOZHARNESS_SCRIPT} ${config_cmds} \
+$WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
   --no-setup-mock \
   --no-checkout-sources \
   --no-clone-tools \
   --no-clobber \
   --no-update \
--- a/testing/taskcluster/tasks/build.yml
+++ b/testing/taskcluster/tasks/build.yml
@@ -47,19 +47,16 @@ task:
 
 
     env:
       # Common environment variables for checking out gecko
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
       GECKO_HEAD_REF: '{{head_ref}}'
-      MOZHARNESS_REPOSITORY: '{{mozharness_repository}}'
-      MOZHARNESS_REV: '{{mozharness_rev}}'
-      MOZHARNESS_REF: '{{mozharness_ref}}'
       TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
       TOOLTOOL_REV: 'master'
 
   extra:
     index:
       rank: {{pushlog_id}}
     treeherder:
       groupSymbol: tc