Bug 1185643: use in-tree mozharness for builds; r=wander
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 27 Jul 2015 15:21:15 -0400
changeset 258851 be6ff275a904254445fbc83c0bf64d5f020fd382
parent 258850 733fe110f356fdaaf48aa3c64a365c005386a3d1
child 258852 780809cead8968a0a90e3f913aa568dfa889a5f7
push id29263
push userryanvm@gmail.com
push dateSun, 23 Aug 2015 21:18:49 +0000
treeherdermozilla-central@4ccdd06e51d7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswander
bugs1185643
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1185643: use in-tree mozharness for builds; r=wander This required moving the custom mozharness configs out of the builder image (which puts them in ~, where mozharness no longer looks) and moving them into mozharness itself.
testing/docker/desktop-build/Dockerfile
testing/docker/desktop-build/bin/checkout-sources.sh
testing/docker/desktop-build/configs/disable_signing.py
testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py
testing/docker/desktop32-build/REGISTRY
testing/docker/desktop32-build/bin/checkout-sources.sh
testing/mozharness/configs/disable_signing.py
testing/mozharness/configs/platform_supports_post_upload_to_latest.py
testing/taskcluster/mach_commands.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/build.yml
--- a/testing/docker/desktop-build/Dockerfile
+++ b/testing/docker/desktop-build/Dockerfile
@@ -1,19 +1,16 @@
 FROM          taskcluster/ubuntu-build:0.0.5
 MAINTAINER    Morgan Reece Phillips <winter2718@gmail.com>
 
 # Add build scripts; these are the entry points from the taskcluster worker, and
 # operate on environment variables
 ADD             bin /home/worker/bin
 RUN             chmod +x /home/worker/bin/*
 
-# Add custom mozharness configs
-ADD             configs /home/worker/configs
-
 ##
 # COMPILER HACKS
 ##
 ADD             hacks.sh /tmp/
 RUN             /tmp/hacks.sh
 
 # So that the compilers can find necessary libraries
 # gcc
--- a/testing/docker/desktop-build/bin/checkout-sources.sh
+++ b/testing/docker/desktop-build/bin/checkout-sources.sh
@@ -5,59 +5,43 @@ set -x -e
 # Inputs, with defaults
 
 # mozharness builds use three repositories: gecko (source), mozharness (build
 # scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
 # revision is not in the standard repo for the codebase, specify *_BASE_REPO as
 # the canonical repo to clone and *_HEAD_REPO as the repo containing the
 # desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
 # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.  For compatibility, we also accept MOZHARNESS_{REV,REF}
+# as *_HEAD_REV.
 
 : GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
 : GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
 : GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
 
-: MOZHARNESS_REPOSITORY         ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
-: MOZHARNESS_BASE_REPOSITORY    ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_HEAD_REPOSITORY    ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_REV                ${MOZHARNESS_REV:=production}
-: MOZHARNESS_REF                ${MOZHARNESS_REF:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REV           ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REF           ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}}
-: MOZHARNESS_DISABLE            ${MOZHARNESS_DISABLE:=false}
-
 : TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
 : TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
 : TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
 : TOOLS_DISABLE                 ${TOOLS_DISABLE:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
-# check out mozharness
-if [ ! "$MOZHARNESS_DISABLE" = "true" ]
-then
-    tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF
-fi
-
 # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
 if [ ! "$TOOLS_DISABLE" = true ]
 then
     tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
 
     if [ ! -d build ]; then
         mkdir -p build
         ln -s $WORKSPACE/build/tools build/tools
     fi
 fi
 
-# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared)
 tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
--- a/testing/docker/desktop32-build/REGISTRY
+++ b/testing/docker/desktop32-build/REGISTRY
@@ -1,1 +1,1 @@
-taskcluster
+quay.io/djmitche
--- a/testing/docker/desktop32-build/bin/checkout-sources.sh
+++ b/testing/docker/desktop32-build/bin/checkout-sources.sh
@@ -5,50 +5,34 @@ set -x -e
 # Inputs, with defaults
 
 # mozharness builds use three repositories: gecko (source), mozharness (build
 # scripts) and tools (miscellaneous) for each, specify *_REPOSITORY.  If the
 # revision is not in the standard repo for the codebase, specify *_BASE_REPO as
 # the canonical repo to clone and *_HEAD_REPO as the repo containing the
 # desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
 # clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
-# as *_HEAD_REV.  For compatibility, we also accept MOZHARNESS_{REV,REF}
+# as *_HEAD_REV.
 
 : GECKO_REPOSITORY              ${GECKO_REPOSITORY:=https://hg.mozilla.org/mozilla-central}
 : GECKO_BASE_REPOSITORY         ${GECKO_BASE_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REPOSITORY         ${GECKO_HEAD_REPOSITORY:=${GECKO_REPOSITORY}}
 : GECKO_HEAD_REV                ${GECKO_HEAD_REV:=default}
 : GECKO_HEAD_REF                ${GECKO_HEAD_REF:=${GECKO_HEAD_REV}}
 
-: MOZHARNESS_REPOSITORY         ${MOZHARNESS_REPOSITORY:=https://hg.mozilla.org/build/mozharness}
-: MOZHARNESS_BASE_REPOSITORY    ${MOZHARNESS_BASE_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_HEAD_REPOSITORY    ${MOZHARNESS_HEAD_REPOSITORY:=${MOZHARNESS_REPOSITORY}}
-: MOZHARNESS_REV                ${MOZHARNESS_REV:=production}
-: MOZHARNESS_REF                ${MOZHARNESS_REF:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REV           ${MOZHARNESS_HEAD_REV:=${MOZHARNESS_REV}}
-: MOZHARNESS_HEAD_REF           ${MOZHARNESS_HEAD_REF:=${MOZHARNESS_REF}}
-
 : TOOLS_REPOSITORY              ${TOOLS_REPOSITORY:=https://hg.mozilla.org/build/tools}
 : TOOLS_BASE_REPOSITORY         ${TOOLS_BASE_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REPOSITORY         ${TOOLS_HEAD_REPOSITORY:=${TOOLS_REPOSITORY}}
 : TOOLS_HEAD_REV                ${TOOLS_HEAD_REV:=default}
 : TOOLS_HEAD_REF                ${TOOLS_HEAD_REF:=${TOOLS_HEAD_REV}}
 
-: MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
-: MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
-: MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
-
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
 set -v
 
-# check out mozharness
-tc-vcs checkout mozharness $MOZHARNESS_BASE_REPOSITORY $MOZHARNESS_HEAD_REPOSITORY $MOZHARNESS_HEAD_REV $MOZHARNESS_HEAD_REF
-
 # check out tools where mozharness expects it to be ($PWD/build/tools and $WORKSPACE/build/tools)
 tc-vcs checkout $WORKSPACE/build/tools $TOOLS_BASE_REPOSITORY $TOOLS_HEAD_REPOSITORY $TOOLS_HEAD_REV $TOOLS_HEAD_REF
 if [ ! -d build ]; then
     mkdir -p build
     ln -s $WORKSPACE/build/tools build/tools
 fi
 
-# and check out mozilla-central where mozharness will use it as a cache (/builds/hg-shared)
 tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF
rename from testing/docker/desktop-build/configs/disable_signing.py
rename to testing/mozharness/configs/disable_signing.py
rename from testing/docker/desktop-build/configs/platform_supports_post_upload_to_latest.py
rename to testing/mozharness/configs/platform_supports_post_upload_to_latest.py
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -278,16 +278,18 @@ class Graph(object):
 
         templates = Templates(ROOT)
         job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
         job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
 
         jobs = templates.load(job_path, {})
 
         job_graph = parse_commit(message, jobs)
+
+        # once everything uses in-tree mozharness (bug 1187706), this can go away.
         mozharness = load_mozharness_info()
 
         # Template parameters used when expanding the graph
         parameters = dict(gaia_info().items() + {
             'index': 'index',
             'project': project,
             'pushlog_id': params.get('pushlog_id', 0),
             'docker_image': docker_image,
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -113,17 +113,17 @@ export TOOLTOOL_CACHE
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
 # Mozharness would ordinarily do the checkouts itself, but they are disabled
 # here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
 
-./${MOZHARNESS_SCRIPT} ${config_cmds} \
+$WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
   --no-setup-mock \
   --no-checkout-sources \
   --no-clone-tools \
   --no-clobber \
   --no-update \
--- a/testing/taskcluster/tasks/build.yml
+++ b/testing/taskcluster/tasks/build.yml
@@ -47,19 +47,16 @@ task:
 
 
     env:
       # Common environment variables for checking out gecko
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
       GECKO_HEAD_REF: '{{head_ref}}'
-      MOZHARNESS_REPOSITORY: '{{mozharness_repository}}'
-      MOZHARNESS_REV: '{{mozharness_rev}}'
-      MOZHARNESS_REF: '{{mozharness_ref}}'
       TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
       TOOLTOOL_REV: 'master'
 
   extra:
     build_product: '{{build_product}}'
     index:
       rank: {{pushlog_id}}
     treeherder: