Bug 1302907 - Run mozharness from source checkout if available; r=dustin
authorGregory Szorc <gps@mozilla.com>
Wed, 14 Sep 2016 18:20:57 -0700
changeset 355308 21fdf73bbb17e34cfe00e372695c4f21e4ba3e6a
parent 355307 dda84d1fb12b3416c81aab675c58b95ecb9f37cd
child 355309 f7acbdb1d9a69b746c0e600968fa3ca4246a4ba3
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-beta@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1302907
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1302907 - Run mozharness from source checkout if available; r=dustin Currently, build tasks produce a zip file artifact containing the content of testing/mozharness. This zip file is downloaded and applied by any test using mozharness (which is most of them). Now that some tests have a source checkout, we don't need to download a mozharness zip because we can just use the source checkout. This commit teaches test-ubuntu.sh to accept a custom path to mozharness files. If that path is defined, we skip downloading the mozharness zip. The taskgraph has been updated to pass the path of mozharness in the checkout to test-ubuntu.sh when a source checkout is available. MozReview-Commit-ID: 2P17Wx1oytI
taskcluster/scripts/tester/test-ubuntu.sh
taskcluster/taskgraph/transforms/tests/make_task_description.py
--- a/taskcluster/scripts/tester/test-ubuntu.sh
+++ b/taskcluster/scripts/tester/test-ubuntu.sh
@@ -15,16 +15,17 @@ fi
 . /home/worker/scripts/xvfb.sh
 
 ####
 # Taskcluster friendly wrapper for performing fx desktop tests via mozharness.
 ####
 
 # Inputs, with defaults
 
+: MOZHARNESS_PATH               ${MOZHARNESS_PATH}
 : MOZHARNESS_URL                ${MOZHARNESS_URL}
 : MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
 : MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
 : NEED_XVFB                     ${NEED_XVFB:=true}
 : NEED_WINDOW_MANAGER           ${NEED_WINDOW_MANAGER:=false}
 : NEED_PULSEAUDIO               ${NEED_PULSEAUDIO:=false}
 : START_VNC                     ${START_VNC:=false}
 : TASKCLUSTER_INTERACTIVE       ${TASKCLUSTER_INTERACTIVE:=false}
@@ -43,43 +44,50 @@ fail() {
 maybe_start_pulse() {
     if $NEED_PULSEAUDIO; then
         pulseaudio --fail --daemonize --start
         pactl load-module module-null-sink
     fi
 }
 
 # test required parameters are supplied
-if [[ -z ${MOZHARNESS_URL} ]]; then fail "MOZHARNESS_URL is not set"; fi
+if [ -z "${MOZHARNESS_PATH}" -a -z "${MOZHARNESS_URL}" ]; then
+    fail "MOZHARNESS_PATH or MOZHARNESS_URL must be defined";
+fi
+
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
 
 mkdir -p ~/artifacts/public
 
 cleanup() {
     local rv=$?
     if [[ -s /home/worker/.xsession-errors ]]; then
       # To share X issues
       cp /home/worker/.xsession-errors ~/artifacts/public/xsession-errors.log
     fi
     cleanup_xvfb
     exit $rv
 }
 trap cleanup EXIT INT
 
-# Unzip the mozharness ZIP file created by the build task
-if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
-    fail "failed to download mozharness zip"
-fi
-rm -rf mozharness
-unzip -q mozharness.zip
-rm mozharness.zip
+# Download mozharness if we're told to.
+if [ ${MOZHARNESS_URL} ]; then
+    if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+        fail "failed to download mozharness zip"
+    fi
+    rm -rf mozharness
+    unzip -q mozharness.zip
+    rm mozharness.zip
 
-if ! [ -d mozharness ]; then
-    fail "mozharness zip did not contain mozharness/"
+    if ! [ -d mozharness ]; then
+        fail "mozharness zip did not contain mozharness/"
+    fi
+
+    MOZHARNESS_PATH=`pwd`/mozharness
 fi
 
 # pulseaudio daemon must be started before xvfb on Ubuntu 12.04.
 if ${UBUNTU_1204}; then
     maybe_start_pulse
 fi
 
 # run XVfb in the background, if necessary
@@ -125,28 +133,28 @@ fi
 # source it is running; tc-vcs obscures this a little, but we can provide
 # it directly.
 export MOZ_SOURCE_REPO="${GECKO_HEAD_REPOSITORY}"
 export MOZ_SOURCE_CHANGESET="${GECKO_HEAD_REV}"
 
 # support multiple, space delimited, config files
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
-  config_cmds="${config_cmds} --config-file $WORKSPACE/mozharness/configs/${cfg}"
+  config_cmds="${config_cmds} --config-file ${MOZHARNESS_PATH}/configs/${cfg}"
 done
 
 mozharness_bin="/home/worker/bin/run-mozharness"
 
 # Save the computed mozharness command to a binary which is useful
 # for interactive mode.
 echo -e "#!/usr/bin/env bash
 # Some mozharness scripts assume base_work_dir is in
 # the current working directory, see bug 1279237
 cd $WORKSPACE
-cmd=\"python2.7 $WORKSPACE/mozharness/scripts/${MOZHARNESS_SCRIPT} ${config_cmds} ${@} \${@}\"
+cmd=\"python2.7 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} ${config_cmds} ${@} \${@}\"
 echo \"Running: \${cmd}\"
 exec \${cmd}" > ${mozharness_bin}
 chmod +x ${mozharness_bin}
 
 # In interactive mode, the user will be prompted with options for what to do.
 if ! $TASKCLUSTER_INTERACTIVE; then
   # run the given mozharness script and configs, but pass the rest of the
   # arguments in from our own invocation
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -157,17 +157,16 @@ def docker_worker_setup(config, test, ta
         'name': 'level-{}-{}-test-workspace'.format(
             config.params['level'], config.params['project']),
         'mount-point': "/home/worker/workspace",
     }]
 
     env = worker['env'] = {
         'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
         'MOZHARNESS_SCRIPT': mozharness['script'],
-        'MOZHARNESS_URL': {'task-reference': mozharness_url},
         'MOZILLA_BUILD_URL': {'task-reference': installer_url},
         'NEED_PULSEAUDIO': 'true',
         'NEED_WINDOW_MANAGER': 'true',
     }
 
     if mozharness['set-moz-node-path']:
         env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
 
@@ -193,19 +192,24 @@ def docker_worker_setup(config, test, ta
 
     # assemble the command line
     command = [
         '/home/worker/bin/run-task',
         # The workspace cache/volume is default owned by root:root.
         '--chown', '/home/worker/workspace',
     ]
 
+    # If we have a source checkout, run mozharness from it instead of
+    # downloading a zip file with the same content.
     if test['checkout']:
         docker_worker_support_vcs_checkout(config, test, taskdesc)
         command.extend(['--vcs-checkout', '/home/worker/checkouts/gecko'])
+        env['MOZHARNESS_PATH'] = '/home/worker/checkouts/gecko/testing/mozharness'
+    else:
+        env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
 
     command.extend([
         '--',
         '/home/worker/bin/test-linux.sh',
     ])
 
     if mozharness.get('no-read-buildbot-config'):
         command.append("--no-read-buildbot-config")