Bug 1247168 - Convert lint image and tasks to use robustcheckout; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Tue, 19 Jul 2016 13:30:03 -0700
changeset 390109 894132c6c65590f56ab0b85ad92bc265b07f41b9
parent 390108 3c9989d24e218bc34fc68a6f93d35e8efc1032c2
child 390110 7da2727e350c708186a3998ee54efacc89c86cdc
push id23599
push userbmo:gps@mozilla.com
push dateWed, 20 Jul 2016 17:43:07 +0000
reviewersdustin
bugs1247168
milestone50.0a1
Bug 1247168 - Convert lint image and tasks to use robustcheckout; r?dustin The robustcheckout Mercurial extension does a clone+checkout optimally. Read the bug for more on it. robustcheckout is already used by mozharness automation. It has resulted in a significant reduction in I/O usage and utilization in automation. This commit replaces tc-vcs with the robustcheckout equivalent. We replace the existing tc-vcs scope and cache with a new one. Because Dustin and I are paranoid, we maintain separate caches per SCM level - even though we could arguably share the same cache. Defense in depth. Robustcheckout (when used with --sharebase) pools storage for related repos automatically. i.e. changesets from inbound and central will be in the same store. This means you likely only have one copy of each changeset per cache. This can result in significant space savings. And, since there are fewer copies floating around, hg.mozilla.org and various network appliances are working less too! Since tc-vcs is no longer used, we stop it from being installed. MozReview-Commit-ID: EDeebuP7TkT
taskcluster/ci/legacy/tasks/lint.yml
taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
testing/docker/lint/system-setup.sh
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -18,26 +18,24 @@ task:
   workerType: b2gtest
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
     - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
     - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
   scopes:
-    # Nearly all of our build tasks use tc-vcs so just include the scope across
-    # the board.
-    - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
+    - 'docker-worker:cache:level-{{level}}-hg-shared'
 
   payload:
     # Thirty minutes should be enough for lint checks
     maxRunTime: 1800
 
     cache:
-      level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
+      level-{{level}}-hg-shared: '/home/worker/hg-shared'
 
   extra:
     build_product: '{{build_product}}'
     build_name: {{build_name}}
     build_type: {{build_type}}
     index:
       rank: {{rank}}
     treeherder:
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -18,17 +18,17 @@ task:
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
 
     command:
       - bash
       - -cx
       - >
-          tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
+          hg robustcheckout --upstream {{base_repository}} --revision {{head_rev}} --purge --sharebase /home/worker/hg-shared {{head_repository}} gecko &&
           cd gecko/tools/lint/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 
   extra:
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -16,17 +16,17 @@ task:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - bash
       - -cx
       - >
-          tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
+          hg robustcheckout --upstream {{base_repository}} --revision {{head_rev}} --purge --sharebase /home/worker/hg-shared {{head_repository}} gecko &&
           cd gecko &&
           ./mach lint -l flake8 -f treeherder
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -16,17 +16,17 @@ task:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - bash
       - -cx
       - >
-          tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
+          hg robustcheckout --upstream {{base_repository}} --revision {{head_rev}} --purge --sharebase /home/worker/hg-shared {{head_repository}} gecko &&
           cd gecko &&
           ./mach taskgraph python-tests
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
--- a/testing/docker/lint/system-setup.sh
+++ b/testing/docker/lint/system-setup.sh
@@ -48,35 +48,34 @@ tooltool_fetch <<'EOF'
     "filename": "node-v4.4.5-linux-x64.tar.xz"
 }
 ]
 EOF
 tar -C /usr/local --strip-components 1 -xJ < node-*.tar.xz
 node -v  # verify
 npm -v
 
-# install taskcluster-vcs@2.3.12
+mkdir -p /usr/local/mercurial
+cd /usr/local/mercurial
 tooltool_fetch <<'EOF'
 [
 {
-"size": 6282161,
-"visibility": "public",
-"digest": "a781a96e596f6403eca6ec2300adb9c1a396659393e16993c66f98a658050e557bc681d521f70b50c1162aa4b435274e0098ffcbd37cbe969c0e4f69be19a1e0",
-"algorithm": "sha512",
-"filename": "taskcluster-vcs-v2.3.12.tar.gz"
+    "size": 11849,
+    "digest": "c88d9b8afd6649bd28bbacfa654ebefec8087a01d1662004aae088d485edeb03a92df1193d1310c0369d7721f475b974fcd4a911428ec65936f7e40cf1609c49",
+    "algorithm": "sha512",
+    "filename": "robustcheckout.py"
 }
 ]
 EOF
-npm install -g taskcluster-vcs-v2.3.12.tar.gz
-
 
 ###
 # Flake8 Setup
 ###
 
+cd /setup
 cat >requirements.txt <<'EOF'
 flake8==2.5.4 \
   --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
 mccabe==0.4.0 \
   --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
 Mercurial==3.8.4 \
   --hash=sha256:4b2e3ef19d34fa1d781cb7425506a05d4b6b1172bab69d6ea78874175fdf3da6
 pep8==1.7.0 \
@@ -95,11 +94,14 @@ cat >/etc/mercurial/hgrc <<EOF
 # By default the progress bar starts after 3s and updates every 0.1s. We
 # change this so it shows and updates every 1.0s.
 [progress]
 delay = 1.0
 refresh = 1.0
 
 [web]
 cacerts = /etc/ssl/certs/ca-certificates.crt
+
+[extensions]
+robustcheckout = /usr/local/mercurial/robustcheckout.py
 EOF
 
 chmod 644 /etc/mercurial/hgrc
\ No newline at end of file