Bug 1247168 - Use a cache for repo checkout in lint tasks; r=dustin
authorGregory Szorc <gps@mozilla.com>
Wed, 20 Jul 2016 16:21:46 -0700
changeset 346549 722707a63810c1c284c4ba5f539002686f2861a6
parent 346548 b4c3e613fee80fdec958f103d7b8deb62346b547
child 346550 9670ec68b521c2f84868ab72ec4273a377aa79f1
push id6389
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:38:22 +0000
treeherdermozilla-beta@01d67bfe6c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1247168
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1247168 - Use a cache for repo checkout in lint tasks; r=dustin Previously, every lint task would have to create its own checkout. This was time consuming. The robustcheckout extension purges the working copy of *all* untracked and ignored files. It also restores modified files to their original state. In other words, as long as you trust Mercurial to go from revision X to revision Y, robustcheckout is as good as a fresh checkout. This commit adds a cache for the working directory checkout so lint tasks only have to effectively perform incremental `hg update` between task executions. This should make tasks spend a lot less time doing version control foo. On Try, time for flake8 tasks is currently hovering around 4 minutes. After this change, I've seen tasks finish as quickly as 11s! But that was with a hacked up legacy.py that added the workspace cache to the whitelist for Try. While I would like to see workspace reuse on Try eventually, this is not the right commit to roll that out in. MozReview-Commit-ID: 66P2rt896qE
taskcluster/ci/legacy/tasks/lint.yml
taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -19,23 +19,25 @@ task:
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
     - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
     - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
   scopes:
     - 'docker-worker:cache:level-{{level}}-hg-shared'
+    - 'docker-worker:cache:level-{{level}}-workspace'
 
   payload:
     # Thirty minutes should be enough for lint checks
     maxRunTime: 1800
 
     cache:
       level-{{level}}-hg-shared: '/home/worker/hg-shared'
+      level-{{level}}-workspace: '/home/worker/workspace'
 
     env:
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
 
   extra:
     build_product: '{{build_product}}'
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -16,21 +16,21 @@ task:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
 
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko/tools/lint/eslint &&
+          cd /home/worker/workspace/gecko/tools/lint/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 
   extra:
     locations:
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko &&
+          cd /home/worker/workspace/gecko &&
           ./mach lint -l flake8 -f treeherder
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         platform: lint
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko &&
+          cd /home/worker/workspace/gecko &&
           ./mach taskgraph python-tests
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         # this should probably get a fresh new platform, but linux64 is what, for example, marionette uses