Bug 1247168 - Use a cache for repo checkout in lint tasks; r=dustin
authorGregory Szorc <gps@mozilla.com>
Wed, 20 Jul 2016 16:21:46 -0700
changeset 331583 722707a63810c1c284c4ba5f539002686f2861a6
parent 331582 b4c3e613fee80fdec958f103d7b8deb62346b547
child 331584 9670ec68b521c2f84868ab72ec4273a377aa79f1
push id9858
push userjlund@mozilla.com
push dateMon, 01 Aug 2016 14:37:10 +0000
treeherdermozilla-aurora@203106ef6cb6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1247168
milestone50.0a1
Bug 1247168 - Use a cache for repo checkout in lint tasks; r=dustin Previously, every lint task would have to create its own checkout. This was time consuming. The robustcheckout extension purges the working copy of *all* untracked and ignored files. It also restores modified files to their original state. In other words, as long as you trust Mercurial to go from revision X to revision Y, robustcheckout is as good as a fresh checkout. This commit adds a cache for the working directory checkout so lint tasks only have to effectively perform incremental `hg update` between task executions. This should make tasks spend a lot less time doing version control foo. On Try, time for flake8 tasks is currently hovering around 4 minutes. After this change, I've seen tasks finish as quickly as 11s! But that was with a hacked up legacy.py that added the workspace cache to the whitelist for Try. While I would like to see workspace reuse on Try eventually, this is not the right commit to roll that out in. MozReview-Commit-ID: 66P2rt896qE
taskcluster/ci/legacy/tasks/lint.yml
taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -19,23 +19,25 @@ task:
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
     - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
     - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
   scopes:
     - 'docker-worker:cache:level-{{level}}-hg-shared'
+    - 'docker-worker:cache:level-{{level}}-workspace'
 
   payload:
     # Thirty minutes should be enough for lint checks
     maxRunTime: 1800
 
     cache:
       level-{{level}}-hg-shared: '/home/worker/hg-shared'
+      level-{{level}}-workspace: '/home/worker/workspace'
 
     env:
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
 
   extra:
     build_product: '{{build_product}}'
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -16,21 +16,21 @@ task:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
 
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko/tools/lint/eslint &&
+          cd /home/worker/workspace/gecko/tools/lint/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 
   extra:
     locations:
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko &&
+          cd /home/worker/workspace/gecko &&
           ./mach lint -l flake8 -f treeherder
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         platform: lint
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - gecko
+      - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
-          cd gecko &&
+          cd /home/worker/workspace/gecko &&
           ./mach taskgraph python-tests
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         # this should probably get a fresh new platform, but linux64 is what, for example, marionette uses