Bug 1289643 - Change path for checkouts from "workspace" to "checkouts"; r=dustin
authorGregory Szorc <gps@mozilla.com>
Fri, 29 Jul 2016 10:44:19 -0700
changeset 336038 50b87c38d18dc57a57de8b5c24079899180af11a
parent 336037 0beb6afa58b70382dc1974bd07326bb8b948561a
child 336039 6ea38e6856796ace377833cffd28dfc06dc362e8
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-esr52@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1289643
milestone50.0a1
Bug 1289643 - Change path for checkouts from "workspace" to "checkouts"; r=dustin Currently, TaskCluster tasks tend to use the "workspace" directory as a cache that manages the source checkout *and* additional state. Historically at Mozilla, we've lumped "source checkout" and "workspace" (sometimes known as an "objdir") into the same directory. This is not ideal. Ideally, there is an immutable, read-only source checkout and all files produced from that source live in a separate directory. In this commit, the "workspace" directory for the "lint" image has been renamed to "checkouts" and all tasks using the image have been updated accordingly. By having "checkout" in the name, we clearly identify this cache as being relevant to source checkouts, which IMO can serve a different role from "workspaces." This distinction is important, as the next commit will prevent the "checkouts" cache from getting optimized out in certain tasks. To hammer this point home, documentation on common caches has been introduced. MozReview-Commit-ID: BSEc4dM5YCt
taskcluster/ci/legacy/tasks/lint.yml
taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
taskcluster/ci/legacy/tasks/tests/mozharness-gecko.yml
taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
taskcluster/docs/caches.rst
taskcluster/docs/index.rst
testing/docker/recipes/checkout-gecko-and-run
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -19,25 +19,25 @@ task:
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
     - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
     - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
   scopes:
     - 'docker-worker:cache:level-{{level}}-hg-shared'
-    - 'docker-worker:cache:level-{{level}}-workspace'
+    - 'docker-worker:cache:level-{{level}}-checkouts'
 
   payload:
     # Thirty minutes should be enough for lint checks
     maxRunTime: 1800
 
     cache:
       level-{{level}}-hg-shared: '/home/worker/hg-shared'
-      level-{{level}}-workspace: '/home/worker/workspace'
+      level-{{level}}-checkouts: '/home/worker/checkouts'
 
     env:
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
 
   extra:
     build_product: '{{build_product}}'
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -16,21 +16,21 @@ task:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
 
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - /home/worker/workspace/gecko
+      - /home/worker/checkouts/gecko
       - bash
       - -cx
       - >
-          cd /home/worker/workspace/gecko/tools/lint/eslint &&
+          cd /home/worker/checkouts/gecko/tools/lint/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
           ln -s ../eslint-plugin-mozilla node_modules &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 
   extra:
--- a/taskcluster/ci/legacy/tasks/tests/mozharness-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozharness-gecko.yml
@@ -22,21 +22,21 @@ task:
       taskId:
         task-reference: "<docker-image>"
 
     cache:
       level-{{level}}-{{project}}-dotcache: '/home/worker/.cache'
 
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - /home/worker/workspace/gecko
+      - /home/worker/checkouts/gecko
       - bash
       - -cx
       - >
-          cd /home/worker/workspace/gecko/testing/mozharness &&
+          cd /home/worker/checkouts/gecko/testing/mozharness &&
           /usr/bin/pip2 install tox &&
           /home/worker/.local/bin/tox -e py27-hg3.7
   extra:
     extra:
         build_product: '{{build_product}}'
     locations:
         build: null
         tests: null
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - /home/worker/workspace/gecko
+      - /home/worker/checkouts/gecko
       - bash
       - -cx
       - >
-          cd /home/worker/workspace/gecko &&
+          cd /home/worker/checkouts/gecko &&
           ./mach lint -l flake8 -f treeherder
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         platform: lint
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -14,21 +14,21 @@ task:
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId:
         task-reference: "<docker-image>"
     command:
       - /home/worker/bin/checkout-gecko-and-run
-      - /home/worker/workspace/gecko
+      - /home/worker/checkouts/gecko
       - bash
       - -cx
       - >
-          cd /home/worker/workspace/gecko &&
+          cd /home/worker/checkouts/gecko &&
           ./mach taskgraph python-tests
   extra:
     locations:
       build: null
       tests: null
     treeherder:
       machine:
         # this should probably get a fresh new platform, but linux64 is what, for example, marionette uses
new file mode 100644
--- /dev/null
+++ b/taskcluster/docs/caches.rst
@@ -0,0 +1,43 @@
+.. taskcluster_caches:
+
+=============
+Common Caches
+=============
+
+There are various caches used by the in-tree tasks. This page attempts to
+document them and their appropriate use.
+
+Version Control Caches
+======================
+
+level-{{level}}-hg-shared
+   This cache holds Mercurial *stores*: this is the historical data of a
+   Mercurial repository (as opposed to a checkout or working directory).
+
+   This cache should ideally be written to by the ``hg robustcheckout``
+   command, which is the preferred mechanism to clone a Mercurial repository
+   in automation.
+
+   When using ``hg robustcheckout``, data for the same logical repository
+   will automatically be stored in the same directory in this cache. e.g.
+   data for the Firefox repository will be in the directory
+   ``8ba995b74e18334ab3707f27e9eb8f4e37ba3d29``, which is the SHA-1 of the
+   first revision in the Firefox repository. This helps ensure that any
+   given changeset is only stored once in the cache. This is why there is
+   no ``{{project}}`` component in the cache name.
+
+level-{{level}}-checkouts
+   This cache holds version control checkouts or working directories. This
+   is distinct from the cache that holds the version control *store* data.
+
+   Ideally, checkouts should be read-only. If a task needs to create new
+   files from content of a checkout, this content should be written in
+   a separate directory/cache (like a workspace).
+
+Workspace Caches
+================
+
+level-{{level}}-*-workspace
+   These caches (of various names typically ending with ``workspace``)
+   contain state to be shared between task invocations. Use cases are
+   dependent on the task.
--- a/taskcluster/docs/index.rst
+++ b/taskcluster/docs/index.rst
@@ -24,8 +24,9 @@ check out the :doc:`how-to section <how-
     taskgraph
     parameters
     attributes
     kinds
     transforms
     yaml-templates
     how-tos
     docker-images
+    caches
--- a/testing/docker/recipes/checkout-gecko-and-run
+++ b/testing/docker/recipes/checkout-gecko-and-run
@@ -1,27 +1,29 @@
 #!/bin/bash
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 set -ex
 
+DEST=$1
+CHECKOUTS=`dirname ${DEST}`
+
 # The script starts executing as root. We need to change ownership
 # of the caches because they are initially owned by root:root. There
 # may not be a cache mount/directory on some instances. So create the
 # directory if missing.
 if [ $(id -u) = 0 ]; then
-    mkdir -p /home/worker/workspace
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
+    mkdir -p ${CHECKOUTS}
+    chown worker:worker /home/worker/hg-shared ${CHECKOUTS}
 
     exec sudo -E -u worker ${0} "${@}"
 fi
 
-DEST=$1
 shift
 
 # We set the base repository to mozilla-central so tc-vcs doesn't get
 # confused. Switch to mozilla-unified because robustcheckout works best
 # with it.
 if [ "${GECKO_BASE_REPOSITORY}" = "https://hg.mozilla.org/mozilla-central" ]; then
     GECKO_BASE_REPOSITORY=https://hg.mozilla.org/mozilla-unified
 fi