Bug 1311791 - Explicitly declare path to shared hg store; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Thu, 20 Oct 2016 12:56:43 -0700
changeset 428275 af4c50bc2837d5de33f5d55746bc2ff15f49401b
parent 428241 5639a9f476d08f300c079117e61697f5026b6367
child 534691 9e009e08fe68cf4c8873618a3eb6d454cbb90793
push id33264
push userbmo:gps@mozilla.com
push dateFri, 21 Oct 2016 22:19:05 +0000
reviewersdustin
bugs1311791
milestone52.0a1
Bug 1311791 - Explicitly declare path to shared hg store; r?dustin 332a08725ed0 changed the store path logic in a quick and crude manner. The code could lead to multiple shared stores existing on a cache if checkouts were in different parent directories. This commit refactors the code to explicitly declare a path to the shared hg store. This restores the behavior of ensuring there is only a single shared store per cache. MozReview-Commit-ID: 19Aa1QVrVQ8
.taskcluster.yml
taskcluster/scripts/tester/run-wizard
taskcluster/taskgraph/action.yml
taskcluster/taskgraph/transforms/job/common.py
testing/docker/decision/VERSION
testing/docker/recipes/run-task
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -70,28 +70,28 @@ tasks:
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
+          HG_STORE_PATH: /home/worker/checkouts/hg-store
 
         cache:
-          level-{{level}}-hg-shared: /home/worker/hg-shared
           level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
           chainOfTrust: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the version
-        image: 'taskcluster/decision:0.1.6'
+        image: 'taskcluster/decision:0.1.7'
 
         maxRunTime: 1800
 
         # TODO use mozilla-unified for the base repository once the tc-vcs
         # tar.gz archives are created or tc-vcs isn't being used.
         command:
           - /home/worker/bin/run-task
           - '--vcs-checkout=/home/worker/checkouts/gecko'
--- a/taskcluster/scripts/tester/run-wizard
+++ b/taskcluster/scripts/tester/run-wizard
@@ -88,24 +88,20 @@ def clone():
         revision = os.environ['GECKO_HEAD_REV']
     elif os.environ.get('GECKO_HEAD_REF'):
         revision_flag = b'--branch'
         revision = os.environ['GECKO_HEAD_REF']
     else:
         print('revision is not specified for checkout')
         return 1
 
-    # Put the shared stores as a sibling of the checkout directory. This ensures
-    # they are on the same cache.
-    share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared'))
-
     # TODO Bug 1301382 - pin hg.mozilla.org fingerprint.
     call([
         b'/usr/bin/hg', b'robustcheckout',
-        b'--sharebase', share_base
+        b'--sharebase', os.environ['HG_STORE_PATH'],
         b'--purge',
         b'--upstream', b'https://hg.mozilla.org/mozilla-unified',
         revision_flag, revision,
         base_repo, dest
     ])
     print("Finished cloning to {} at revision {}.".format(dest, revision))
 
 
--- a/taskcluster/taskgraph/action.yml
+++ b/taskcluster/taskgraph/action.yml
@@ -27,26 +27,27 @@ routes:
   - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
 payload:
   env:
     GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
     GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
     GECKO_HEAD_REF: '{{head_ref}}'
     GECKO_HEAD_REV: '{{head_rev}}'
+    HG_STORE_PATH: /home/worker/checkouts/hg-store
 
   cache:
     level-{{level}}-checkouts: /home/worker/checkouts
 
   features:
     taskclusterProxy: true
 
   # Note: This task is built server side without the context or tooling that
   # exist in tree so we must hard code the version
-  image: 'taskcluster/decision:0.1.6'
+  image: 'taskcluster/decision:0.1.7'
 
   # Virtually no network or other potentially risky operations happen as part
   # of the task timeout aside from the initial clone. We intentionally have
   # set this to a lower value _all_ decision tasks should use a root
   # repository which is cached.
   maxRunTime: 1800
 
   command:
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -70,16 +70,17 @@ def docker_worker_support_vcs_checkout(c
         'name': 'level-%s-checkouts' % level,
         'mount-point': '/home/worker/checkouts',
     })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
+        'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
     })
 
     # Give task access to hgfingerprint secret so it can pin the certificate
     # for hg.mozilla.org.
     taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
     taskdesc['worker']['taskcluster-proxy'] = True
 
 
--- a/testing/docker/decision/VERSION
+++ b/testing/docker/decision/VERSION
@@ -1,1 +1,1 @@
-0.1.6
+0.1.7
--- a/testing/docker/recipes/run-task
+++ b/testing/docker/recipes/run-task
@@ -74,17 +74,18 @@ def run_and_prefix_output(prefix, args, 
         if data == b'':
             break
 
         print_line(prefix, data)
 
     return p.wait()
 
 
-def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None):
+def vcs_checkout(source_repo, dest, store_path,
+                 base_repo=None, revision=None, branch=None):
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
     # branch flag.
     if revision:
         revision_flag = b'--revision'
         revision_value = revision
     elif branch:
         revision_flag = b'--branch'
@@ -108,25 +109,21 @@ def vcs_checkout(source_repo, dest, base
         print_line(b'vcs', 'Unable to retrieve current hg.mozilla.org fingerprint'
                            'using the secret service, using fallback instead.')
         # XXX This fingerprint will not be accurate if running on an old
         #     revision after the server fingerprint has changed.
         secret = {'secret': FALLBACK_FINGERPRINT}
 
     hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii')
 
-    # Put the shared stores as a sibling of the checkout directory. This ensures
-    # they are on the same cache.
-    share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared'))
-
     args = [
         b'/usr/bin/hg',
         b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
         b'robustcheckout',
-        b'--sharebase', share_base,
+        b'--sharebase', store_path,
         b'--purge',
     ]
 
     if base_repo:
         args.extend([b'--upstream', base_repo])
 
     args.extend([
         revision_flag, revision_value,
@@ -263,26 +260,29 @@ def main(args):
             os.makedirs(os.path.dirname(checkout))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # And that it is owned by the appropriate user/group.
         os.chown(os.path.dirname(checkout), uid, gid)
 
-        # And ensure the shared store path (derived from the checkout path)
-        # exists and has proper permissions.
-        share_path = os.path.join(os.path.dirname(checkout), 'hg-shared')
+        # And ensure the shared store path exists and has proper permissions.
+        if 'HG_STORE_PATH' not in os.environ:
+            print('error: HG_STORE_PATH environment variable not set')
+            sys.exit(1)
+
+        store_path = os.environ['HG_STORE_PATH']
         try:
-            os.makedirs(share_path)
+            os.makedirs(store_path)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
-        os.chown(share_path, uid, gid)
+        os.chown(store_path, uid, gid)
 
     prepare_checkout_dir(args.vcs_checkout)
     prepare_checkout_dir(args.tools_checkout)
 
     # Drop permissions to requested user.
     # This code is modeled after what `sudo` was observed to do in a Docker
     # container. We do not bother calling setrlimit() because containers have
     # their own limits.
@@ -302,28 +302,30 @@ def main(args):
         # reasons. Switch to mozilla-unified because robustcheckout works best
         # with it.
         if base_repo == 'https://hg.mozilla.org/mozilla-central':
             base_repo = b'https://hg.mozilla.org/mozilla-unified'
 
         os.environ['GECKO_HEAD_REV'] = vcs_checkout(
             os.environ['GECKO_HEAD_REPOSITORY'],
             args.vcs_checkout,
+            os.environ['HG_STORE_PATH'],
             base_repo=base_repo,
             revision=os.environ.get('GECKO_HEAD_REV'),
             branch=os.environ.get('GECKO_HEAD_REF'))
 
     elif not os.environ.get('GECKO_HEAD_REV') and \
             os.environ.get('GECKO_HEAD_REF'):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
     if args.tools_checkout:
         vcs_checkout(b'https://hg.mozilla.org/build/tools',
                      args.tools_checkout,
+                     os.environ['HG_STORE_PATH'],
                      # Always check out the latest commit on default branch.
                      # This is non-deterministic!
                      branch=b'default')
 
     return run_and_prefix_output(b'task', task_args)
 
 
 if __name__ == '__main__':