Bug 1311791 - Explicitly declare path to shared hg store; r=dustin
authorGregory Szorc <gps@mozilla.com>
Thu, 20 Oct 2016 12:56:43 -0700
changeset 318976 ff5a4bab0813f5cd2b5aeb9f4d9003864721763f
parent 318975 273cfb79666c8a14f5a1054716f9cc5f7cb433d4
child 318977 a621c81ab1a260882c30498d210fbc447507baf4
push id30857
push userryanvm@gmail.com
push dateSun, 23 Oct 2016 17:08:16 +0000
treeherdermozilla-central@3d0ad7bb854d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1311791
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1311791 - Explicitly declare path to shared hg store; r=dustin 332a08725ed0 changed the store path logic in a quick and crude manner. The code could lead to multiple shared stores existing on a cache if checkouts were in different parent directories. This commit refactors the code to explicitly declare a path to the shared hg store. This restores the behavior of ensuring there is only a single shared store per cache. MozReview-Commit-ID: 19Aa1QVrVQ8
.taskcluster.yml
taskcluster/scripts/tester/run-wizard
taskcluster/taskgraph/action.yml
taskcluster/taskgraph/transforms/job/common.py
testing/docker/decision/VERSION
testing/docker/recipes/run-task
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -70,28 +70,28 @@ tasks:
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
+          HG_STORE_PATH: /home/worker/checkouts/hg-store
 
         cache:
-          level-{{level}}-hg-shared: /home/worker/hg-shared
           level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
           chainOfTrust: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the version
-        image: 'taskcluster/decision:0.1.6'
+        image: 'taskcluster/decision:0.1.7'
 
         maxRunTime: 1800
 
         # TODO use mozilla-unified for the base repository once the tc-vcs
         # tar.gz archives are created or tc-vcs isn't being used.
         command:
           - /home/worker/bin/run-task
           - '--vcs-checkout=/home/worker/checkouts/gecko'
--- a/taskcluster/scripts/tester/run-wizard
+++ b/taskcluster/scripts/tester/run-wizard
@@ -88,24 +88,20 @@ def clone():
         revision = os.environ['GECKO_HEAD_REV']
     elif os.environ.get('GECKO_HEAD_REF'):
         revision_flag = b'--branch'
         revision = os.environ['GECKO_HEAD_REF']
     else:
         print('revision is not specified for checkout')
         return 1
 
-    # Put the shared stores as a sibling of the checkout directory. This ensures
-    # they are on the same cache.
-    share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared'))
-
     # TODO Bug 1301382 - pin hg.mozilla.org fingerprint.
     call([
         b'/usr/bin/hg', b'robustcheckout',
-        b'--sharebase', share_base
+        b'--sharebase', os.environ['HG_STORE_PATH'],
         b'--purge',
         b'--upstream', b'https://hg.mozilla.org/mozilla-unified',
         revision_flag, revision,
         base_repo, dest
     ])
     print("Finished cloning to {} at revision {}.".format(dest, revision))
 
 
--- a/taskcluster/taskgraph/action.yml
+++ b/taskcluster/taskgraph/action.yml
@@ -27,26 +27,27 @@ routes:
   - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
 payload:
   env:
     GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
     GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
     GECKO_HEAD_REF: '{{head_ref}}'
     GECKO_HEAD_REV: '{{head_rev}}'
+    HG_STORE_PATH: /home/worker/checkouts/hg-store
 
   cache:
     level-{{level}}-checkouts: /home/worker/checkouts
 
   features:
     taskclusterProxy: true
 
   # Note: This task is built server side without the context or tooling that
   # exist in tree so we must hard code the version
-  image: 'taskcluster/decision:0.1.6'
+  image: 'taskcluster/decision:0.1.7'
 
   # Virtually no network or other potentially risky operations happen as part
   # of the task timeout aside from the initial clone. We intentionally have
   # set this to a lower value _all_ decision tasks should use a root
   # repository which is cached.
   maxRunTime: 1800
 
   command:
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -70,16 +70,17 @@ def docker_worker_support_vcs_checkout(c
         'name': 'level-%s-checkouts' % level,
         'mount-point': '/home/worker/checkouts',
     })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
+        'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
     })
 
     # Give task access to hgfingerprint secret so it can pin the certificate
     # for hg.mozilla.org.
     taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
     taskdesc['worker']['taskcluster-proxy'] = True
 
 
--- a/testing/docker/decision/VERSION
+++ b/testing/docker/decision/VERSION
@@ -1,1 +1,1 @@
-0.1.6
+0.1.7
--- a/testing/docker/recipes/run-task
+++ b/testing/docker/recipes/run-task
@@ -74,17 +74,18 @@ def run_and_prefix_output(prefix, args, 
         if data == b'':
             break
 
         print_line(prefix, data)
 
     return p.wait()
 
 
-def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None):
+def vcs_checkout(source_repo, dest, store_path,
+                 base_repo=None, revision=None, branch=None):
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
     # branch flag.
     if revision:
         revision_flag = b'--revision'
         revision_value = revision
     elif branch:
         revision_flag = b'--branch'
@@ -108,25 +109,21 @@ def vcs_checkout(source_repo, dest, base
         print_line(b'vcs', 'Unable to retrieve current hg.mozilla.org fingerprint'
                            'using the secret service, using fallback instead.')
         # XXX This fingerprint will not be accurate if running on an old
         #     revision after the server fingerprint has changed.
         secret = {'secret': FALLBACK_FINGERPRINT}
 
     hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii')
 
-    # Put the shared stores as a sibling of the checkout directory. This ensures
-    # they are on the same cache.
-    share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared'))
-
     args = [
         b'/usr/bin/hg',
         b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
         b'robustcheckout',
-        b'--sharebase', share_base,
+        b'--sharebase', store_path,
         b'--purge',
     ]
 
     if base_repo:
         args.extend([b'--upstream', base_repo])
 
     args.extend([
         revision_flag, revision_value,
@@ -263,26 +260,29 @@ def main(args):
             os.makedirs(os.path.dirname(checkout))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # And that it is owned by the appropriate user/group.
         os.chown(os.path.dirname(checkout), uid, gid)
 
-        # And ensure the shared store path (derived from the checkout path)
-        # exists and has proper permissions.
-        share_path = os.path.join(os.path.dirname(checkout), 'hg-shared')
+        # And ensure the shared store path exists and has proper permissions.
+        if 'HG_STORE_PATH' not in os.environ:
+            print('error: HG_STORE_PATH environment variable not set')
+            sys.exit(1)
+
+        store_path = os.environ['HG_STORE_PATH']
         try:
-            os.makedirs(share_path)
+            os.makedirs(store_path)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
-        os.chown(share_path, uid, gid)
+        os.chown(store_path, uid, gid)
 
     prepare_checkout_dir(args.vcs_checkout)
     prepare_checkout_dir(args.tools_checkout)
 
     # Drop permissions to requested user.
     # This code is modeled after what `sudo` was observed to do in a Docker
     # container. We do not bother calling setrlimit() because containers have
     # their own limits.
@@ -302,28 +302,30 @@ def main(args):
         # reasons. Switch to mozilla-unified because robustcheckout works best
         # with it.
         if base_repo == 'https://hg.mozilla.org/mozilla-central':
             base_repo = b'https://hg.mozilla.org/mozilla-unified'
 
         os.environ['GECKO_HEAD_REV'] = vcs_checkout(
             os.environ['GECKO_HEAD_REPOSITORY'],
             args.vcs_checkout,
+            os.environ['HG_STORE_PATH'],
             base_repo=base_repo,
             revision=os.environ.get('GECKO_HEAD_REV'),
             branch=os.environ.get('GECKO_HEAD_REF'))
 
     elif not os.environ.get('GECKO_HEAD_REV') and \
             os.environ.get('GECKO_HEAD_REF'):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
     if args.tools_checkout:
         vcs_checkout(b'https://hg.mozilla.org/build/tools',
                      args.tools_checkout,
+                     os.environ['HG_STORE_PATH'],
                      # Always check out the latest commit on default branch.
                      # This is non-deterministic!
                      branch=b'default')
 
     return run_and_prefix_output(b'task', task_args)
 
 
 if __name__ == '__main__':