Bug 1538134: [mozharness] Set repository path in taskcluster; r=aki
authorTom Prince <mozilla@hocat.ca>
Mon, 25 Mar 2019 16:00:14 +0000
changeset 525733 bb986fa5395d4cbc0660e401ed62ae5c214023c2
parent 525732 7bd47d94daba0c54a474f737d5f0f6fd529f4f0e
child 525734 3c0a452881327cecc40fe36a073c077e0f26f188
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1538134
milestone67.0
Bug 1538134: [mozharness] Set repository path in taskcluster; r=aki Taskcluster has authorative information about the repository being built, so pass that to mozharness, rather than have mozharness reconstruct it from hand-maintained mozharness config. Differential Revision: https://phabricator.services.mozilla.com/D24561
taskcluster/taskgraph/transforms/job/mozharness.py
testing/mozharness/configs/builds/branch_specifics.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -22,16 +22,20 @@ from taskgraph.transforms.job.common imp
     docker_worker_add_workspace_cache,
     setup_secrets,
     docker_worker_add_artifacts,
     docker_worker_add_tooltool,
     generic_worker_add_artifacts,
     generic_worker_hg_commands,
     support_vcs_checkout,
 )
+from taskgraph.transforms.task import (
+    get_branch_repo,
+    get_branch_rev,
+)
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
     Required('script'): basestring,
 
@@ -162,17 +166,18 @@ def mozharness_on_docker_worker_setup(co
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'GECKO_PATH': '{workdir}/workspace/build/src'.format(**run),
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
-        'MOZ_SOURCE_CHANGESET': env['GECKO_HEAD_REV'],
+        'MOZ_SOURCE_CHANGESET': get_branch_rev(config),
+        'MOZ_SOURCE_REPO': get_branch_repo(config),
         'MH_BUILD_POOL': 'taskcluster',
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
         'PYTHONUNBUFFERED': '1',
     })
 
     if 'actions' in run:
@@ -191,19 +196,16 @@ def mozharness_on_docker_worker_setup(co
         env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
 
     if 'job-script' in run:
         env['JOB_SCRIPT'] = run['job-script']
 
     if config.params.is_try():
         env['TRY_COMMIT_MSG'] = config.params['message']
 
-    if run['comm-checkout']:
-        env['MOZ_SOURCE_CHANGESET'] = env['COMM_HEAD_REV']
-
     # if we're not keeping artifacts, set some env variables to empty values
     # that will cause the build process to skip copying the results to the
     # artifacts directory.  This will have no effect for operations that are
     # not builds.
     if not run['keep-artifacts']:
         env['DIST_TARGET_UPLOADS'] = ''
         env['DIST_UPLOADS'] = ''
 
@@ -270,34 +272,32 @@ def mozharness_on_generic_worker(config,
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
         'MH_BRANCH': config.params['project'],
-        'MOZ_SOURCE_CHANGESET': env['GECKO_HEAD_REV'],
+        'MOZ_SOURCE_CHANGESET': get_branch_rev(config),
+        'MOZ_SOURCE_REPO': get_branch_repo(config),
     })
     if run['use-simple-package']:
         env.update({'MOZ_SIMPLE_PACKAGE_NAME': 'target'})
 
     if 'extra-config' in run:
         env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
 
     # The windows generic worker uses batch files to pass environment variables
     # to commands.  Setting a variable to empty in a batch file unsets, so if
     # there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that
     # mozharness doesn't try to find the commit message on its own.
     if config.params.is_try():
         env['TRY_COMMIT_MSG'] = config.params['message'] or 'no commit message'
 
-    if run['comm-checkout']:
-        env['MOZ_SOURCE_CHANGESET'] = env['COMM_HEAD_REV']
-
     if not job['attributes']['build_platform'].startswith('win'):
         raise Exception(
             "Task generation for mozharness build jobs currently only supported on Windows"
         )
 
     mh_command = [r'c:\mozilla-build\python\python.exe']
     mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
 
--- a/testing/mozharness/configs/builds/branch_specifics.py
+++ b/testing/mozharness/configs/builds/branch_specifics.py
@@ -6,17 +6,16 @@
 
 # example config and explanation of how it works:
 # config = {
 #     # if a branch matches a key below, override items in self.config with
 #     # items in the key's value.
 #     # this override can be done for every platform or at a platform level
 #     '<branch-name>': {
 #         # global config items (applies to all platforms and build types)
-#         'repo_path': "projects/<branch-name>",
 #
 #         # platform config items (applies to specific platforms)
 #         'platform_overrides': {
 #             # if a platform matches a key below, override items in
 #             # self.config with items in the key's value
 #             'linux64-debug': {
 #                 'upload_symbols': False,
 #             },
@@ -25,48 +24,41 @@
 #             },
 #         }
 #     },
 # }
 
 config = {
     ### release branches
     "mozilla-central": {
-        "repo_path": 'mozilla-central',
     },
     'mozilla-release': {
         'enable_release_promotion': True,
-        'repo_path': 'releases/mozilla-release',
         'branch_uses_per_checkin_strategy': True,
     },
     'mozilla-esr60': {
         'enable_release_promotion': True,
-        'repo_path': 'releases/mozilla-esr60',
         'branch_uses_per_checkin_strategy': True,
     },
     'mozilla-beta': {
         'enable_release_promotion': 1,
-        'repo_path': 'releases/mozilla-beta',
         'branch_uses_per_checkin_strategy': True,
     },
     'try': {
-        'repo_path': 'try',
         'branch_supports_uploadsymbols': False,
     },
 
     ### project branches
     #'fx-team': {},   #Bug 1296396
     'gum': {
         'branch_uses_per_checkin_strategy': True,
     },
     'mozilla-inbound': {
-        'repo_path': 'integration/mozilla-inbound',
     },
     'autoland': {
-        'repo_path': 'integration/autoland',
     },
     'ux': {},
     'cypress': {
         # bug 1164935
         'branch_uses_per_checkin_strategy': True,
     },
 
     ### other branches that do not require anything special:
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -50,18 +50,16 @@ from mozharness.base.python import (
 
 AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
 AUTOMATION_EXIT_CODES.sort()
 
 MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
 Please add this to your config."
 
 ERROR_MSGS = {
-    'undetermined_repo_path': 'The repo could not be determined. \
-Please make sure that "repo" is in your config.',
     'comments_undetermined': '"comments" could not be determined. This may be \
 because it was a forced build.',
     'tooltool_manifest_undetermined': '"tooltool_manifest_src" not set, \
 Skipping run_tooltool...',
 }
 
 
 # Output Parsers
@@ -630,17 +628,16 @@ class BuildScript(AutomationMixin,
         self.branch = self.config.get('branch')
         self.stage_platform = self.config.get('stage_platform')
         if not self.branch or not self.stage_platform:
             if not self.branch:
                 self.error("'branch' not determined and is required")
             if not self.stage_platform:
                 self.error("'stage_platform' not determined and is required")
             self.fatal("Please add missing items to your config")
-        self.repo_path = None
         self.buildid = None
         self.query_buildid()  # sets self.buildid
         self.generated_build_props = False
         self.client_id = None
         self.access_token = None
 
         # Call this before creating the virtualenv so that we can support
         # substituting config values with other config values.
@@ -790,34 +787,16 @@ or run without that action (ie: --no-{ac
         if self.objdir:
             return self.objdir
 
         if not self.config.get('objdir'):
             return self.fatal(MISSING_CFG_KEY_MSG % ('objdir',))
         self.objdir = self.config['objdir']
         return self.objdir
 
-    def _query_repo(self):
-        if self.repo_path:
-            return self.repo_path
-        c = self.config
-
-        # we actually supply the repo in mozharness so if it's in
-        #  the config, we use that (automation does not require it in
-        # props)
-        if not c.get('repo_path'):
-            repo_path = 'projects/%s' % (self.branch,)
-            self.info(
-                "repo_path not in config. Using '%s' instead" % (repo_path,)
-            )
-        else:
-            repo_path = c['repo_path']
-        self.repo_path = '%s/%s' % (c['repo_base'], repo_path,)
-        return self.repo_path
-
     def query_is_nightly_promotion(self):
         platform_enabled = self.config.get('enable_nightly_promotion')
         branch_enabled = self.branch in self.config.get('nightly_promotion_branches')
         return platform_enabled and branch_enabled
 
     def query_build_env(self, **kwargs):
         c = self.config
 
@@ -825,23 +804,16 @@ or run without that action (ie: --no-{ac
         # as we don't always want every key below added to the same dict
         env = copy.deepcopy(
             super(BuildScript, self).query_env(**kwargs)
         )
 
         # first grab the buildid
         env['MOZ_BUILD_DATE'] = self.query_buildid()
 
-        # Set the source repository to what we're building from since
-        # the default is to query `hg paths` which isn't reliable with pooled
-        # storage
-        repo_path = self._query_repo()
-        assert repo_path
-        env['MOZ_SOURCE_REPO'] = repo_path
-
         if self.query_is_nightly() or self.query_is_nightly_promotion():
             # in branch_specifics.py we might set update_channel explicitly
             if c.get('update_channel'):
                 update_channel = c['update_channel']
                 if isinstance(update_channel, unicode):
                     update_channel = update_channel.encode("utf-8")
                 env["MOZ_UPDATE_CHANNEL"] = update_channel
             elif c.get('enable_release_promotion'):