Bug 1440546: [partner-repack] Mozharness support for new partner repack script; r=Callek, a=release
authorNick Thomas <nthomas@mozilla.com>
Thu, 19 Apr 2018 10:01:17 -0600
changeset 463414 fc7c05c57f5b1a204439762555279032da67e627
parent 463413 1a2759c169162bd18b6e49415a56e5177b925252
child 463415 cc9fa41c9580b3169b7b6f6da14878420751214f
push id1683
push usersfraser@mozilla.com
push dateThu, 26 Apr 2018 16:43:40 +0000
treeherdermozilla-release@5af6cb21869d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek, release
bugs1440546
milestone60.0
Bug 1440546: [partner-repack] Mozharness support for new partner repack script; r=Callek, a=release Differential Revision: https://phabricator.services.mozilla.com/D973
testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
testing/mozharness/mozharness/mozilla/secrets.py
testing/mozharness/scripts/desktop_partner_repacks.py
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
@@ -1,7 +1,6 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/mozilla-sha1-manifest",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
 }
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
copy from testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
copy to testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
-    "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
+    "repack_manifests_url": "git@github.com:mozilla-partners/mozilla-EME-free-manifest.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
--- a/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
--- a/testing/mozharness/mozharness/mozilla/secrets.py
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -40,16 +40,18 @@ class SecretsMixin(object):
         secret is used as the value to be written to disk.
 
         The `filename` key in the dictionary gives the filename to which the
         secret should be written.
 
         The optional `min_scm_level` key gives a minimum SCM level at which this
         secret is required.  For lower levels, the value of the 'default` key
         is used, or no secret is written.
+
+        The optional 'mode' key allows a mode change (chmod) after the file is written
         """
         secret_files = self.config.get('secret_files', [])
 
         scm_level = self.config.get('scm_level', 1)
         subst = {
             'scm-level': scm_level,
         }
 
@@ -63,8 +65,11 @@ class SecretsMixin(object):
                     secret = sf['default']
                 else:
                     self.info("No default for secret; not writing " + filename)
                     continue
             else:
                 secret = self._fetch_secret(secret_name)
 
             open(filename, "w").write(secret)
+
+            if sf.get('mode'):
+                os.chmod(filename, sf['mode'])
--- a/testing/mozharness/scripts/desktop_partner_repacks.py
+++ b/testing/mozharness/scripts/desktop_partner_repacks.py
@@ -13,28 +13,28 @@ import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.script import BaseScript
 from mozharness.mozilla.buildbot import BuildbotMixin
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.release import ReleaseMixin
+from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.log import FATAL
 
 
 # DesktopPartnerRepacks {{{1
 class DesktopPartnerRepacks(ReleaseMixin, BuildbotMixin, PurgeMixin,
-                            BaseScript, VirtualenvMixin):
+                            BaseScript, VirtualenvMixin, SecretsMixin):
     """Manages desktop partner repacks"""
     actions = [
                 "clobber",
-                "create-virtualenv",
-                "activate-virtualenv",
+                "get-secrets",
                 "setup",
                 "repack",
                 "summary",
               ]
     config_options = [
         [["--version", "-v"], {
           "dest": "version",
           "help": "Version of Firefox to repack",
@@ -46,89 +46,61 @@ class DesktopPartnerRepacks(ReleaseMixin
         [["--platform"], {
           "dest": "platform",
           "help": "Platform to repack (e.g. linux64, macosx64, ...)",
           }],
         [["--partner", "-p"], {
           "dest": "partner",
           "help": "Limit repackaging to partners matching this string",
           }],
-        [["--s3cfg"], {
-          "dest": "s3cfg",
-          "help": "Configuration file for uploading to S3 using s3cfg",
-          }],
-        [["--hgroot"], {
-          "dest": "hgroot",
-          "help": "Use a different hg server for retrieving files",
-          }],
-        [["--hgrepo"], {
-          "dest": "hgrepo",
-          "help": "Use a different base repo for retrieving files",
-          }],
-        [["--require-buildprops"], {
-            "action": "store_true",
-            "dest": "require_buildprops",
-            "default": False,
-            "help": "Read in config options (like partner) from the buildbot properties file."
-          }],
+        [["--taskid", "-t"], {
+            "dest": "taskIds",
+            "action": "extend",
+            "help": "taskId(s) of upstream tasks for vanilla Firefox artifacts",
+        }],
     ]
 
     def __init__(self):
         # fxbuild style:
         buildscript_kwargs = {
             'all_actions': DesktopPartnerRepacks.actions,
             'default_actions': DesktopPartnerRepacks.actions,
             'config': {
-                'buildbot_json_path': 'buildprops.json',
                 "log_name": "partner-repacks",
                 "hashType": "sha512",
-                'virtualenv_modules': [
-                    'requests==2.2.1',
-                    'PyHawk-with-a-single-extra-commit==0.1.5',
-                    'taskcluster==0.0.15',
-                    's3cmd==1.6.0',
-                ],
-                'virtualenv_path': 'venv',
                 'workdir': 'partner-repacks',
             },
         }
         #
 
         BaseScript.__init__(
             self,
             config_options=self.config_options,
             **buildscript_kwargs
         )
 
     def _pre_config_lock(self, rw_config):
-        self.read_buildbot_config()
-        if not self.buildbot_config:
-            self.warning("Skipping buildbot properties overrides")
-        else:
-            if self.config.get('require_buildprops', False) is True:
-                if not self.buildbot_config:
-                    self.fatal("Unable to load properties from file: %s" %
-                               self.config.get('buildbot_json_path'))
-            props = self.buildbot_config["properties"]
-            for prop in ['version', 'build_number', 'revision', 'repo_file',
-                         'repo_url', 'repack_manifests_url', 'partner']:
-                if props.get(prop):
-                    self.info("Overriding %s with %s" % (prop, props[prop]))
-                    self.config[prop] = props.get(prop)
+        if os.getenv('REPACK_MANIFESTS_URL'):
+            self.info('Overriding repack_manifests_url to %s' % os.getenv('REPACK_MANIFESTS_URL'))
+            self.config['repack_manifests_url'] = os.getenv('REPACK_MANIFESTS_URL')
+        if os.getenv('UPSTREAM_TASKIDS'):
+            self.info('Overriding taskIds with %s' % os.getenv('UPSTREAM_TASKIDS'))
+            self.config['taskIds'] = os.getenv('UPSTREAM_TASKIDS').split()
+        self.config['scm_level'] = os.environ.get('MOZ_SCM_LEVEL', '1')
 
         if 'version' not in self.config:
             self.fatal("Version (-v) not supplied.")
         if 'build_number' not in self.config:
             self.fatal("Build number (-n) not supplied.")
         if 'repo_file' not in self.config:
             self.fatal("repo_file not supplied.")
-        if 'repo_url' not in self.config:
-            self.fatal("repo_url not supplied.")
         if 'repack_manifests_url' not in self.config:
-            self.fatal("repack_manifests_url not supplied.")
+            self.fatal("repack_manifests_url not supplied in config or via REPACK_MANIFESTS_URL")
+        if 'taskIds' not in self.config:
+            self.fatal('Need upstream taskIds from command line or in UPSTREAM_TASKIDS')
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(DesktopPartnerRepacks, self).query_abs_dirs()
         for directory in abs_dirs:
             value = abs_dirs[directory]
             abs_dirs[directory] = value
@@ -144,58 +116,57 @@ class DesktopPartnerRepacks(ReleaseMixin
 
     # Actions {{{
     def _repo_cleanup(self):
         self.rmtree(self.query_abs_dirs()['abs_repo_dir'])
         self.rmtree(self.query_abs_dirs()['abs_partners_dir'])
         self.rmtree(self.query_abs_dirs()['abs_scripts_dir'])
 
     def _repo_init(self, repo):
+        partial_env = {
+            'GIT_SSH_COMMAND': 'ssh -oIdentityFile={}'.format(self.config['ssh_key'])
+        }
         status = self.run_command([repo, "init", "--no-repo-verify",
-                                   "--repo-url", self.config['repo_url'],
                                    "-u", self.config['repack_manifests_url']],
-                                  cwd=self.query_abs_dirs()['abs_work_dir'])
+                                  cwd=self.query_abs_dirs()['abs_work_dir'],
+                                  partial_env=partial_env)
         if status:
             return status
-        return self.run_command([repo, "sync"],
-                                cwd=self.query_abs_dirs()['abs_work_dir'])
+        return self.run_command([repo, "sync", "--current-branch", "--no-tags"],
+                                cwd=self.query_abs_dirs()['abs_work_dir'],
+                                partial_env=partial_env)
 
     def setup(self):
         """setup step"""
         repo = self.download_file(self.config['repo_file'],
                                   file_name='repo',
                                   parent_dir=self.query_abs_dirs()['abs_work_dir'],
                                   error_level=FATAL)
         if not os.path.exists(repo):
             self.fatal("Unable to download repo tool.")
-        self.chmod(repo, 0755)
+        self.chmod(repo, 0o755)
         self.retry(self._repo_init,
                    args=(repo,),
                    error_level=FATAL,
                    cleanup=self._repo_cleanup(),
                    good_statuses=[0],
                    sleeptime=5)
 
     def repack(self):
         """creates the repacks"""
-        repack_cmd = [sys.executable, "partner-repacks.py",
+        repack_cmd = [sys.executable, "tc-partner-repacks.py",
                       "-v", self.config['version'],
                       "-n", str(self.config['build_number'])]
         if self.config.get('platform'):
             repack_cmd.extend(["--platform", self.config['platform']])
         if self.config.get('partner'):
             repack_cmd.extend(["--partner", self.config['partner']])
-        if self.config.get('s3cfg'):
-            repack_cmd.extend(["--s3cfg", self.config['s3cfg']])
-        if self.config.get('hgroot'):
-            repack_cmd.extend(["--hgroot", self.config['hgroot']])
-        if self.config.get('hgrepo'):
-            repack_cmd.extend(["--repo", self.config['hgrepo']])
-        if self.config.get('revision'):
-            repack_cmd.extend(["--tag", self.config["revision"]])
+        if self.config.get('taskIds'):
+            for taskId in self.config['taskIds']:
+                repack_cmd.extend(["--taskid", taskId])
 
         return self.run_command(repack_cmd,
                                 cwd=self.query_abs_dirs()['abs_scripts_dir'])
 
 
 # main {{{
 if __name__ == '__main__':
     partner_repacks = DesktopPartnerRepacks()