Bug 1440546: [partner-repack] Mozharness support for new partner repack script; r=Callek
authorNick Thomas <nthomas@mozilla.com>
Thu, 19 Apr 2018 10:01:17 -0600
changeset 468094 0a202b97948b0c517531219786fe136969052ddd
parent 468093 5ddb466ecccf327fa71304d2486797350c77e373
child 468095 2e43f41eb784c98fffccb8cfa61cefc0f745af38
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1440546
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1440546: [partner-repack] Mozharness support for new partner repack script; r=Callek Differential Revision: https://phabricator.services.mozilla.com/D973
testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
testing/mozharness/mozharness/mozilla/secrets.py
testing/mozharness/scripts/desktop_partner_repacks.py
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
@@ -1,7 +1,6 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/mozilla-sha1-manifest",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
 }
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
copy from testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
copy to testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop_EME-free.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
-    "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
+    "repack_manifests_url": "git@github.com:mozilla-partners/mozilla-EME-free-manifest.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
--- a/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
+++ b/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
@@ -1,7 +1,15 @@
 config = {
     "appName": "Firefox",
     "log_name": "partner_repack",
     "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
     "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-    "repo_url": "git@github.com:mozilla/git-repo.git",
+    "secret_files": [
+        {
+            "filename": "/builds/partner-github-ssh",
+            "secret_name": "project/releng/gecko/build/level-%(scm-level)s/partner-github-ssh",
+            "min_scm_level": 3,
+            "mode": 0o600,
+        },
+    ],
+    "ssh_key": "/builds/partner-github-ssh",
 }
--- a/testing/mozharness/mozharness/mozilla/secrets.py
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -40,16 +40,18 @@ class SecretsMixin(object):
         secret is used as the value to be written to disk.
 
         The `filename` key in the dictionary gives the filename to which the
         secret should be written.
 
         The optional `min_scm_level` key gives a minimum SCM level at which this
         secret is required.  For lower levels, the value of the 'default` key
         is used, or no secret is written.
+
+        The optional 'mode' key allows a mode change (chmod) after the file is written
         """
         secret_files = self.config.get('secret_files', [])
 
         scm_level = self.config.get('scm_level', 1)
         subst = {
             'scm-level': scm_level,
         }
 
@@ -63,8 +65,11 @@ class SecretsMixin(object):
                     secret = sf['default']
                 else:
                     self.info("No default for secret; not writing " + filename)
                     continue
             else:
                 secret = self._fetch_secret(secret_name)
 
             open(filename, "w").write(secret)
+
+            if sf.get('mode'):
+                os.chmod(filename, sf['mode'])
--- a/testing/mozharness/scripts/desktop_partner_repacks.py
+++ b/testing/mozharness/scripts/desktop_partner_repacks.py
@@ -13,28 +13,28 @@ import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.script import BaseScript
 from mozharness.mozilla.buildbot import BuildbotMixin
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.release import ReleaseMixin
+from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.log import FATAL
 
 
 # DesktopPartnerRepacks {{{1
 class DesktopPartnerRepacks(ReleaseMixin, BuildbotMixin, PurgeMixin,
-                            BaseScript, VirtualenvMixin):
+                            BaseScript, VirtualenvMixin, SecretsMixin):
     """Manages desktop partner repacks"""
     actions = [
                 "clobber",
-                "create-virtualenv",
-                "activate-virtualenv",
+                "get-secrets",
                 "setup",
                 "repack",
                 "summary",
               ]
     config_options = [
         [["--version", "-v"], {
           "dest": "version",
           "help": "Version of Firefox to repack",
@@ -46,89 +46,61 @@ class DesktopPartnerRepacks(ReleaseMixin
         [["--platform"], {
           "dest": "platform",
           "help": "Platform to repack (e.g. linux64, macosx64, ...)",
           }],
         [["--partner", "-p"], {
           "dest": "partner",
           "help": "Limit repackaging to partners matching this string",
           }],
-        [["--s3cfg"], {
-          "dest": "s3cfg",
-          "help": "Configuration file for uploading to S3 using s3cfg",
-          }],
-        [["--hgroot"], {
-          "dest": "hgroot",
-          "help": "Use a different hg server for retrieving files",
-          }],
-        [["--hgrepo"], {
-          "dest": "hgrepo",
-          "help": "Use a different base repo for retrieving files",
-          }],
-        [["--require-buildprops"], {
-            "action": "store_true",
-            "dest": "require_buildprops",
-            "default": False,
-            "help": "Read in config options (like partner) from the buildbot properties file."
-          }],
+        [["--taskid", "-t"], {
+            "dest": "taskIds",
+            "action": "extend",
+            "help": "taskId(s) of upstream tasks for vanilla Firefox artifacts",
+        }],
     ]
 
     def __init__(self):
         # fxbuild style:
         buildscript_kwargs = {
             'all_actions': DesktopPartnerRepacks.actions,
             'default_actions': DesktopPartnerRepacks.actions,
             'config': {
-                'buildbot_json_path': 'buildprops.json',
                 "log_name": "partner-repacks",
                 "hashType": "sha512",
-                'virtualenv_modules': [
-                    'requests==2.2.1',
-                    'PyHawk-with-a-single-extra-commit==0.1.5',
-                    'taskcluster==0.0.15',
-                    's3cmd==1.6.0',
-                ],
-                'virtualenv_path': 'venv',
                 'workdir': 'partner-repacks',
             },
         }
         #
 
         BaseScript.__init__(
             self,
             config_options=self.config_options,
             **buildscript_kwargs
         )
 
     def _pre_config_lock(self, rw_config):
-        self.read_buildbot_config()
-        if not self.buildbot_config:
-            self.warning("Skipping buildbot properties overrides")
-        else:
-            if self.config.get('require_buildprops', False) is True:
-                if not self.buildbot_config:
-                    self.fatal("Unable to load properties from file: %s" %
-                               self.config.get('buildbot_json_path'))
-            props = self.buildbot_config["properties"]
-            for prop in ['version', 'build_number', 'revision', 'repo_file',
-                         'repo_url', 'repack_manifests_url', 'partner']:
-                if props.get(prop):
-                    self.info("Overriding %s with %s" % (prop, props[prop]))
-                    self.config[prop] = props.get(prop)
+        if os.getenv('REPACK_MANIFESTS_URL'):
+            self.info('Overriding repack_manifests_url to %s' % os.getenv('REPACK_MANIFESTS_URL'))
+            self.config['repack_manifests_url'] = os.getenv('REPACK_MANIFESTS_URL')
+        if os.getenv('UPSTREAM_TASKIDS'):
+            self.info('Overriding taskIds with %s' % os.getenv('UPSTREAM_TASKIDS'))
+            self.config['taskIds'] = os.getenv('UPSTREAM_TASKIDS').split()
+        self.config['scm_level'] = os.environ.get('MOZ_SCM_LEVEL', '1')
 
         if 'version' not in self.config:
             self.fatal("Version (-v) not supplied.")
         if 'build_number' not in self.config:
             self.fatal("Build number (-n) not supplied.")
         if 'repo_file' not in self.config:
             self.fatal("repo_file not supplied.")
-        if 'repo_url' not in self.config:
-            self.fatal("repo_url not supplied.")
         if 'repack_manifests_url' not in self.config:
-            self.fatal("repack_manifests_url not supplied.")
+            self.fatal("repack_manifests_url not supplied in config or via REPACK_MANIFESTS_URL")
+        if 'taskIds' not in self.config:
+            self.fatal('Need upstream taskIds from command line or in UPSTREAM_TASKIDS')
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(DesktopPartnerRepacks, self).query_abs_dirs()
         for directory in abs_dirs:
             value = abs_dirs[directory]
             abs_dirs[directory] = value
@@ -144,58 +116,57 @@ class DesktopPartnerRepacks(ReleaseMixin
 
     # Actions {{{
     def _repo_cleanup(self):
         self.rmtree(self.query_abs_dirs()['abs_repo_dir'])
         self.rmtree(self.query_abs_dirs()['abs_partners_dir'])
         self.rmtree(self.query_abs_dirs()['abs_scripts_dir'])
 
     def _repo_init(self, repo):
+        partial_env = {
+            'GIT_SSH_COMMAND': 'ssh -oIdentityFile={}'.format(self.config['ssh_key'])
+        }
         status = self.run_command([repo, "init", "--no-repo-verify",
-                                   "--repo-url", self.config['repo_url'],
                                    "-u", self.config['repack_manifests_url']],
-                                  cwd=self.query_abs_dirs()['abs_work_dir'])
+                                  cwd=self.query_abs_dirs()['abs_work_dir'],
+                                  partial_env=partial_env)
         if status:
             return status
-        return self.run_command([repo, "sync"],
-                                cwd=self.query_abs_dirs()['abs_work_dir'])
+        return self.run_command([repo, "sync", "--current-branch", "--no-tags"],
+                                cwd=self.query_abs_dirs()['abs_work_dir'],
+                                partial_env=partial_env)
 
     def setup(self):
         """setup step"""
         repo = self.download_file(self.config['repo_file'],
                                   file_name='repo',
                                   parent_dir=self.query_abs_dirs()['abs_work_dir'],
                                   error_level=FATAL)
         if not os.path.exists(repo):
             self.fatal("Unable to download repo tool.")
-        self.chmod(repo, 0755)
+        self.chmod(repo, 0o755)
         self.retry(self._repo_init,
                    args=(repo,),
                    error_level=FATAL,
                    cleanup=self._repo_cleanup(),
                    good_statuses=[0],
                    sleeptime=5)
 
     def repack(self):
         """creates the repacks"""
-        repack_cmd = [sys.executable, "partner-repacks.py",
+        repack_cmd = [sys.executable, "tc-partner-repacks.py",
                       "-v", self.config['version'],
                       "-n", str(self.config['build_number'])]
         if self.config.get('platform'):
             repack_cmd.extend(["--platform", self.config['platform']])
         if self.config.get('partner'):
             repack_cmd.extend(["--partner", self.config['partner']])
-        if self.config.get('s3cfg'):
-            repack_cmd.extend(["--s3cfg", self.config['s3cfg']])
-        if self.config.get('hgroot'):
-            repack_cmd.extend(["--hgroot", self.config['hgroot']])
-        if self.config.get('hgrepo'):
-            repack_cmd.extend(["--repo", self.config['hgrepo']])
-        if self.config.get('revision'):
-            repack_cmd.extend(["--tag", self.config["revision"]])
+        if self.config.get('taskIds'):
+            for taskId in self.config['taskIds']:
+                repack_cmd.extend(["--taskid", taskId])
 
         return self.run_command(repack_cmd,
                                 cwd=self.query_abs_dirs()['abs_scripts_dir'])
 
 
 # main {{{
 if __name__ == '__main__':
     partner_repacks = DesktopPartnerRepacks()