Bug 1381577 - Part O; Run mach repackage for windows nightlies. r=kmoir
authorJustin Wood <Callek@gmail.com>
Tue, 18 Jul 2017 12:13:31 -0400
changeset 418978 362aeaaac09ce7e01520b04c8180f71a881644fc
parent 418977 8fed3841b32a327f9fa0ed2e5261848de0e7b7df
child 418979 428371d18530d130588f2ee545032c0752fbc1ad
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmoir
bugs1381577
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1381577 - Part O; Run mach repackage for windows nightlies. r=kmoir Land date changes to support windows nightlies onto central This patch also allows us to not reconfigure to run repackage on windows, which saves us both in potential pain points and execution time. MozReview-Commit-ID: xnz98Z5N06
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/repackage.py
testing/mozharness/configs/repackage/win32_signed.py
testing/mozharness/configs/repackage/win64_signed.py
testing/mozharness/scripts/repackage.py
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -10,8 +10,10 @@ transforms:
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - build-signing
 
 only-for-build-platforms:
   - macosx64-nightly/opt
+  - win32-nightly/opt
+  - win64-nightly/opt
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -73,16 +73,17 @@ JOB_NAME_WHITELIST = set([
     'sm-package-opt',
     'sm-plain-opt',
     'sm-plaindebug-debug',
     'sm-rootanalysis-debug',
     'sm-tsan-opt',
     'win32-add-on-devel',
     'win32-clang-tidy',
     'win32-debug',
+    'win32-nightly-repackage',
     'win32-noopt-debug',
     'win32-opt',
     'win32-pgo',
     'win32-st-an-debug',
     'win32-st-an-opt',
     'win32-dmd-opt',
     'win64-add-on-devel',
     'win64-clang-tidy',
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -133,24 +133,27 @@ def make_job_description(config, jobs):
             'extra-workspace-cache-key': 'repackage',
         }
 
         worker = {
             'env': _generate_task_env(build_platform, build_task_ref,
                                       signing_task_ref, locale=locale),
             'artifacts': _generate_task_output_files(build_platform, locale=locale),
             'chain-of-trust': True,
-            'max-run-time': 3600,
+            'max-run-time': 7200 if build_platform.startswith('win') else 3600,
         }
 
         if locale:
             # Make sure we specify the locale-specific upload dir
             worker['env'].update(LOCALE=locale)
 
-        if build_platform.startswith('macosx'):
+        if build_platform.startswith('win'):
+            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
+            run['use-magic-mh-args'] = False
+        elif build_platform.startswith('macosx'):
             worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
 
             run['tooltool-downloads'] = 'internal'
             worker['docker-image'] = {"in-tree": "desktop-build"},
 
             cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
             cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
@@ -169,29 +172,46 @@ def make_job_description(config, jobs):
             'run': run,
         }
         yield task
 
 
 def _generate_task_mozharness_config(build_platform):
     if build_platform.startswith('macosx'):
         return ['repackage/osx_signed.py']
+    elif build_platform.startswith('win'):
+        return ['repackage/win32_signed.py'] if '32' in build_platform \
+            else ['repackage/win64_signed.py']
     else:
         raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
 
 
 def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
     mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
     signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
 
     if build_platform.startswith('macosx'):
         return {
             'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)},
             'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
         }
+    elif build_platform.startswith('win'):
+        task_env = {
+            'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
+            'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
+            'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)},
+        }
+
+        # Stub installer is only generated on win32
+        if '32' in build_platform:
+            task_env['SIGNED_SETUP_STUB'] = {
+                'task-reference': '{}setup-stub.exe'.format(signed_prefix),
+            }
+        return task_env
+
     else:
         raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
 
 
 def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
     if locale:
         postfix = '{}/{}'.format(locale, postfix)
 
@@ -207,10 +227,30 @@ def _generate_task_output_files(build_pl
                     .format(locale_output_path),
             'name': 'public/build/{}target.dmg'.format(locale_output_path),
         }, {
             'type': 'file',
             'path': '/home/worker/workspace/build/artifacts/{}target.complete.mar'
                     .format(locale_output_path),
             'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
         }]
+    elif build_platform.startswith('win'):
+        output_files = [{
+            'type': 'file',
+            'path': 'public/build/{}target.installer.exe'.format(locale_output_path),
+            'name': 'public/build/{}target.installer.exe'.format(locale_output_path),
+        }, {
+            'type': 'file',
+            'path': 'public/build/{}target.complete.mar'.format(locale_output_path),
+            'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
+        }]
+
+        # Stub installer is only generated on win32
+        if '32' in build_platform:
+            output_files.append({
+                'type': 'file',
+                'path': 'public/build/{}target.stub-installer.exe'.format(locale_output_path),
+                'name': 'public/build/{}target.stub-installer.exe'.format(locale_output_path),
+            })
+
+        return output_files
     else:
         raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/repackage/win32_signed.py
@@ -0,0 +1,51 @@
+import os
+import sys
+
+platform = "win32"
+
+download_config = {
+        "target.zip": os.environ.get("SIGNED_ZIP"),
+        "setup.exe": os.environ.get("SIGNED_SETUP"),
+        "mar.exe": os.environ.get("UNSIGNED_MAR"),
+        # stub installer is only built on win32
+        "setup-stub.exe": os.environ.get("SIGNED_SETUP_STUB"),
+    }
+
+repackage_config = [[
+        "installer",
+        "--package", "{abs_work_dir}\\inputs\\target.zip",
+        "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag",
+        "--setupexe", "{abs_work_dir}\\inputs\\setup.exe",
+        "-o", "{output_home}\\target.installer.exe"
+    ], [
+        "mar",
+        "-i", "{abs_work_dir}\\inputs\\target.zip",
+        "--mar", "{abs_work_dir}\\inputs\\mar.exe",
+        "-o", "{output_home}\\target.complete.mar"
+    ], [
+        "installer",
+        "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\stub.tag",
+         "--setupexe", "{abs_work_dir}\\inputs\\setup-stub.exe",
+         "-o", "{output_home}\\target.stub-installer.exe"
+    ]]
+
+config = {
+    "input_home": "{abs_work_dir}\\inputs",
+    "output_home": "{base_work_dir}\\public\\build{locale}",
+    "src_mozconfig": "browser/config/mozconfigs/{}/repack".format(platform),
+
+    "locale": os.environ.get("LOCALE"),
+
+    "download_config": download_config,
+
+    "repackage_config": repackage_config,
+
+    # ToolTool
+    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
+    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_script': [sys.executable,
+                        'C:/mozilla-build/tooltool.py'],
+    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+
+    'run_configure': False,
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/repackage/win64_signed.py
@@ -0,0 +1,44 @@
+import os
+import sys
+
+platform = "win64"
+
+download_config = {
+        "target.zip": os.environ.get("SIGNED_ZIP"),
+        "setup.exe": os.environ.get("SIGNED_SETUP"),
+        "mar.exe": os.environ.get("UNSIGNED_MAR"),
+    }
+
+repackage_config = [[
+        "installer",
+        "--package", "{abs_work_dir}\\inputs\\target.zip",
+        "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag",
+        "--setupexe", "{abs_work_dir}\\inputs\\setup.exe",
+        "-o", "{output_home}\\target.installer.exe"
+    ], [
+        "mar",
+        "-i", "{abs_work_dir}\\inputs\\target.zip",
+        "--mar", "{abs_work_dir}\\inputs\\mar.exe",
+        "-o", "{output_home}\\target.complete.mar"
+    ]]
+
+config = {
+    "input_home": "{abs_work_dir}\\inputs",
+    "output_home": "{base_work_dir}\\public\\build{locale}",
+    "src_mozconfig": "browser/config/mozconfigs/{}/repack".format(platform),
+
+    "locale": os.environ.get("LOCALE"),
+
+    "download_config": download_config,
+
+    "repackage_config": repackage_config,
+
+    # ToolTool
+    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
+    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_script': [sys.executable,
+                        'C:/mozilla-build/tooltool.py'],
+    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+
+    'run_configure': False,
+}
--- a/testing/mozharness/scripts/repackage.py
+++ b/testing/mozharness/scripts/repackage.py
@@ -38,35 +38,36 @@ class Repackage(BaseScript):
                 self.fatal("Unable to fetch signed input from %s" % url)
 
             if 'mar' in path:
                 # Ensure mar is executable
                 self.chmod(os.path.join(input_home, path), 0755)
 
     def setup(self):
         self._run_tooltool()
-        self._get_mozconfig()
-        self._run_configure()
+        if self.config.get("run_configure", True):
+            self._get_mozconfig()
+            self._run_configure()
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(Repackage, self).query_abs_dirs()
         config = self.config
         for directory in abs_dirs:
             value = abs_dirs[directory]
             abs_dirs[directory] = value
+
         dirs = {}
         dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
         dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
         locale_dir = ''
         if config.get('locale'):
             locale_dir = "{}{}".format(os.path.sep, config['locale'])
         dirs['output_home'] = config['output_home'].format(locale=locale_dir, **abs_dirs)
-
         for key in dirs.keys():
             if key not in abs_dirs:
                 abs_dirs[key] = dirs[key]
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     def repackage(self):
         config = self.config