Bug 1357126 - L10n repacks for win64
authorJustin Wood <Callek@gmail.com>
Mon, 10 Jul 2017 15:16:24 -0400
changeset 608244 bca07b439f1308ab07792c7dc07735007d913990
parent 608243 14a74a96b5e3eaff292ba1ac3ca1efbccec2138a
child 608245 538d1fc9337354100284535d37084f0ffaaaede9
push id68219
push usersfraser@mozilla.com
push dateThu, 13 Jul 2017 10:20:26 +0000
bugs1357126
milestone56.0a1
Bug 1357126 - L10n repacks for win64 MozReview-Commit-ID: BSzRM0bKvkh
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/l10n.py
testing/mozharness/configs/single_locale/tc_win32.py
testing/mozharness/configs/single_locale/tc_win64.py
testing/mozharness/scripts/desktop_l10n.py
toolkit/locales/l10n.mk
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -91,19 +91,19 @@ job-template:
                - platform-config=single_locale/macosx64.py
                - total-chunks=1
                - this-chunk=1
             default:
                - total-chunks=1
                - this-chunk=1
       actions:
          by-build-platform:
-            default: [clone-locales list-locales setup repack summary]
-            android-api-15-l10n: [clone-locales list-locales setup repack
-                                  upload-repacks summary]
+            default: ['clone-locales', 'list-locales', 'setup', 'repack', 'summary']
+            android-api-15-l10n: ['clone-locales', 'list-locales', 'setup', 'repack',
+                                  'upload-repacks', 'summary']
       script:
          by-build-platform:
             default: mozharness/scripts/desktop_l10n.py
             android-api-15-l10n: mozharness/scripts/mobile_l10n.py
    when:
       files-changed:
          - browser/locales/all-locales
          - testing/mozharness/configs/single_locale/**
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -96,19 +96,22 @@ job-template:
             EN_US_PACKAGE_NAME: target.dmg
             EN_US_BINARY_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
             MAR_TOOLS_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build/host/bin
          win.*:
             EN_US_PACKAGE_NAME: target.zip
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
+               task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/public/build
+            EN_US_INSTALLER_BINARY_URL:
+               task-reference: https://queue.taskcluster.net/v1/task/<repackage-signed>/artifacts/public/build
             MAR_TOOLS_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build/host/bin
+            NO_MAGIC_MH_BUILD_ARGS: "1"
          android-api-15-nightly:
             EN_US_PACKAGE_NAME: target.apk
             EN_US_BINARY_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build/en-US
    mozharness:
       config:
          by-build-platform:
             linux-nightly:
@@ -154,16 +157,16 @@ job-template:
                - platform-config=single_locale/win32.py
             win64-nightly:
                - environment-config=single_locale/production.py
                - branch-config=single_locale/{project}.py
                - platform-config=single_locale/win64.py
             default: [ ]
       actions:
          by-build-platform:
-            default: [clone-locales list-locales setup repack
-                      submit-to-balrog summary]
-            android-api-15-nightly: [clone-locales list-locales setup repack
-                                     upload-repacks submit-to-balrog summary]
+            default: ['clone-locales', 'list-locales', 'setup', 'repack',
+                      'submit-to-balrog', 'summary']
+            android-api-15-l10n: ['clone-locales', 'list-locales', 'setup', 'repack',
+                                  'upload-repacks', 'submit-to-balrog', 'summary']
       script:
          by-build-platform:
             default: mozharness/scripts/desktop_l10n.py
             android-api-15-nightly: mozharness/scripts/mobile_l10n.py
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -224,16 +224,17 @@ def mozharness_on_generic_worker(config,
         # force the passing of these params
         mh_command.append('--branch ' + config.params['project'])
         mh_command.append(r'--skip-buildbot-actions')
     mh_command.append(r'--work-dir %cd:Z:=z:%\build')
     for action in run.get('actions', []):
         mh_command.append('--' + action)
 
     for option in run.get('options', []):
+        assert ' ' not in option
         mh_command.append('--' + option)
 
     hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
     hg_command.append('robustcheckout')
     hg_command.extend(['--sharebase', 'y:\\hg-shared'])
     hg_command.append('--purge')
     hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
     hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -210,16 +210,22 @@ def validate_early(config, jobs):
 @transforms.add
 def setup_nightly_dependency(config, jobs):
     """ Sets up a task dependency to the signing job this relates to """
     for job in jobs:
         if not job['attributes'].get('nightly'):
             yield job
             continue  # do not add a dep unless we're a nightly
         job['dependencies'] = {'unsigned-build': job['dependent-task'].label}
+        if job['attributes']['build_platform'].startswith('win'):
+            # Weave these in and just assume they will be there in the resulting graph
+            job['dependencies'].update({
+                'signed-build': 'signing-{}'.format(job['name']),
+                'repackage-signed': 'repackage-signing-repackage-{}'.format(job['name'])
+            })
         yield job
 
 
 @transforms.add
 def handle_keyed_by(config, jobs):
     """Resolve fields that can be keyed by platform, etc."""
     fields = [
         "locales-file",
@@ -371,26 +377,24 @@ def make_job_description(config, jobs):
         if job.get('extra'):
             job_description['extra'] = job['extra']
 
         if job['worker-type'].endswith("-b-win2012"):
             job_description['worker'] = {
                 'os': 'windows',
                 'max-run-time': 7200,
             }
-            #print("XXX: CALLEK: WORKER {}".format(job['build-platform']))
         else:
             job_description['worker'] = {
                 'docker-image': {'in-tree': 'desktop-build'},
                 'max-run-time': job['run-time'],
                 'chain-of-trust': True,
             }
             job_description['run']['tooltool-downloads'] = job['tooltool']
             job_description['run']['need-xvfb'] = True
-            #print("XXX: CALLEK: OLD WORKER: {}".format(job['build-platform']))
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
                 'type': job['index'].get('type', 'generic'),
             }
 
--- a/testing/mozharness/configs/single_locale/tc_win32.py
+++ b/testing/mozharness/configs/single_locale/tc_win32.py
@@ -1,30 +1,40 @@
 import os
 
 EN_US_BINARY_URL = "%(en_us_binary_url)s"
 
 config = {
     "locales_file": "src/browser/locales/all-locales",
     "tools_repo": "https://hg.mozilla.org/build/tools",
     "mozconfig": "src/browser/config/mozconfigs/win32/l10n-mozconfig",
+    'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "obj-l10n",
         "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
+        # EN_US_INSTALLER_BINARY_URL falls back on EN_US_BINARY_URL
+        "EN_US_INSTALLER_BINARY_URL": os.environ.get(
+            "EN_US_INSTALLER_BINARY_URL", os.environ.get(
+                "EN_US_BINARY_URL", EN_US_BINARY_URL)),
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "L10NBASEDIR": "../../l10n",
         "MOZ_MAKE_COMPLETE_MAR": "1",
-        'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
+        'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE', 'c:/builds/tooltool_cache'),
+        'EN_US_PACKAGE_NAME': 'target.zip',
+        'EN_US_PKG_INST_BASENAME': 'target.installer',
     },
     "upload_env": {
         'UPLOAD_HOST': 'localhost',
-        'UPLOAD_PATH': '/home/worker/artifacts/',
+        'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
     },
 
     "tooltool_url": 'http://relengapi/tooltool/',
     'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
     "mozilla_dir": "src/",
     "simple_name_move": True,
+
+    # Override the buildbot path to not read it
+    "buildbot_json_path": None,
 }
--- a/testing/mozharness/configs/single_locale/tc_win64.py
+++ b/testing/mozharness/configs/single_locale/tc_win64.py
@@ -1,31 +1,41 @@
 import os
 
 EN_US_BINARY_URL = "%(en_us_binary_url)s"
 
 config = {
     "locales_file": "src/browser/locales/all-locales",
     "tools_repo": "https://hg.mozilla.org/build/tools",
     "mozconfig": "src/browser/config/mozconfigs/win64/l10n-mozconfig",
+    'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "obj-l10n",
         "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
+        # EN_US_INSTALLER_BINARY_URL falls back on EN_US_BINARY_URL
+        "EN_US_INSTALLER_BINARY_URL": os.environ.get(
+            "EN_US_INSTALLER_BINARY_URL", os.environ.get(
+                "EN_US_BINARY_URL", EN_US_BINARY_URL)),
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "L10NBASEDIR": "../../l10n",
         "MOZ_MAKE_COMPLETE_MAR": "1",
-        'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
+        'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE', 'c:/builds/tooltool_cache'),
+        'EN_US_PACKAGE_NAME': 'target.zip',
+        'EN_US_PKG_INST_BASENAME': 'target.installer',
     },
     "upload_env": {
         'UPLOAD_HOST': 'localhost',
-        'UPLOAD_PATH': '/home/worker/artifacts/',
+        'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
     },
 
     "tooltool_url": 'http://relengapi/tooltool/',
     'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
     "mozilla_dir": "src/",
     "simple_name_move": True,
+
+    # Override the buildbot path to not read it
+    "buildbot_json_path": None,
 }
 
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -265,16 +265,18 @@ class DesktopSingleLocale(LocalesMixin, 
             for t in unknown_tokens:
                 msg.append(t)
             self.fatal(' '.join(msg))
         self.info('configuration looks ok')
 
         self.read_buildbot_config()
         if not self.buildbot_config:
             self.warning("Skipping buildbot properties overrides")
+            # Set an empty dict
+            self.buildbot_config = {"properties": {}}
             return
         props = self.buildbot_config["properties"]
         for prop in ['mar_tools_url']:
             if props.get(prop):
                 self.info("Overriding %s with %s" % (prop, props[prop]))
                 self.config[prop] = props.get(prop)
 
     def _get_configuration_tokens(self, iterable):
@@ -796,19 +798,21 @@ class DesktopSingleLocale(LocalesMixin, 
         if self.config.get("simple_name_move"):
             # Assume an UPLOAD PATH
             upload_target = self.config["upload_env"]["UPLOAD_PATH"]
             target_path = os.path.join(upload_target, locale)
             self.mkdir_p(target_path)
             glob_name = "*.%s.*" % locale
             matches = (glob.glob(os.path.join(upload_target, glob_name)) +
                        glob.glob(os.path.join(upload_target, 'update', glob_name)) +
-                       glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)))
+                       glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)) +
+                       glob.glob(os.path.join(upload_target, 'install', 'sea', glob_name)))
             targets_exts = ["tar.bz2", "dmg", "langpack.xpi",
-                            "complete.mar", "checksums"]
+                            "complete.mar", "checksums", "zip",
+                            "installer.exe"]
             targets = ["target.%s" % ext for ext in targets_exts]
             for f in matches:
                 target_file = next(target_file for target_file in targets
                                     if f.endswith(target_file[6:]))
                 if target_file:
                     # Remove from list of available options for this locale
                     targets.remove(target_file)
                 else:
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -176,38 +176,43 @@ langpack-%: libs-%
 	$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) \
 	  -DTK_DEFINES=$(TK_DEFINES) -DAPP_DEFINES=$(APP_DEFINES) $(MOZILLA_DIR)/toolkit/locales/generic/install.rdf -o $(DIST)/xpi-stage/$(XPI_NAME)/install.rdf)
 	$(call py_action,zip,-C $(DIST)/xpi-stage/locale-$(AB_CD) $(LANGPACK_FILE) install.rdf $(PKG_ZIP_DIRS) chrome.manifest)
 
 # This variable is to allow the wget-en-US target to know which ftp server to download from
 ifndef EN_US_BINARY_URL 
 EN_US_BINARY_URL = $(error You must set EN_US_BINARY_URL)
 endif
+# In taskcluster the installer comes from another location
+ifndef EN_US_INSTALLER_BINARY_URL
+EN_US_INSTALLER_BINARY_URL = $(EN_US_BINARY_URL)
+endif
 
 # Allow the overriding of PACKAGE format so we can get an EN_US build with a different
 # PACKAGE format than we are creating l10n packages with.
 EN_US_PACKAGE_NAME ?= $(PACKAGE)
+EN_US_PKG_INST_BASENAME ?= $(PKG_INST_BASENAME)
 
 # This make target allows us to wget the latest en-US binary from a specified website
 # The make installers-% target needs the en-US binary in dist/
 # and for the windows repackages we need the .installer.exe in dist/sea
 wget-en-US:
 ifndef WGET
 	$(error Wget not installed)
 endif
 	$(NSINSTALL) -D $(ABS_DIST)/$(PKG_PATH)
 	(cd $(ABS_DIST)/$(PKG_PATH) && \
         $(WGET) --no-cache -nv --no-iri -N -O $(PACKAGE) '$(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME)')
 	@echo 'Downloaded $(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME) to $(ABS_DIST)/$(PKG_PATH)/$(PACKAGE)'
 ifdef RETRIEVE_WINDOWS_INSTALLER
 ifeq ($(OS_ARCH), WINNT)
 	$(NSINSTALL) -D $(ABS_DIST)/$(PKG_INST_PATH)
 	(cd $(ABS_DIST)/$(PKG_INST_PATH) && \
-        $(WGET) --no-cache -nv --no-iri -N '$(EN_US_BINARY_URL)/$(PKG_PATH)$(PKG_INST_BASENAME).exe')
-	@echo 'Downloaded $(EN_US_BINARY_URL)/$(PKG_PATH)$(PKG_INST_BASENAME).exe to $(ABS_DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe'
+        $(WGET) --no-cache -nv --no-iri -N -O $(PKG_INST_BASENAME).exe '$(EN_US_INSTALLER_BINARY_URL)/$(PKG_PATH)$(EN_US_PKG_INST_BASENAME).exe')
+	@echo 'Downloaded $(EN_US_INSTALLER_BINARY_URL)/$(PKG_PATH)$(EN_US_PKG_INST_BASENAME).exe to $(ABS_DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe'
 endif
 endif
 
 generate-snippet-%:
 	$(PYTHON) $(MOZILLA_DIR)/tools/update-packaging/generatesnippet.py \
           --mar-path=$(ABS_DIST)/update \
           --application-ini-file=$(STAGEDIST)/application.ini \
           --locale=$* \