Bug 1293789 - Single Local TC, mozharness bits. r=rail
authorJustin Wood <Callek@gmail.com>
Sat, 03 Sep 2016 10:27:46 -0400
changeset 354098 7c27f240537616695c12232dae2a27b24c3ed757
parent 354097 dc3fed21da749ac87ede92fe98a523e664c032ce
child 354099 80a5b362f9a79142e16fc9e2244629f5155d679b
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-beta@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail
bugs1293789
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1293789 - Single Local TC, mozharness bits. r=rail MozReview-Commit-ID: mH5MKaUodA
taskcluster/scripts/builder/build-l10n.sh
testing/mozharness/configs/single_locale/mozilla-central_android-api-15.py
testing/mozharness/configs/single_locale/tc_android_api_15.py
testing/mozharness/scripts/mobile_l10n.py
old mode 100644
new mode 100755
--- a/testing/mozharness/configs/single_locale/mozilla-central_android-api-15.py
+++ b/testing/mozharness/configs/single_locale/mozilla-central_android-api-15.py
@@ -1,16 +1,18 @@
 BRANCH = "mozilla-central"
 MOZ_UPDATE_CHANNEL = "nightly"
 MOZILLA_DIR = BRANCH
 OBJDIR = "obj-l10n"
 EN_US_BINARY_URL = "http://archive.mozilla.org/pub/mobile/nightly/latest-%s-android-api-15/en-US" % BRANCH
 HG_SHARE_BASE_DIR = "/builds/hg-shared"
 
 config = {
+    # XXX For Try
+    "update_gecko_source_to_enUS": False,
     "branch": BRANCH,
     "log_name": "single_locale",
     "objdir": OBJDIR,
     "is_automation": True,
     "buildbot_json_path": "buildprops.json",
     "force_clobber": True,
     "clobberer_url": "https://api.pub.build.mozilla.org/clobberer/lastclobber",
     "locales_file": "%s/mobile/android/locales/all-locales" % MOZILLA_DIR,
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/single_locale/tc_android_api_15.py
@@ -0,0 +1,30 @@
+import os
+
+config = {
+    "locales_file": "src/mobile/android/locales/all-locales",
+    "tools_repo": "https://hg.mozilla.org/build/tools",
+    "mozconfig": "src/mobile/android/config/mozconfigs/android-api-15/l10n-nightly",
+    "tooltool_config": {
+        "manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
+        "output_dir": "%(abs_work_dir)s/src",
+    },
+    #"tooltool_servers": ['http://relengapi/tooltool/'],
+
+    #"bootstrap_env": {
+    #    "NO_MERCURIAL_SETUP_CHECK": "1",
+    #    "MOZ_OBJDIR": "obj-l10n",
+    #    "EN_US_BINARY_URL": "%(en_us_binary_url)s",
+    #    "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
+    #    "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
+    #    "DIST": "%(abs_objdir)s",
+    #    "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
+    #    "L10NBASEDIR": "../../l10n",
+    #    "MOZ_MAKE_COMPLETE_MAR": "1",
+    #    'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
+    #},
+    "upload_env": {
+        'UPLOAD_HOST': 'localhost',
+        'UPLOAD_PATH': '/home/worker/artifacts/',
+    },
+    "mozilla_dir": "src/",
+}
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -101,26 +101,41 @@ class MobileSingleLocale(MockMixin, Loca
          }
     ], [
         ['--total-chunks', ],
         {"action": "store",
          "dest": "total_locale_chunks",
          "type": "int",
          "help": "Specify the total number of chunks of locales"
          }
+    ], [
+        ["--disable-mock"],
+        {"dest": "disable_mock",
+         "action": "store_true",
+         "help": "do not run under mock despite what gecko-config says",
+        }
+    ], [
+        ['--revision', ],
+        {"action": "store",
+         "dest": "revision",
+         "type": "string",
+         "help": "Override the gecko revision to use (otherwise use buildbot supplied"
+                 " value, or en-US revision) "}
     ]]
 
     def __init__(self, require_config_file=True):
         buildscript_kwargs = {
             'all_actions': [
                 "clobber",
                 "pull",
+                "clone-locales",
                 "list-locales",
                 "setup",
                 "repack",
+                "validate-repacks-signed",
                 "upload-repacks",
                 "create-virtualenv",
                 "taskcluster-upload",
                 "submit-to-balrog",
                 "summary",
             ],
             'config': {
                 'taskcluster_credentials_file': 'oauth.txt',
@@ -351,16 +366,18 @@ class MobileSingleLocale(MockMixin, Loca
             # deepcopy() needed because of self.config lock bug :(
             for repo_dict in deepcopy(c['repos']):
                 repo_dict['repo'] = repo_dict['repo'] % replace_dict
                 repos.append(repo_dict)
         else:
             repos = c['repos']
         self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
                                 tag_override=c.get('tag_override'))
+
+    def clone_locales(self):
         self.pull_locale_source()
 
     # list_locales() is defined in LocalesMixin.
 
     def _setup_configure(self, buildid=None):
         c = self.config
         dirs = self.query_abs_dirs()
         env = self.query_repack_env()
@@ -391,17 +408,16 @@ class MobileSingleLocale(MockMixin, Loca
     def setup(self):
         c = self.config
         dirs = self.query_abs_dirs()
         mozconfig_path = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig')
         self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']),
                       mozconfig_path)
         # TODO stop using cat
         cat = self.query_exe("cat")
-        hg = self.query_exe("hg")
         make = self.query_exe("make")
         self.run_command_m([cat, mozconfig_path])
         env = self.query_repack_env()
         if self.config.get("tooltool_config"):
             self.tooltool_fetch(
                 self.config['tooltool_config']['manifest'],
                 output_dir=self.config['tooltool_config']['output_dir'] % self.query_abs_dirs(),
             )
@@ -411,55 +427,71 @@ class MobileSingleLocale(MockMixin, Loca
                            env=env,
                            error_list=MakefileErrorList,
                            halt_on_failure=True)
         self.run_command_m([make, "unpack"],
                            cwd=dirs['abs_locales_dir'],
                            env=env,
                            error_list=MakefileErrorList,
                            halt_on_failure=True)
-        revision = self.query_revision()
-        if not revision:
-            self.fatal("Can't determine revision!")
-        # TODO do this through VCSMixin instead of hardcoding hg
-        self.run_command_m([hg, "update", "-r", revision],
-                           cwd=dirs["abs_mozilla_dir"],
-                           env=env,
-                           error_list=BaseErrorList,
-                           halt_on_failure=True)
-        self.set_buildbot_property('revision', revision, write_to_file=True)
-        # Configure again since the hg update may have invalidated it.
-        buildid = self.query_buildid()
-        self._setup_configure(buildid=buildid)
+
+        # on try we want the source we already have, otherwise update to the
+        # same as the en-US binary
+        if self.config.get("update_gecko_source_to_enUS", True):
+            revision = self.query_revision()
+            if not revision:
+                self.fatal("Can't determine revision!")
+            hg = self.query_exe("hg")
+            # TODO do this through VCSMixin instead of hardcoding hg
+            self.run_command_m([hg, "update", "-r", revision],
+                               cwd=dirs["abs_mozilla_dir"],
+                               env=env,
+                               error_list=BaseErrorList,
+                               halt_on_failure=True)
+            self.set_buildbot_property('revision', revision, write_to_file=True)
+            # Configure again since the hg update may have invalidated it.
+            buildid = self.query_buildid()
+            self._setup_configure(buildid=buildid)
 
     def repack(self):
         # TODO per-locale logs and reporting.
-        c = self.config
         dirs = self.query_abs_dirs()
         locales = self.query_locales()
         make = self.query_exe("make")
         repack_env = self.query_repack_env()
-        base_package_name = self.query_base_package_name()
-        base_package_dir = os.path.join(dirs['abs_objdir'], 'dist')
         success_count = total_count = 0
         for locale in locales:
             total_count += 1
             self.enable_mock()
             result = self.run_compare_locales(locale)
             self.disable_mock()
             if result:
                 self.add_failure(locale, message="%s failed in compare-locales!" % locale)
                 continue
             if self.run_command_m([make, "installers-%s" % locale],
                                   cwd=dirs['abs_locales_dir'],
                                   env=repack_env,
                                   error_list=MakefileErrorList,
                                   halt_on_failure=False):
                 self.add_failure(locale, message="%s failed in make installers-%s!" % (locale, locale))
                 continue
+            success_count += 1
+        self.summarize_success_count(success_count, total_count,
+                                     message="Repacked %d of %d binaries successfully.")
+
+    def validate_repacks_signed(self):
+        c = self.config
+        dirs = self.query_abs_dirs()
+        locales = self.query_locales()
+        base_package_name = self.query_base_package_name()
+        base_package_dir = os.path.join(dirs['abs_objdir'], 'dist')
+        repack_env = self.query_repack_env()
+        success_count = total_count = 0
+        for locale in locales:
+            total_count += 1
             signed_path = os.path.join(base_package_dir,
                                        base_package_name % {'locale': locale})
             # We need to wrap what this function does with mock, since
             # MobileSigningMixin doesn't know about mock
             self.enable_mock()
             status = self.verify_android_signature(
                 signed_path,
                 script=c['signature_verification_script'],
@@ -468,17 +500,17 @@ class MobileSingleLocale(MockMixin, Loca
             )
             self.disable_mock()
             if status:
                 self.add_failure(locale, message="Errors verifying %s binary!" % locale)
                 # No need to rm because upload is per-locale
                 continue
             success_count += 1
         self.summarize_success_count(success_count, total_count,
-                                     message="Repacked %d of %d binaries successfully.")
+                                     message="Validated signatures on %d of %d binaries successfully.")
 
     def taskcluster_upload(self):
         auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
         credentials = {}
         execfile(auth, credentials)
         client_id = credentials.get('taskcluster_clientId')
         access_token = credentials.get('taskcluster_accessToken')
         if not client_id or not access_token:
@@ -574,30 +606,24 @@ class MobileSingleLocale(MockMixin, Loca
             parser = OutputParser(config=self.config, log_obj=self.log_obj,
                                   error_list=MakefileErrorList)
             parser.add_lines(output)
             if parser.num_errors:
                 self.add_failure(locale, message="%s failed in make upload!" % (locale))
                 continue
             package_name = base_package_name % {'locale': locale}
             r = re.compile("(http.*%s)" % package_name)
-            success = False
             for line in output.splitlines():
                 m = r.match(line)
                 if m:
                     self.upload_urls[locale] = m.groups()[0]
                     self.info("Found upload url %s" % self.upload_urls[locale])
-                    success = True
-            if not success:
-                self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale))
-                print output
-                continue
             success_count += 1
         self.summarize_success_count(success_count, total_count,
-                                     message="Uploaded %d of %d binaries successfully.")
+                                     message="Make Upload for %d of %d locales successful.")
 
     def checkout_tools(self):
         dirs = self.query_abs_dirs()
 
         # We need hg.m.o/build/tools checked out
         self.info("Checking out tools")
         repos = [{
             'repo': self.config['tools_repo'],
@@ -636,16 +662,26 @@ class MobileSingleLocale(MockMixin, Loca
         if not self.config.get("balrog_servers"):
             self.info("balrog_servers not set; skipping balrog submission.")
             return
 
         self.checkout_tools()
 
         dirs = self.query_abs_dirs()
         locales = self.query_locales()
+        balrogReady = True
+        for locale in locales:
+            apk_url = self.query_upload_url(locale)
+            if not apk_url:
+                self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale))
+                balrogReady = False
+                continue
+        if not balrogReady:
+            return self.fatal(message="Not all repacks successful, abort without submitting to balrog")
+
         for locale in locales:
             apkfile = self.query_apkfile_path(locale)
             apk_url = self.query_upload_url(locale)
 
             # Set other necessary properties for Balrog submission. None need to
             # be passed back to buildbot, so we won't write them to the properties
             #files.
             self.set_buildbot_property("locale", locale)