Bug 1282959 - implement new relpro release sanity. refactoring upon review. r=rail
authorMihai Tabara <mtabara@mozilla.com>
Mon, 18 Jul 2016 01:06:22 +0100
changeset 6910 53eab5ff4978fb25e6dad0336f08bded275904ed
parent 6909 c0e9a28a0548f2a3d3fab1ba08869058c37eceb6
child 6911 82fcd04ef489ec29ed0079c52430205e1dce7eb3
push id5153
push usermtabara@mozilla.com
push dateMon, 18 Jul 2016 13:17:04 +0000
reviewersrail
bugs1282959
Bug 1282959 - implement new relpro release sanity. refactoring upon review. r=rail MozReview-Commit-ID: 5qmn3MnUfJW
buildfarm/release/release-runner.py
lib/python/kickoff/__init__.py
lib/python/kickoff/sanity.py
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -76,48 +76,16 @@ def update_channels(version, mappings):
 
     """
     for pattern, channels in mappings:
         if re.match(pattern, version):
             return channels
     raise RuntimeError("Cannot find update channels for %s" % version)
 
 
-def get_display_version(repo_path, revision):
-    """Get display version from remote repository
-
-    >>> get_display_version("releases/mozilla-beta", "59f372c35b24")
-    '46.0b3'
-
-    >>> get_display_version("releases/mozilla-beta", "59f372c35b2416ac84d6572d64c49227481a8a6c")
-    '46.0b3'
-    """
-    # The location is the same for both Firefox and Fennec
-    url = "https://hg.mozilla.org/{repo_path}/raw-file/{revision}/browser/config/version_display.txt"
-    url = url.format(repo_path=repo_path, revision=revision)
-
-    def _get():
-        req = requests.get(url, timeout=60)
-        req.raise_for_status()
-        return req.content.strip()
-
-    return retry(_get)
-
-
-def validate_version(repo_path, revision, version):
-    actual_version = get_display_version(repo_path, revision)
-    if version != actual_version:
-        raise SanityException(
-            "In-tree version '%s' doesn't match ship-it version '%s'" %
-            (actual_version, version))
-    else:
-        log.info("In-tree version '%s' matches ship-it version '%s'",
-                 actual_version, version)
-
-
 def validate_signatures(checksums, signature, dir_path, gpg_key_path):
     try:
         cmd = ['gpg', '--batch', '--homedir', dir_path, '--import',
                gpg_key_path]
         subprocess.check_call(cmd)
         cmd = ['gpg', '--homedir', dir_path, '--verify', signature, checksums]
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
@@ -243,22 +211,18 @@ def get_hash(path, hash_type="sha512"):
     h = hashlib.new(hash_type)
     with open(path, "rb") as f:
         for chunk in iter(functools.partial(f.read, 4096), ''):
             h.update(chunk)
     return h.hexdigest()
 
 
 def validate_graph_kwargs(queue, gpg_key_path, **kwargs):
-    # We don't export "esr" in the version
-    # TODO: redundant, to be removed soon, once new relpro sanity is in place
-    validate_version(repo_path=kwargs["repo_path"],
-                     revision=kwargs["revision"],
-                     version=kwargs["version"].replace("esr", ""))
     # TODO: to be moved under kickoff soon, once new relpro sanity is in place
+    # bug 1282959
     platforms = kwargs.get('en_US_config', {}).get('platforms', {})
     for platform in platforms.keys():
         task_id = platforms.get(platform).get('task_id', {})
         log.info('Performing release sanity for %s en-US binary', platform)
         sanitize_en_US_binary(queue, task_id, gpg_key_path)
 
     log.info("Release sanity for all en-US is now completed!")
 
@@ -408,16 +372,17 @@ def main(options):
                     l10n_platforms=branchConfig['l10n_release_platforms'],
                     l10n_changesets=l10n_changesets
                 ),
                 "en_US_config": get_en_US_config(
                     index=index, product=release["product"], branch=branch,
                     revision=release['mozillaRevision'],
                     platforms=branchConfig['release_platforms']
                 ),
+                "dashboard_check": release['dashboardCheck'],
                 "verifyConfigs": {},
                 "balrog_api_root": branchConfig["balrog_api_root"],
                 "funsize_balrog_api_root": branchConfig["funsize_balrog_api_root"],
                 "balrog_username": balrog_username,
                 "balrog_password": balrog_password,
                 "beetmover_aws_access_key_id": beetmover_aws_access_key_id,
                 "beetmover_aws_secret_access_key": beetmover_aws_secret_access_key,
                 # TODO: stagin specific, make them configurable
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -228,17 +228,17 @@ def bump_version(version):
         v.append("0")
     v[-1] = str(int(v[-1]) + 1)
     return split_by.join(v) + suffix
 
 
 def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
                                   beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
                                   beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
-                                  build_tools_repo_path, checksums_enabled, en_US_config,
+                                  build_tools_repo_path, checksums_enabled, dashboard_check, en_US_config,
                                   extra_balrog_submitter_params, final_verify_channels,
                                   final_verify_platforms, uptake_monitoring_platforms,
                                   funsize_balrog_api_root, l10n_config,
                                   l10n_changesets, mozharness_changeset, next_version,
                                   partial_updates, partner_repacks_platforms,
                                   postrelease_bouncer_aliases_enabled, uptake_monitoring_enabled,
                                   postrelease_version_bump_enabled,
                                   product, public_key, push_to_candidates_enabled,
@@ -256,16 +256,17 @@ def make_task_graph_strict_kwargs(appVer
         beetmover_aws_access_key_id=beetmover_aws_access_key_id,
         beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
         beetmover_candidates_bucket=beetmover_candidates_bucket,
         bouncer_enabled=bouncer_enabled,
         branch=branch,
         buildNumber=buildNumber,
         build_tools_repo_path=build_tools_repo_path,
         checksums_enabled=checksums_enabled,
+        dashboard_check=dashboard_check,
         en_US_config=en_US_config,
         final_verify_channels=final_verify_channels,
         final_verify_platforms=final_verify_platforms,
         uptake_monitoring_platforms=uptake_monitoring_platforms,
         funsize_balrog_api_root=funsize_balrog_api_root,
         l10n_changesets=l10n_changesets,
         l10n_config=l10n_config,
         mozharness_changeset=mozharness_changeset,
--- a/lib/python/kickoff/sanity.py
+++ b/lib/python/kickoff/sanity.py
@@ -4,17 +4,16 @@ It's functionality is to replace the old
 http://hg.mozilla.org/build/tools/file/old-release-runner/buildbot-helpers/release_sanity.py
 
 Additionally, more checks are performed to cope with the new release promotion
 world regulations and constraints.
 """
 import sys
 import site
 import logging
-import pprint
 from os import path
 
 import requests
 
 site.addsitedir(path.join(path.dirname(__file__), ".."))
 from util.retry import retry
 from util.hg import make_hg_url
 from release.versions import getL10nDashboardVersion
@@ -105,17 +104,16 @@ class SanityException(Exception):
     custom exception is to be thrown in release runner.
     """
     pass
 
 
 class OpsMixin(object):
     """Helper class Mixin to enrich ReleaseSanitizerTestSuite behavior
     """
-
     def assertEqual(self, result, first, second, err_msg):
         """Method inspired from unittest implementation
         The :result is the aggregation object to collect all potential errors
         """
         if not first == second:
             result.add_error(err_msg)
 
 
@@ -128,17 +126,16 @@ class ReleaseSanitizerTestSuite(OpsMixin
 
     Once instance needs to hold the task graph arguments that come from
     Ship-it via release runner. Using the arguments, certain behaviors are
     tested (e.g. partials, l10n, versions, config, etc)
 
     To add more testing methods, please prefix the method with 'test_' in
     order to have it run by sanitize() main method.
     """
-
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.repo_path = self.kwargs["repo_path"]
         self.revision = self.kwargs["revision"]
         self.version = self.kwargs["version"]
         self.branch = self.kwargs["branch"]
         self.locales = self.kwargs["l10n_changesets"]
         self.product = self.kwargs["product"]
@@ -319,24 +316,25 @@ class ReleaseSanitizerTestSuite(OpsMixin
             )
 
             try:
                 make_generic_head_request(locale_url)
             except requests.HTTPError:
                 err_msg = "{locale} not found".format(locale=locale_url)
                 result.add_error(err_msg, sys.exc_info())
 
-    def _test_l10n_dashboard(self, result):
+    def test_l10n_dashboard(self, result):
         """test_l10n method
         Tests if l10n dashboard changesets match the current l10n changesets
         """
-        # TODO: this should be turned on for all betas and turned off for all
-        # other releases. Skip this test for now till we find a better
-        # approach to nicely tweak the test on/off depending on the branch
         log.info("Testing l10n dashboard changesets ...")
+        if not self.kwargs["dashboard_check"]:
+            log.info("Skipping l10n dashboard check")
+            return
+
         try:
             dash_changesets = get_l10_dashboard_changeset(self.version,
                                                           self.product)
         except requests.HTTPError:
             err_msg = ("Failed to retrieve l10n dashboard changes for"
                        "{product} product, version {version}").format(
                            product=self.product,
                            version=self.version)
@@ -354,47 +352,59 @@ class ReleaseSanitizerResult(object):
     This is usefule to avoid incremenatal fixes in release sanity
     """
     def __init__(self):
         self.errors = []
 
     def add_error(self, err_msg, err=None):
         """Method to collect a new errors. It collects the exception
         stacktrace and stores the exception value along with the message"""
+        # each error consist of a tuple containing the error message and any
+        # other potential information we might get useful from the
+        # sys.exc_info(). If there is no such, explanatory string will be added
         self.errors.append((err_msg, self._exc_info_to_string(err)))
+        # append an empty line after each exceptions to have a nicer output
         log.info("Collecting a new exception: %s", err_msg)
 
     def _exc_info_to_string(self, err):
         if err is None:
-            return 'No more details to show'
+            return "Result of assertion, no exception stacktrace available"
         # trim the traceback part from the exc_info result tuple
         _, value = err[:2]
         return value
 
+    def __str__(self):
+        """Define the output to be user-friendly readable"""
+        # make some room to separate the output from the exception stacktrace
+        ret = "\n\n"
+        for msg, err in self.errors:
+            ret += "* {msg}:\n{err}\n\n".format(msg=msg, err=err)
+        return ret
+
 
 class ReleaseSanitizerRunner(object):
     """Runner class that is to be called from release runner. It wraps up
     the logic to interfere with both the ReleaseSanitizerTestSuite and the
     ReleaseSanitizerResult. Upon successful run, errors in results should be
     an empty list. Otherwise, the errors list can be retrieved and processed.
     """
     resultClass = ReleaseSanitizerResult
     testSuite = ReleaseSanitizerTestSuite
 
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.result = self.resultClass()
 
     def run(self):
-        """Main method to call for the actual tests to perform release sanity"""
+        """Main method to call for the actual test of release sanity"""
         test_suite = self.testSuite(**self.kwargs)
         log.info("Attempting to sanitize ...")
         test_suite.sanitize(self.result)
 
     def was_successful(self):
         """Tells whether or not the result was a success"""
         return len(self.result.errors) == 0
 
     def get_errors(self):
         """Retrieves the list of errors from the result objecti
         in a nicely-formatted string
         """
-        return pprint.pformat(self.result.errors)
+        return self.result