Bug 1492128: [mozrelease] Cleanup buglist_creator code; r=mtabara a=release
authorTom Prince <mozilla@hocat.ca>
Tue, 30 Oct 2018 18:21:38 +0000
changeset 501022 ca613265746bfed11bb80a512283b1b1d784ef23
parent 501021 a63fa7222a0f69164cf25980f073141f23151a18
child 501023 824771ceb6f16cedde18560a2813df04a5ac97bc
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara, release
bugs1492128
milestone64.0
Bug 1492128: [mozrelease] Cleanup buglist_creator code; r=mtabara a=release Differential Revision: https://phabricator.services.mozilla.com/D10153
python/mozrelease/mozrelease/buglist_creator.py
python/mozrelease/test/test_buglist_creator.py
--- a/python/mozrelease/mozrelease/buglist_creator.py
+++ b/python/mozrelease/mozrelease/buglist_creator.py
@@ -5,65 +5,62 @@
 
 from __future__ import absolute_import, print_function
 
 import logging
 import os
 import re
 import requests
 from operator import itemgetter
-from pkg_resources import parse_version
+
+from mozilla_version.gecko import GeckoVersion
 
 BUGLIST_PREFIX = 'Bugs since previous changeset: '
-BACKOUT_REGEX = r'back(\s?)out|backed out|backing out'
+BACKOUT_REGEX = re.compile(r'back(\s?)out|backed out|backing out', re.IGNORECASE)
 BACKOUT_PREFIX = 'Backouts since previous changeset: '
 BUGZILLA_BUGLIST_TEMPLATE = 'https://bugzilla.mozilla.org/buglist.cgi?bug_id={bugs}'
-BUG_NUMBER_REGEX = r'bug \d+'
+BUG_NUMBER_REGEX = re.compile(r'bug \d+', re.IGNORECASE)
 CHANGELOG_TO_FROM_STRING = '{product}_{version}_RELEASE'
 CHANGESET_URL_TEMPLATE = (
     'https://hg.mozilla.org/{release_branch}/{logtype}'
     '?fromchange={from_version}&tochange={to_version}&full=1'
 )
 FULL_CHANGESET_PREFIX = 'Full Mercurial changelog: '
 LIST_DESCRIPTION_TEMPLATE = 'Comparing Mercurial tag {from_version} to {to_version}:'
 MAX_BUGS_IN_BUGLIST = 250
 MERCURIAL_TAGS_URL_TEMPLATE = 'https://hg.mozilla.org/{release_branch}/json-tags'
 NO_BUGS = ''  # Return this when bug list can't be created
 URL_SHORTENER_TEMPLATE = 'https://bugzilla.mozilla.org/rest/bitly/shorten?url={url}'
 
 log = logging.getLogger(__name__)
 
 
-def create_bugs_url(release):
+def create_bugs_url(product, current_version, current_revision):
     """
     Creates list of bugs and backout bugs for release-drivers email
 
     :param release: dict -> containing information about release, from Ship-It
     :return: str -> description of compared releases, with Bugzilla links
         containing all bugs in changeset
     """
     try:
         # Extract the important data, ignore if beta1 release
-        current_version_dot = release['version']
-        if re.search(r'b1$', current_version_dot):
+        if current_version.beta_number == 1:
             # If the version is beta 1, don't make any links
             return NO_BUGS
 
-        product = release['product']
-        branch = release['branch']
-        current_revision = release['mozillaRevision']
-
+        branch = get_branch_by_version(current_version)
         # Get the tag version, for display purposes
-        current_version_tag = dot_version_to_tag_version(product, current_version_dot)
+        current_version_tag = tag_version(product, current_version)
 
         # Get all Hg tags for this branch, determine the previous version
         tag_url = MERCURIAL_TAGS_URL_TEMPLATE.format(release_branch=branch)
         mercurial_tags_json = requests.get(tag_url).json()
         previous_version_tag = get_previous_tag_version(
-            product, current_version_dot, current_version_tag, mercurial_tags_json)
+            product, current_version, current_version_tag, mercurial_tags_json)
 
         # Get the changeset between these versions, parse for all unique bugs and backout bugs
         resp = requests.get(CHANGESET_URL_TEMPLATE.format(release_branch=branch,
                                                           from_version=previous_version_tag,
                                                           to_version=current_revision,
                                                           logtype='json-pushes'))
         changeset_data = resp.json()
         unique_bugs, unique_backout_bugs = get_bugs_in_changeset(changeset_data)
@@ -91,40 +88,40 @@ def create_bugs_url(release):
 
 def get_bugs_in_changeset(changeset_data):
     unique_bugs, unique_backout_bugs = set(), set()
     for data in changeset_data.values():
         for changeset in data['changesets']:
             if is_excluded_change(changeset):
                 continue
 
-            changeset_desc_lower = changeset['desc'].lower()
-            bug_re = re.search(BUG_NUMBER_REGEX, changeset_desc_lower)
+            changeset_desc = changeset['desc']
+            bug_re = BUG_NUMBER_REGEX.search(changeset_desc)
 
             if bug_re:
                 bug_number = bug_re.group().split(' ')[1]
 
-                if is_backout_bug(changeset_desc_lower):
+                if is_backout_bug(changeset_desc):
                     unique_backout_bugs.add(bug_number)
                 else:
                     unique_bugs.add(bug_number)
 
     return unique_bugs, unique_backout_bugs
 
 
 def is_excluded_change(changeset):
     excluded_change_keywords = [
         'a=test-only',
         'a=release',
     ]
     return any(keyword in changeset['desc'] for keyword in excluded_change_keywords)
 
 
-def is_backout_bug(changeset_description_lowercase):
-    return re.search(BACKOUT_REGEX, changeset_description_lowercase)
+def is_backout_bug(changeset_description):
+    return bool(BACKOUT_REGEX.search(changeset_description))
 
 
 def create_short_url_with_prefix(buglist, backout_buglist):
     # Create link if there are bugs, else empty string
     urls = []
     for set_of_bugs, prefix in [(buglist, BUGLIST_PREFIX), (backout_buglist, BACKOUT_PREFIX)]:
         if set_of_bugs and len(set_of_bugs) < MAX_BUGS_IN_BUGLIST:
             try:
@@ -140,64 +137,76 @@ def create_short_url_with_prefix(buglist
         else:
             url = ''
 
         urls.append(url)
 
     return urls[0], urls[1]
 
 
-def dot_version_to_tag_version(product, dot_version):
-    underscore_version = dot_version.replace('.', '_')
+def tag_version(product, version):
+    underscore_version = str(version).replace('.', '_')
     return CHANGELOG_TO_FROM_STRING.format(product=product.upper(), version=underscore_version)
 
 
-def tag_version_to_dot_version_parse(tag):
+def parse_tag_version(tag):
     dot_version = '.'.join(tag.split('_')[1:-1])
-    return parse_version(dot_version)
+    return GeckoVersion.parse(dot_version)
 
 
 def get_previous_tag_version(
-    product, current_version_dot, current_version_tag, mercurial_tags_json,
+    product, current_version, current_version_tag, mercurial_tags_json,
 ):
     """
     Gets the previous hg version tag for the product and branch, given the current version tag
     """
 
     def _invalid_tag_filter(tag):
         """Filters by product and removes incorrect major version + base, end releases"""
-        major_version = current_version_dot.split('.')[0]
-        prod_major_version_re = r'^{product}_{major_version}'.format(product=product.upper(),
-                                                                     major_version=major_version)
+        prod_major_version_re = r'^{product}_{major_version}'.format(
+            product=product.upper(), major_version=current_version.major_number)
 
         return 'BASE' not in tag and \
                'END' not in tag and \
                'RELEASE' in tag and \
                re.match(prod_major_version_re, tag)
 
     # Get rid of irrelevant tags, sort by date and extract the tag string
-    tags = set(map(itemgetter('tag'), mercurial_tags_json['tags']))
-    tags = filter(_invalid_tag_filter, tags)
-    dot_tag_version_mapping = zip(map(tag_version_to_dot_version_parse, tags), tags)
-    dot_tag_version_mapping.append(  # Add the current version to the list
-        (parse_version(current_version_dot), current_version_tag)
-    )
-    dot_tag_version_mapping = sorted(dot_tag_version_mapping, key=itemgetter(0))
+    tags = {
+        (parse_tag_version(item['tag']), item['tag'])
+        for item in mercurial_tags_json['tags']
+        if _invalid_tag_filter(item['tag'])
+    }
+    # Add the current version to the list
+    tags.add((current_version, current_version_tag))
+    tags = sorted(tags, key=lambda tag: tag[0])
 
     # Find where the current version is and go back one to get the previous version
-    next_version_index = (
-        map(itemgetter(0), dot_tag_version_mapping).index(parse_version(current_version_dot)) - 1
-    )
+    next_version_index = (map(itemgetter(0), tags).index(current_version) - 1)
 
-    return dot_tag_version_mapping[next_version_index][1]
+    return tags[next_version_index][1]
 
 
 def format_return_value(description, unique_bugs, unique_backout_bugs, changeset_html):
     reg_bugs_link, backout_bugs_link = create_short_url_with_prefix(
         unique_bugs, unique_backout_bugs)
     changeset_full = FULL_CHANGESET_PREFIX + changeset_html
     return_str = '{description}\n{regular_bz_url}{backout_bz_url}{changeset_full}'\
         .format(description=description, regular_bz_url=reg_bugs_link,
                 backout_bz_url=backout_bugs_link, changeset_full=changeset_full)
 
     return return_str
 
 
+def get_branch_by_version(version):
+    """
+    Get the branch a given version is found on.
+    """
+    if version.is_beta:
+        return 'releases/mozilla-beta'
+    elif version.is_release:
+        return 'releases/mozilla-release'
+    elif version.is_esr:
+        return 'releases/mozilla-esr{}'.format(version.major_number)
+    else:
+        raise Exception(
+            'Unsupported version type {}: {}'.format(
+                version.version_type.name, version))
--- a/python/mozrelease/test/test_buglist_creator.py
+++ b/python/mozrelease/test/test_buglist_creator.py
@@ -4,83 +4,81 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function
 
 import json
 from pathlib2 import Path
 
 import mozunit
-from pkg_resources import parse_version
+import pytest
+
+from mozilla_version.gecko import GeckoVersion
 from mozrelease.buglist_creator import (
     is_excluded_change, create_bugs_url, is_backout_bug, get_previous_tag_version,
-    get_bugs_in_changeset, dot_version_to_tag_version, tag_version_to_dot_version_parse,
+    get_bugs_in_changeset, tag_version, parse_tag_version,
 )
 
 
 DATA_PATH = Path(__file__).with_name("data")
 
 
 def test_beta_1_release():
-    release_object_54_0b1 = {
-        'branch': 'releases/mozilla-beta',
-        'product': 'firefox',
-        'version': '54.0b1',
-        'mozillaRevision': 'cf76e00dcd6f',
-    }
-    buglist_str_54_0b1 = create_bugs_url(release_object_54_0b1)
+    buglist_str_54_0b1 = create_bugs_url(
+        product='firefox',
+        current_version=GeckoVersion.parse('54.0b1'),
+        current_revision='cf76e00dcd6f',
+    )
     assert buglist_str_54_0b1 == '', 'There should be no bugs to compare for beta 1.'
 
 
-def test_is_excluded_change():
-    excluded_changesets = [
-        {'desc': 'something something something a=test-only something something something'},
-        {'desc': 'this is a a=release change!'},
-    ]
-    assert all(is_excluded_change(excluded) for excluded in excluded_changesets), \
-        'is_excluded_change failed to exclude a changeset.'
+@pytest.mark.parametrize('description,is_excluded', (
+    ('something something something a=test-only something something something', True),
+    ('this is a a=release change!', True),
+))
+def test_is_excluded_change(description, is_excluded):
+    assert is_excluded_change({'desc': description}) == is_excluded
+
+
+@pytest.mark.parametrize('description,is_backout', (
+    ('I backed out this bug because', True),
+    ('Backing out this bug due to', True),
+    ('Backout bug xyz', True),
+    ('Back out bug xyz', True),
+    ('this is a regular bug description', False),
+))
+def test_is_backout_bug(description, is_backout):
+    assert is_backout_bug(description) == is_backout
 
 
-def test_is_backout_bug():
-    backout_bugs_descs = [
-        'I backed out this bug because',
-        'Backing out this bug due to',
-        'Backout bug xyz',
-        'Back out bug xyz',
-    ]
-
-    not_backout_bugs = [
-        'this is a regular bug description',
-    ]
-
-    assert all(is_backout_bug(backout_desc.lower()) for backout_desc in backout_bugs_descs)
-    assert all(not is_backout_bug(regular_desc.lower()) for regular_desc in not_backout_bugs)
+@pytest.mark.parametrize('product,version,tag', (
+    ('firefox', GeckoVersion.parse('53.0b10'), 'FIREFOX_53_0b10_RELEASE'),
+    ('firefox', GeckoVersion.parse('52.0'), 'FIREFOX_52_0_RELEASE'),
+    ('fennec', GeckoVersion.parse('52.0.2'), 'FENNEC_52_0_2_RELEASE'),
+))
+def test_tag_version(product, version, tag):
+    assert tag_version(product, version) == tag
 
 
-def test_dot_version_to_tag_version():
-    test_tuples = [
-        (['firefox', '53.0b10'], 'FIREFOX_53_0b10_RELEASE'),
-        (['firefox', '52.0'], 'FIREFOX_52_0_RELEASE'),
-        (['fennec', '52.0.2'], 'FENNEC_52_0_2_RELEASE'),
-    ]
-
-    assert all(dot_version_to_tag_version(*args) == results for args, results in test_tuples)
+@pytest.mark.parametrize('tag,version', (
+    ('FIREFOX_53_0b10_RELEASE', GeckoVersion.parse('53.0b10')),
+    ('FIREFOX_52_0_RELEASE', GeckoVersion.parse('52.0')),
+    ('FENNEC_52_0_2_RELEASE', GeckoVersion.parse('52.0.2')),
+))
+def test_parse_tag_version(tag, version):
+    assert parse_tag_version(tag) == version
 
 
-def test_tag_version_to_dot_version_parse():
-    test_tuples = [
-        ('FIREFOX_53_0b10_RELEASE', parse_version('53.0b10')),
-        ('FIREFOX_52_0_RELEASE', parse_version('52.0')),
-        ('FENNEC_52_0_2_RELEASE', parse_version('52.0.2')),
-    ]
-
-    assert all(tag_version_to_dot_version_parse(tag) == expected for tag, expected in test_tuples)
-
-
-def test_get_previous_tag_version():
+@pytest.mark.parametrize('version,tag,previous_tag', (
+    (GeckoVersion.parse('48.0b4'), 'FIREFOX_48_0b4_RELEASE', 'FIREFOX_48_0b3_RELEASE'),
+    (GeckoVersion.parse('48.0b9'), 'FIREFOX_48_0b9_RELEASE', 'FIREFOX_48_0b7_RELEASE'),
+    (GeckoVersion.parse('48.0.2'), 'FIREFOX_48_0_2_RELEASE', 'FIREFOX_48_0_1_RELEASE'),
+    (GeckoVersion.parse('48.0.1'), 'FIREFOX_48_0_1_RELEASE', 'FIREFOX_48_0_RELEASE'),
+))
+def test_get_previous_tag_version(version, tag, previous_tag):
     product = 'firefox'
     ff_48_tags = [
         u'FIREFOX_BETA_48_END',
         u'FIREFOX_RELEASE_48_END',
         u'FIREFOX_48_0_2_RELEASE',
         u'FIREFOX_48_0_2_BUILD1',
         u'FIREFOX_48_0_1_RELEASE',
         u'FIREFOX_48_0_1_BUILD3',
@@ -111,26 +109,17 @@ def test_get_previous_tag_version():
     ]
 
     mock_hg_json = {
         'tags': [
             {'tag': tag} for tag in ff_48_tags
         ],
     }
 
-    test_tuples = [
-        ('48.0b4', 'FIREFOX_48_0b4_RELEASE', 'FIREFOX_48_0b3_RELEASE'),
-        ('48.0b9', 'FIREFOX_48_0b9_RELEASE', 'FIREFOX_48_0b7_RELEASE'),
-        ('48.0.2', 'FIREFOX_48_0_2_RELEASE', 'FIREFOX_48_0_1_RELEASE'),
-        ('48.0.1', 'FIREFOX_48_0_1_RELEASE', 'FIREFOX_48_0_RELEASE'),
-    ]
-
-    assert all(
-        get_previous_tag_version(product, dot_version, tag_version, mock_hg_json) == expected
-        for dot_version, tag_version, expected in test_tuples)
+    assert get_previous_tag_version(product, version, tag, mock_hg_json) == previous_tag
 
 
 def test_get_bugs_in_changeset():
     with DATA_PATH.joinpath("buglist_changesets.json").open("r") as fp:
         changeset_data = json.load(fp)
     bugs, backouts = get_bugs_in_changeset(changeset_data)
 
     assert bugs == {u'1356563', u'1348409', u'1341190', u'1360626', u'1332731', u'1328762',