Bug 1509962: [update-verify] Run lint against python code in update-verify; r=mtabara
authorTom Prince <mozilla@hocat.ca>
Mon, 03 Dec 2018 06:44:31 +0000
changeset 505630 7eb869c7fd0a51a5809065ecaca31dfcc3b45e78
parent 505629 4c4352291d629aaaaf39718c3cea57870db048d1
child 505631 3e98ee808894e9d4a27926e310d08a687a1b7909
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1509962
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509962: [update-verify] Run lint against python code in update-verify; r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D12978
tools/update-verify/python/util/commands.py
tools/update-verify/release/compare-directories.py
tools/update-verify/release/replace-updater-certs.py
tools/update-verify/scripts/chunked-verify.py
--- a/tools/update-verify/python/util/commands.py
+++ b/tools/update-verify/python/util/commands.py
@@ -1,9 +1,12 @@
 """Functions for running commands"""
+
+from __future__ import absolute_import, print_function
+
 import subprocess
 import os
 import time
 import logging
 log = logging.getLogger(__name__)
 
 
 # timeout message, used in TRANSIENT_HG_ERRORS and in tests.
--- a/tools/update-verify/release/compare-directories.py
+++ b/tools/update-verify/release/compare-directories.py
@@ -1,10 +1,12 @@
 #! /usr/bin/env python
 
+from __future__ import absolute_import, print_function
+
 import argparse
 import difflib
 import hashlib
 import logging
 import os
 import sys
 
 
@@ -14,78 +16,84 @@ Required:
 The files list describes the files which a transform may be used on.
 The 'side' is one of ('source', 'target') and defines where each transform is applied
 The 'channel_prefix' list controls which channels a transform may be used for, where a value of
 'beta' means all of beta, beta-localtest, beta-cdntest, etc.
 
 One or more:
 A 'deletion' specifies a start of line to match on, removing the whole line
 A 'substitution' is a list of full string to match and its replacement
-
-Future note - this may need to move into the tree to make staging releases or release-on-try easier
 """
 TRANSFORMS = [
     # channel-prefs.js
     {
         # preprocessor comments, eg //@line 6 "/builds/worker/workspace/...
         # this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342)
-        'files': ['defaults/pref/channel-prefs.js', 'Contents/Resources/defaults/pref/channel-prefs.js'],
+        'files': [
+            'defaults/pref/channel-prefs.js',
+            'Contents/Resources/defaults/pref/channel-prefs.js',
+        ],
         'channel_prefix': ['aurora', 'beta', 'release', 'esr'],
         'side': 'source',
         'deletion': '//@line 6 "',
     },
     {
         # updates from a beta to an RC build, the latter specifies the release channel
-        'files': ['defaults/pref/channel-prefs.js', 'Contents/Resources/defaults/pref/channel-prefs.js'],
+        'files': [
+            'defaults/pref/channel-prefs.js',
+            'Contents/Resources/defaults/pref/channel-prefs.js',
+        ],
         'channel_prefix': ['beta'],
         'side': 'target',
         'substitution': [
             'pref("app.update.channel", "release");\n',
-            'pref("app.update.channel", "beta");\n'
+            'pref("app.update.channel", "beta");\n',
         ],
     },
     {
         # updates from an RC to a beta build
-        'files': ['defaults/pref/channel-prefs.js', 'Contents/Resources/defaults/pref/channel-prefs.js'],
+        'files': [
+            'defaults/pref/channel-prefs.js',
+            'Contents/Resources/defaults/pref/channel-prefs.js',
+        ],
         'channel_prefix': ['beta'],
         'side': 'source',
         'substitution': [
             'pref("app.update.channel", "release");\n',
-            'pref("app.update.channel", "beta");\n'
+            'pref("app.update.channel", "beta");\n',
         ],
     },
     # update-settings.ini
     {
         # updates from a beta to an RC build, the latter specifies the release channel
-        # on mac, we actually have both files. The second location is the real one but we copy to the first
-        # to run the linux64 updater
+        # on mac, we actually have both files. The second location is the real
+        # one but we copy to the first to run the linux64 updater
         'files': ['update-settings.ini', 'Contents/Resources/update-settings.ini'],
         'channel_prefix': ['beta'],
         'side': 'target',
         'substitution': [
             'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n',
-            'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n'
+            'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n',
         ],
     },
     {
         # updates from an RC to a beta build
-        # on mac, we only need to modify the legit file this time. unpack_build handles the copy for the updater in
-        # both source and target
+        # on mac, we only need to modify the legit file this time. unpack_build
+        # handles the copy for the updater in both source and target
         'files': ['Contents/Resources/update-settings.ini'],
         'channel_prefix': ['beta'],
         'side': 'source',
         'substitution': [
             'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n',
-            'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n'
+            'ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n',
         ],
     },
 ]
 
 
-
 def walk_dir(path):
     all_files = []
     all_dirs = []
 
     for root, dirs, files in os.walk(path):
         all_dirs.extend([os.path.join(root, d) for d in dirs])
         all_files.extend([os.path.join(root, f) for f in files])
 
@@ -135,19 +143,23 @@ def compare_common_files(files, channel,
         if os.stat(source_file).st_size != os.stat(target_file).st_size or \
                 hash_file(source_file) != hash_file(target_file):
             logging.info('Difference found in {}'.format(filename))
             file_contents = {
                 'source': open(source_file).readlines(),
                 'target': open(target_file).readlines(),
             }
 
-            transforms = [t for t in TRANSFORMS if filename in t['files'] and
-                          channel.startswith(tuple(t['channel_prefix']))]
-            logging.debug('Got {} transform(s) to consider for {}'.format(len(transforms), filename))
+            transforms = [
+                t for t in TRANSFORMS
+                if filename in t['files']
+                and channel.startswith(tuple(t['channel_prefix']))
+            ]
+            logging.debug(
+                'Got {} transform(s) to consider for {}'.format(len(transforms), filename))
             for transform in transforms:
                 side = transform['side']
 
                 if 'deletion' in transform:
                     d = transform['deletion']
                     logging.debug('Trying deleting lines starting {} from {}'.format(d, side))
                     file_contents[side] = [l for l in file_contents[side] if not l.startswith(d)]
 
@@ -165,17 +177,18 @@ def compare_common_files(files, channel,
                 logging.error('{} still differs after transforms, residual diff:'.format(filename))
                 for l in difflib.unified_diff(file_contents['source'], file_contents['target']):
                     logging.error(l.rstrip())
 
     return difference_found
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser('Compare two directories recursively, with transformations for expected diffs')
+    parser = argparse.ArgumentParser(
+        'Compare two directories recursively, with transformations for expected diffs')
     parser.add_argument('source', help='Directory containing updated Firefox')
     parser.add_argument('target', help='Directory containing expected Firefox')
     parser.add_argument('channel', help='Update channel used')
     parser.add_argument('--verbose', '-v', action='store_true', help='Enable verbose logging')
 
     args = parser.parse_args()
     level = logging.INFO
     if args.verbose:
@@ -189,18 +202,20 @@ if __name__ == '__main__':
         sys.exit(3)
 
     logging.info('Comparing {} with {}...'.format(source, target))
     source_dirs, source_files = walk_dir(source)
     target_dirs, target_files = walk_dir(target)
 
     dir_list_diff = compare_listings(source_dirs, target_dirs, 'Directories', source, target)
     file_list_diff = compare_listings(source_files, target_files, 'Files', source, target)
-    file_diff = compare_common_files(set(source_files) & set(target_files), args.channel, source, target)
+    file_diff = compare_common_files(
+        set(source_files) & set(target_files), args.channel, source, target)
 
     if file_diff:
         # Use status of 2 since python will use 1 if there is an error running the script
         sys.exit(2)
     elif dir_list_diff or file_list_diff:
-        # this has traditionally been a WARN, but we don't have files on one side anymore so lets FAIL
+        # this has traditionally been a WARN, but we don't have files on one
+        # side anymore so lets FAIL
         sys.exit(2)
     else:
         logging.info('No differences found')
--- a/tools/update-verify/release/replace-updater-certs.py
+++ b/tools/update-verify/release/replace-updater-certs.py
@@ -1,8 +1,10 @@
+from __future__ import absolute_import, print_function
+
 import os.path
 import sys
 
 cert_dir = sys.argv[1]
 # Read twice, because strings cannot be copied
 updater_data = open(sys.argv[2], "rb").read()
 new_updater = open(sys.argv[2], "rb").read()
 outfile = sys.argv[3]
@@ -19,15 +21,17 @@ for find_cert, replace_cert in zip(*[ite
     print("Looking for {}...".format(find_cert))
     if find in new_updater:
         print("Replacing {} with {}".format(find_cert, replace_cert))
         new_updater = new_updater.replace(find, replace)
     else:
         print("Didn't find {}...".format(find_cert))
 
 if len(updater_data) != len(new_updater):
-    print("WARNING: new updater is not the same length as the old one (old: {}, new: {})".format(len(updater_data), len(new_updater)))
+    print(
+        "WARNING: new updater is not the same length as the old one (old: {}, new: {})".format(
+            len(updater_data), len(new_updater)))
 
 if updater_data == new_updater:
     print("WARNING: updater is unchanged")
 
 with open(outfile, 'wb+') as f:
     f.write(new_updater)
--- a/tools/update-verify/scripts/chunked-verify.py
+++ b/tools/update-verify/scripts/chunked-verify.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 
+from __future__ import absolute_import, print_function
+
 import logging
 import os
 from os import path
 import sys
 from tempfile import mkstemp
 
 sys.path.append(path.join(path.dirname(__file__), "../python"))
 logging.basicConfig(