Bug 1438839 - Finish the flake8 fixes on tools/ r=ahal
authorSylvestre Ledru <sledru@mozilla.com>
Thu, 15 Mar 2018 11:31:12 +0100
changeset 408629 b86e592046e4c4b89883fe755945cc36cb613c77
parent 408628 7985866da5d5cc055a6309eabcb954e372544baf
child 408630 55b8fe1015cb42340e3270bd9ad1f06efa5f7006
push id33649
push userbtara@mozilla.com
push dateSat, 17 Mar 2018 10:29:43 +0000
treeherdermozilla-central@97160a734959 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1438839
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1438839 - Finish the flake8 fixes on tools/ r=ahal MozReview-Commit-ID: F9c4laxe8Pt
tools/jprof/split-profile.py
tools/power/mach_commands.py
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test_make_incremental_updates.py
--- a/tools/jprof/split-profile.py
+++ b/tools/jprof/split-profile.py
@@ -28,17 +28,19 @@
 # split profiles into segments.  It takes as input a single text file,
 # and from that text file creates a series of jprof profiles in the
 # directory the text file is in.
 #
 # The input file format looks like the following:
 #
 #   poll g_main_poll
 #   GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *)
-#   RuleProcessorData RuleProcessorData::RuleProcessorData(nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#   RuleProcessorData RuleProcessorData::RuleProcessorData
+#        (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#
 #
 # From this input file, the script will construct a profile called
 # jprof-0.html that contains the whole profile, a profile called
 # jprof-1-poll.html that includes only stacks with g_main_poll, a
 # profile called jprof-2-GetRuleCascade.html that includes only stacks
 # that have GetRuleCascade and do not have g_main_poll, a profile called
 # jprof-3-RuleProcessorData.html that includes only stacks that have the
 # RuleProcessorData constructor and do not have GetRuleCascade or
--- a/tools/power/mach_commands.py
+++ b/tools/power/mach_commands.py
@@ -129,12 +129,12 @@ class MachCommands(MachCommandBase):
             # - 'Terminal' is for the terminal. If no browser is running from
             #   within the terminal, it will show up unnecessarily. This is a
             #   minor disadvantage of this very simple parsing strategy.
             #
             # - 'WindowServer' is for the WindowServer.
             #
             # - 'kernel' is for the kernel.
             #
-            if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line):
+            if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line):  # NOQA: E501
                 print(line)
 
         return 0
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -198,25 +198,26 @@ def copy_file(src_file_abs_path, dst_fil
     dst_file_dir = os.path.dirname(dst_file_abs_path)
     if not os.path.exists(dst_file_dir):
         os.makedirs(dst_file_dir)
     # Copy the file over
     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 
 
 def xz_file(filename):
-    """ XZ compresses the file in place.  The original file is replaced with the xz compressed version of itself
-        assumes the path is absolute"""
+    """ XZ compresses the file in place.  The original file is replaced
+        with the xz compressed version of itself assumes the path is absolute"""
     exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename + '"')
     os.rename(filename + ".xz", filename)
 
 
 def xzunzip_file(filename):
-    """ xz decompresses the file in palce.  The original file is replaced with a xz decompressed version of itself.
-        doesn't matter if the filename ends in .xz or not"""
+    """ xz decompresses the file in palce.  The original file is replaced
+        with a xz decompressed version of itself. doesn't matter if the
+        filename ends in .xz or not"""
     if not filename.endswith(".xz"):
         os.rename(filename, filename + ".xz")
         filename = filename + ".xz"
     exec_shell_cmd('xz -d "' + filename + '"')
 
 
 def extract_mar(filename, work_dir):
     """ Extracts the marfile intot he work_dir
@@ -285,24 +286,26 @@ def create_partial_patch_for_file(from_m
             file_in_manifest_name = to_marfile_entry.name
             patch_info.append_add_instruction(file_in_manifest_name)
         # Copy the pre-calculated file into our new patch work aread
         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
         patch_info.archive_files.append('"' + file_in_manifest_name + '"')
 
 
 def create_add_patch_for_file(to_marfile_entry, patch_info):
-    """  Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
+    """  Copy the file to the working dir, add the add instruction,
+         and add it to the list of archive files """
     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
     patch_info.append_add_instruction(to_marfile_entry.name)
     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 
 
 def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
-    """  Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
+    """  Copy the file to the working dir, add the add-if-not instruction,
+         and add it to the list of archive files """
     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
     patch_info.append_add_if_not_instruction(to_marfile_entry.name)
     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 
 
 def process_explicit_remove_files(dir_path, patch_info):
     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
     this will throw.  If found adds the removed-files
@@ -330,17 +333,18 @@ def process_explicit_remove_files(dir_pa
             # Exclude any blank and comment lines.
             if line and not line.startswith("#"):
                 # Python on windows uses \ for path separators and the update
                 # manifests expects / for path separators on all platforms.
                 line = line.replace("\\", "/")
                 patch_info.append_remove_instruction(line)
 
 
-def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
+def create_partial_patch(from_dir_path, to_dir_path, patch_filename,
+                         shas, patch_info, forced_updates, add_if_not_list):
     """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
     # Cannocolize the paths for safey
     from_dir_path = os.path.abspath(from_dir_path)
     to_dir_path = os.path.abspath(to_dir_path)
     # Create a hashtable of the from  and to directories
     from_dir_hash, from_file_set, from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
     to_dir_hash, to_file_set, to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
     # Create a list of the forced updates
@@ -467,28 +471,28 @@ def get_buildid(work_dir):
 
 def decode_filename(filepath):
     """ Breaks filename/dir structure into component parts based on regex
         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
         Or linux-i686/en-US/firefox-3.0b3.complete.mar
         Returns dict with keys product, version, locale, platform, type
     """
     try:
-      m = re.search(
-        '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
-      os.path.basename(filepath))
-      return m.groupdict()
+        m = re.search(
+            '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',  # NOQA: E501
+            os.path.basename(filepath))
+        return m.groupdict()
     except Exception as exc:
-      try:
-        m = re.search(
-          '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
-        filepath)
-        return m.groupdict()
-      except:
-        raise Exception("could not parse filepath %s: %s" % (filepath, exc))
+        try:
+            m = re.search(
+                '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',  # NOQA: E501
+                filepath)
+            return m.groupdict()
+        except Exception:
+            raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 
 
 def create_partial_patches(patches):
     """ Given the patches generates a set of partial patches"""
     shas = {}
 
     work_dir_root = None
     metadata = []
@@ -524,18 +528,24 @@ def create_partial_patches(patches):
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to)
             to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
-            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, [
-                                                    'update.manifest', 'updatev2.manifest', 'updatev3.manifest'], []), forced_updates, ['channel-prefs.js', 'update-settings.ini'])
+            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename,
+                                                    shas, PatchInfo(work_dir, [
+                                                        'update.manifest',
+                                                        'updatev2.manifest',
+                                                        'updatev3.manifest'
+                                                    ], []),
+                                                    forced_updates,
+                                                    ['channel-prefs.js', 'update-settings.ini'])
             partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
                 'to_filename': os.path.basename(to_filename),
                 'from_filename': os.path.basename(from_filename),
                 'partial_filename': os.path.basename(partial_filename),
                 'to_buildid': to_buildid,
@@ -546,18 +556,18 @@ def create_partial_patches(patches):
                 'to_size': to_size,
                 'from_size': from_size,
                 'partial_size': partial_size,
                 'to_version': to_decoded['version'],
                 'from_version': from_decoded['version'],
                 'locale': from_decoded['locale'],
                 'platform': from_decoded['platform'],
             })
-            print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),
-                                                                                        str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))
+            print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),  # NOQA: E501
+                                                                                        str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))  # NOQA: E501
             patch_num += 1
         return metadata
     finally:
         # If we fail or get a ctrl-c during run be sure to clean up temp dir
         if (work_dir_root and os.path.exists(work_dir_root)):
             shutil.rmtree(work_dir_root)
 
 
--- a/tools/update-packaging/test_make_incremental_updates.py
+++ b/tools/update-packaging/test_make_incremental_updates.py
@@ -28,37 +28,41 @@ class TestPatchInfo(unittest.TestCase):
     def test_append_add_instruction(self):
         self.patch_info.append_add_instruction('file.test')
         self.assertEquals(['add "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['add "file.test"'], self.patch_info.manifestv3)
 
     def test_append_add_if_instruction(self):
         self.patch_info.append_add_instruction('distribution/extensions/extension/file.test')
         self.assertEquals(
-            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv2)
         self.assertEquals(
-            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv3)
 
     def test_append_add_if_not_instruction(self):
         self.patch_info.append_add_if_not_instruction('file.test')
         self.assertEquals([], self.patch_info.manifestv2)
         self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3)
 
     def test_append_patch_instruction(self):
         self.patch_info.append_patch_instruction('file.test', 'patchname')
         self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3)
 
     def test_append_patch_if_instruction(self):
         self.patch_info.append_patch_instruction(
             'distribution/extensions/extension/file.test', 'patchname')
         self.assertEquals(
-            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv2)
         self.assertEquals(
-            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv3)
 
     def test_append_remove_instruction(self):
         self.patch_info.append_remove_instruction('file.test')
         self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3)
 
     def test_append_rmdir_instruction(self):
         self.patch_info.append_remove_instruction('dirtest/')
@@ -105,17 +109,19 @@ class TestMarFileEntry(unittest.TestCase
         self.assertEquals(goodSha, sha)
 """
 
 
 class TestMakeIncrementalUpdates(unittest.TestCase):
     def setUp(self):
         work_dir = '.'
         self.patch_info = PatchInfo(
-            work_dir, ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'], ['/readme.txt'])
+            work_dir,
+            ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'],
+            ['/readme.txt'])
         root_path = '/'
         filename = 'test.file'
         self.mar_file_entry = MarFileEntry(root_path, filename)
 
     """ FIXME makes direct shell calls, need refactoring
     def test_exec_shell_cmd(self):
         mkup.exec_shell_cmd('echo test')
 
@@ -127,26 +133,28 @@ class TestMakeIncrementalUpdates(unittes
 
     def test_bunzip_file(self):
         mkup.bunzip_file('filename')
 
     def test_extract_mar(self):
         mkup.extract_mar('filename', 'work_dir')
 
     def test_create_partial_patch_for_file(self):
-        mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info)
+        mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas',
+                                           self.patch_info)
 
     def test_create_add_patch_for_file(self):
         mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info)
 
     def test_process_explicit_remove_files(self):
         mkup.process_explicit_remove_files('dir_path', self.patch_info)
 
     def test_create_partial_patch(self):
-        mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates')
+        mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas',
+                                  self.patch_info, 'forced_updates')
 
     def test_create_partial_patches(patches):
         mkup.create_partial_patches('patches')
 
     """
 
     """ FIXME touches the filesystem, need refactoring
     def test_get_buildid(self):