author | Sylvestre Ledru <sledru@mozilla.com> |
Thu, 15 Mar 2018 11:31:12 +0100 | |
changeset 408629 | b86e592046e4c4b89883fe755945cc36cb613c77 |
parent 408628 | 7985866da5d5cc055a6309eabcb954e372544baf |
child 408630 | 55b8fe1015cb42340e3270bd9ad1f06efa5f7006 |
push id | 33649 |
push user | btara@mozilla.com |
push date | Sat, 17 Mar 2018 10:29:43 +0000 |
treeherder | mozilla-central@97160a734959 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahal |
bugs | 1438839 |
milestone | 61.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/tools/jprof/split-profile.py +++ b/tools/jprof/split-profile.py @@ -28,17 +28,19 @@ # split profiles into segments. It takes as input a single text file, # and from that text file creates a series of jprof profiles in the # directory the text file is in. # # The input file format looks like the following: # # poll g_main_poll # GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *) -# RuleProcessorData RuleProcessorData::RuleProcessorData(nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *) +# RuleProcessorData RuleProcessorData::RuleProcessorData +# (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *) +# # # From this input file, the script will construct a profile called # jprof-0.html that contains the whole profile, a profile called # jprof-1-poll.html that includes only stacks with g_main_poll, a # profile called jprof-2-GetRuleCascade.html that includes only stacks # that have GetRuleCascade and do not have g_main_poll, a profile called # jprof-3-RuleProcessorData.html that includes only stacks that have the # RuleProcessorData constructor and do not have GetRuleCascade or
--- a/tools/power/mach_commands.py +++ b/tools/power/mach_commands.py @@ -129,12 +129,12 @@ class MachCommands(MachCommandBase): # - 'Terminal' is for the terminal. If no browser is running from # within the terminal, it will show up unnecessarily. This is a # minor disadvantage of this very simple parsing strategy. # # - 'WindowServer' is for the WindowServer. # # - 'kernel' is for the kernel. # - if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line): + if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line): # NOQA: E501 print(line) return 0
--- a/tools/update-packaging/make_incremental_updates.py +++ b/tools/update-packaging/make_incremental_updates.py @@ -198,25 +198,26 @@ def copy_file(src_file_abs_path, dst_fil dst_file_dir = os.path.dirname(dst_file_abs_path) if not os.path.exists(dst_file_dir): os.makedirs(dst_file_dir) # Copy the file over shutil.copy2(src_file_abs_path, dst_file_abs_path) def xz_file(filename): - """ XZ compresses the file in place. The original file is replaced with the xz compressed version of itself - assumes the path is absolute""" + """ XZ compresses the file in place. The original file is replaced + with the xz compressed version of itself assumes the path is absolute""" exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename + '"') os.rename(filename + ".xz", filename) def xzunzip_file(filename): - """ xz decompresses the file in palce. The original file is replaced with a xz decompressed version of itself. - doesn't matter if the filename ends in .xz or not""" + """ xz decompresses the file in palce. The original file is replaced + with a xz decompressed version of itself. doesn't matter if the + filename ends in .xz or not""" if not filename.endswith(".xz"): os.rename(filename, filename + ".xz") filename = filename + ".xz" exec_shell_cmd('xz -d "' + filename + '"') def extract_mar(filename, work_dir): """ Extracts the marfile intot he work_dir @@ -285,24 +286,26 @@ def create_partial_patch_for_file(from_m file_in_manifest_name = to_marfile_entry.name patch_info.append_add_instruction(file_in_manifest_name) # Copy the pre-calculated file into our new patch work aread copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name)) patch_info.archive_files.append('"' + file_in_manifest_name + '"') def create_add_patch_for_file(to_marfile_entry, patch_info): - """ Copy the file to the working dir, add the add instruction, and add it to the list of archive files """ + """ Copy the file to the working dir, add the add instruction, + and add it to the list of archive files """ copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name)) patch_info.append_add_instruction(to_marfile_entry.name) patch_info.archive_files.append('"' + to_marfile_entry.name + '"') def create_add_if_not_patch_for_file(to_marfile_entry, patch_info): - """ Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """ + """ Copy the file to the working dir, add the add-if-not instruction, + and add it to the list of archive files """ copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name)) patch_info.append_add_if_not_instruction(to_marfile_entry.name) patch_info.archive_files.append('"' + to_marfile_entry.name + '"') def process_explicit_remove_files(dir_path, patch_info): """ Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist this will throw. If found adds the removed-files @@ -330,17 +333,18 @@ def process_explicit_remove_files(dir_pa # Exclude any blank and comment lines. if line and not line.startswith("#"): # Python on windows uses \ for path separators and the update # manifests expects / for path separators on all platforms. line = line.replace("\\", "/") patch_info.append_remove_instruction(line) -def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list): +def create_partial_patch(from_dir_path, to_dir_path, patch_filename, + shas, patch_info, forced_updates, add_if_not_list): """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path""" # Cannocolize the paths for safey from_dir_path = os.path.abspath(from_dir_path) to_dir_path = os.path.abspath(to_dir_path) # Create a hashtable of the from and to directories from_dir_hash, from_file_set, from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path) to_dir_hash, to_file_set, to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path) # Create a list of the forced updates @@ -467,28 +471,28 @@ def get_buildid(work_dir): def decode_filename(filepath): """ Breaks filename/dir structure into component parts based on regex for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar Or linux-i686/en-US/firefox-3.0b3.complete.mar Returns dict with keys product, version, locale, platform, type """ try: - m = re.search( - '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)', - os.path.basename(filepath)) - return m.groupdict() + m = re.search( + '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)', # NOQA: E501 + os.path.basename(filepath)) + return m.groupdict() except Exception as exc: - try: - m = re.search( - '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar', - filepath) - return m.groupdict() - except: - raise Exception("could not parse filepath %s: %s" % (filepath, exc)) + try: + m = re.search( + '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar', # NOQA: E501 + filepath) + return m.groupdict() + except Exception: + raise Exception("could not parse filepath %s: %s" % (filepath, exc)) def create_partial_patches(patches): """ Given the patches generates a set of partial patches""" shas = {} work_dir_root = None metadata = [] @@ -524,18 +528,24 @@ def create_partial_patches(patches): extract_mar(to_filename, work_dir_to) to_decoded = decode_filename(from_filename) to_buildid = get_buildid(work_dir_to) to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest() to_size = str(os.path.getsize(to_filename)) mar_extract_time = time.time() - partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, [ - 'update.manifest', 'updatev2.manifest', 'updatev3.manifest'], []), forced_updates, ['channel-prefs.js', 'update-settings.ini']) + partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, + shas, PatchInfo(work_dir, [ + 'update.manifest', + 'updatev2.manifest', + 'updatev3.manifest' + ], []), + forced_updates, + ['channel-prefs.js', 'update-settings.ini']) partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest() partial_size = str(os.path.getsize(partial_filename)) metadata.append({ 'to_filename': os.path.basename(to_filename), 'from_filename': os.path.basename(from_filename), 'partial_filename': os.path.basename(partial_filename), 'to_buildid': to_buildid, @@ -546,18 +556,18 @@ def create_partial_patches(patches): 'to_size': to_size, 'from_size': from_size, 'partial_size': partial_size, 'to_version': to_decoded['version'], 'from_version': from_decoded['version'], 'locale': from_decoded['locale'], 'platform': from_decoded['platform'], }) - print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num), - str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime)) + print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num), # NOQA: E501 + str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime)) # NOQA: E501 patch_num += 1 return metadata finally: # If we fail or get a ctrl-c during run be sure to clean up temp dir if (work_dir_root and os.path.exists(work_dir_root)): shutil.rmtree(work_dir_root)
--- a/tools/update-packaging/test_make_incremental_updates.py +++ b/tools/update-packaging/test_make_incremental_updates.py @@ -28,37 +28,41 @@ class TestPatchInfo(unittest.TestCase): def test_append_add_instruction(self): self.patch_info.append_add_instruction('file.test') self.assertEquals(['add "file.test"'], self.patch_info.manifestv2) self.assertEquals(['add "file.test"'], self.patch_info.manifestv3) def test_append_add_if_instruction(self): self.patch_info.append_add_instruction('distribution/extensions/extension/file.test') self.assertEquals( - ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2) + ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], # NOQA: E501 + self.patch_info.manifestv2) self.assertEquals( - ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3) + ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], # NOQA: E501 + self.patch_info.manifestv3) def test_append_add_if_not_instruction(self): self.patch_info.append_add_if_not_instruction('file.test') self.assertEquals([], self.patch_info.manifestv2) self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3) def test_append_patch_instruction(self): self.patch_info.append_patch_instruction('file.test', 'patchname') self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2) self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3) def test_append_patch_if_instruction(self): self.patch_info.append_patch_instruction( 'distribution/extensions/extension/file.test', 'patchname') self.assertEquals( - ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2) + ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], # NOQA: E501 + self.patch_info.manifestv2) self.assertEquals( - ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3) + ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], # NOQA: E501 + self.patch_info.manifestv3) def test_append_remove_instruction(self): self.patch_info.append_remove_instruction('file.test') self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2) self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3) def test_append_rmdir_instruction(self): self.patch_info.append_remove_instruction('dirtest/') @@ -105,17 +109,19 @@ class TestMarFileEntry(unittest.TestCase self.assertEquals(goodSha, sha) """ class TestMakeIncrementalUpdates(unittest.TestCase): def setUp(self): work_dir = '.' self.patch_info = PatchInfo( - work_dir, ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'], ['/readme.txt']) + work_dir, + ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'], + ['/readme.txt']) root_path = '/' filename = 'test.file' self.mar_file_entry = MarFileEntry(root_path, filename) """ FIXME makes direct shell calls, need refactoring def test_exec_shell_cmd(self): mkup.exec_shell_cmd('echo test') @@ -127,26 +133,28 @@ class TestMakeIncrementalUpdates(unittes def test_bunzip_file(self): mkup.bunzip_file('filename') def test_extract_mar(self): mkup.extract_mar('filename', 'work_dir') def test_create_partial_patch_for_file(self): - mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info) + mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', + self.patch_info) def test_create_add_patch_for_file(self): mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info) def test_process_explicit_remove_files(self): mkup.process_explicit_remove_files('dir_path', self.patch_info) def test_create_partial_patch(self): - mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates') + mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', + self.patch_info, 'forced_updates') def test_create_partial_patches(patches): mkup.create_partial_patches('patches') """ """ FIXME touches the filesystem, need refactoring def test_get_buildid(self):