Bug 1187494 - Keep reproducibility of Firefox UI tests up-to-date. DONTBUILD r=bhearsum a=testonly
authorArmen Zambrano Gasparnian <armenzg@mozilla.com>
Mon, 27 Jul 2015 11:03:39 -0400
changeset 275457 694ac8f225aae5940ae940ebdfd370b721d09e1c
parent 275456 ef641c2968c2d255a36823364b8f8ebf943aa370
child 275458 df90f57c5003ef49a88aa08eab7af7724c7c788b
push id863
push userraliiev@mozilla.com
push dateMon, 03 Aug 2015 13:22:43 +0000
treeherdermozilla-release@f6321b14228d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbhearsum, testonly
bugs1187494
milestone40.0
Bug 1187494 - Keep reproducibility of Firefox UI tests up-to-date. DONTBUILD r=bhearsum a=testonly When a local fails on Firefox UI tests we output the mozharness command which a person could run on their local development machine to reproduce the exact failure that we encoutered. As parameters and options changes over time, we can fall out of date. This patchs tries to reduce how we can fall out of date by constructing the command rather than having a hardcoded output. Another aspect which this patch solves is that for Firefox releases we have to do some string manipulation to determine where we can download Firefox crash symbols. We've modified the script to output in the log the candidate URLs that we're evaluating. This is useful if anyone wants to determine why --symbols-path was or was not included. For instance, older candidate directories on FTP are deleted after some time, hence, deleting the crash symbols available. Another matter we fix is that we include in the output what command was run internally to the Firefox UI test binaries. This is useful if anyone wants to determine that the right command are arguments were used.
testing/mozharness/scripts/firefox_ui_updates.py
--- a/testing/mozharness/scripts/firefox_ui_updates.py
+++ b/testing/mozharness/scripts/firefox_ui_updates.py
@@ -119,16 +119,17 @@ class FirefoxUIUpdates(FirefoxUITests):
             # We want to make sure that anyone trying to reproduce a job will
             # is using the exact tools tag for reproducibility's sake
             self.fatal('Make sure to specify the --tools-tag')
 
         self.tools_repo = self.config['tools_repo']
         self.tools_tag = self.config['tools_tag']
 
         if self.config.get('update_verify_config'):
+            self.update_verify_config = self.config['update_verify_config']
             self.updates_config_file = os.path.join(
                 dirs['abs_tools_dir'], 'release', 'updates',
                 self.config['update_verify_config']
             )
         else:
             self.fatal('Make sure to specify --update-verify-config. '
                        'See under the directory release/updates in %s.' % self.tools_repo)
 
@@ -247,27 +248,30 @@ class FirefoxUIUpdates(FirefoxUITests):
         # packages are.
         # We want this:
         # https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
         # https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
         installer_from = rel_info['from']
         version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], installer_from)).group(1)
 
         temp_from = installer_from.replace(version, '%s-candidates/build%s' % (version, self.config["build_number"]), 1).replace('releases', 'candidates')
+        temp_url = rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US'))
+        self.info('Installer url under stage/candidates dir %s' % temp_url)
 
-        return rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US'))
+        return temp_url
 
 
     def _query_symbols_url(self, installer_url):
         for suffix in INSTALLER_SUFFIXES:
             if installer_url.endswith(suffix):
                 symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
                 continue
 
         if symbols_url:
+            self.info('Candidate symbols_url: %s' % symbols_url)
             if not symbols_url.startswith('http'):
                 return symbols_url
 
             try:
                 # Let's see if the symbols are available
                 urllib2.urlopen(symbols_url)
                 return symbols_url
 
@@ -316,16 +320,19 @@ class FirefoxUIUpdates(FirefoxUITests):
             cmd += ['--update-channel', update_channel]
 
         return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
                                        output_timeout=300,
                                        env=env)
 
         # Return more output if we fail
         if return_code != 0:
+            self.info('Internally this is the command fx-ui-updates executed')
+            self.info('%s' % ' '.join(map(str, cmd)))
+
             if os.path.exists(gecko_log):
                 contents = self.read_from_file(gecko_log, verbose=False)
                 self.warning('== Dumping gecko output ==')
                 self.warning(contents)
                 self.warning('== End of gecko output ==')
             else:
                 # We're outputting to stdout with --gecko-log=- so there is not log to
                 # complaing about. Remove the commented line below when changing
@@ -382,47 +389,52 @@ class FirefoxUIUpdates(FirefoxUITests):
                 for locale in rel_info['locales']:
                     self.info("Running %s %s" % (build_id, locale))
 
                     # Safe temp hack to determine symbols URL from en-US build1 in the candidates dir
                     ftp_candidates_installer_url = self._modify_url(rel_info)
                     symbols_url = self._query_symbols_url(installer_url=ftp_candidates_installer_url)
 
                     # Determine from where to download the file
-                    url = '%s/%s' % (
+                    installer_url = '%s/%s' % (
                         rel_info['ftp_server_from'],
                         urllib.quote(rel_info['from'].replace('%locale%', locale))
                     )
                     installer_path = self.download_file(
-                        url=url,
+                        url=installer_url,
                         parent_dir=dirs['abs_work_dir']
                     )
 
                     marionette_port += 1
 
                     retcode = self._run_test(
                         installer_path=installer_path,
                         symbols_url=symbols_url,
                         update_channel=self.channel,
                         marionette_port=marionette_port)
 
                     if retcode != 0:
                         self.warning('FAIL: firefox-ui-update has failed.' )
-                        self.info('You can run the following command on the same machine to reproduce the issue:')
-                        self.info('python scripts/firefox_ui_updates.py --cfg generic_releng_config.py '
-                                  '--firefox-ui-branch %s --update-verify-config %s '
-                                  '--tools-tag %s --installer-url %s '
-                                  '--determine-testing-configuration --run-tests '
-                                  % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
-                        self.info('If you want to run this on your development machine:')
-                        self.info('python scripts/firefox_ui_updates.py '
-                                  '--firefox-ui-branch %s --update-verify-config %s '
-                                  '--tools-tag %s --installer-url %s '
-                                  '--cfg developer_config.py '
-                                  % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
+
+                        base_cmd = 'python scripts/firefox_ui_updates.py'
+                        for c in self.config['config_files']:
+                            base_cmd += ' --cfg %s' % c
+
+                        base_cmd += ' --firefox-ui-branch %s --update-verify-config %s --tools-tag %s' % \
+                            (self.firefox_ui_branch, self.update_verify_config, self.tools_tag)
+
+                        base_cmd += ' --installer-url %s' % installer_url
+                        if symbols_url:
+                            base_cmd += ' --symbols-path %s' % symbols_url
+
+                        self.info('You can run the *specific* locale on the same machine with:')
+                        self.info('%s' % base_cmd)
+
+                        self.info('You can run the *specific* locale on *your* machine with:')
+                        self.info('%s --cfg developer_config.py' % base_cmd)
 
                     results[build_id][locale] = retcode
 
             # Determine which locales have failed and set scripts exit code
             exit_status = TBPL_SUCCESS
             for build_id in sorted(results.keys()):
                 failed_locales = []
                 for locale in sorted(results[build_id].keys()):