Bug 1186083 - Land mozharness changes required for Firefox UI tests on aurora and beta. DONTBUILD. r=jlund a=testing
authorArmen Zambrano Gasparnian <armenzg@mozilla.com>
Tue, 21 Jul 2015 18:28:58 -0400
changeset 275396 01c603755e16cf614d6a5ccabb3983f7f407acba
parent 275393 47e626a4275a08a5513d1879d2a1aede42549f3c
child 275397 527271c3a4bc19af9a4542ccb3c98e902f97a667
push id863
push userraliiev@mozilla.com
push dateMon, 03 Aug 2015 13:22:43 +0000
treeherdermozilla-release@f6321b14228d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund, testing
bugs1186083
milestone40.0
Bug 1186083 - Land mozharness changes required for Firefox UI tests on aurora and beta. DONTBUILD. r=jlund a=testing
testing/mozharness/configs/generic_releng_linux.py
testing/mozharness/configs/generic_releng_linux64.py
testing/mozharness/configs/generic_releng_macosx64.py
testing/mozharness/configs/generic_releng_win32.py
testing/mozharness/configs/generic_releng_win64.py
testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
testing/mozharness/scripts/firefox_ui_updates.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_linux.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_linux64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_macosx64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/osx64/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_win32.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win32/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_win64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win64/minidump_stackwalk',
+    }
+}
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
@@ -13,71 +13,57 @@ import sys
 import os
 
 from mozharness.base.python import (
     PreScriptAction,
     VirtualenvMixin,
     virtualenv_config_options,
 )
 from mozharness.mozilla.vcstools import VCSToolsScript
-from mozharness.mozilla.purge import PurgeMixin
 
 
-class FirefoxUITests(VCSToolsScript, VirtualenvMixin, PurgeMixin):
+class FirefoxUITests(VCSToolsScript, VirtualenvMixin):
     config_options = [
         [['--firefox-ui-repo'], {
             'dest': 'firefox_ui_repo',
             'default': 'https://github.com/mozilla/firefox-ui-tests.git',
             'help': 'which firefox_ui_tests repo to use',
         }],
         [['--firefox-ui-branch'], {
             'dest': 'firefox_ui_branch',
             'help': 'which branch to use for firefox_ui_tests',
         }],
     ] + copy.deepcopy(virtualenv_config_options)
 
-    def __init__(self,
-                 config={},
-                 config_options=[],
-                 all_actions=[],
-                 **kwargs):
-        default_config = {
-            'purge_minsize': 2,
-        }
-        default_config.update(config)
 
+    def __init__(self, config_options=[], all_actions=[], **kwargs):
         self.config_options += config_options
 
         if all_actions is None:
             # Default actions
             all_actions = [
-                'purge-builds',
                 'clobber',
                 'checkout',
                 'create-virtualenv',
                 'run-tests',
             ]
 
         super(FirefoxUITests, self).__init__(
             config_options=self.config_options,
-            require_config_file=True,
-            config=default_config,
             all_actions=all_actions,
             **kwargs
         )
 
         self.firefox_ui_repo = self.config['firefox_ui_repo']
+        self.firefox_ui_branch = self.config.get('firefox_ui_branch')
 
-        if 'checkout' in self.actions:
-            try:
-                self.firefox_ui_branch = self.config['firefox_ui_branch']
-            except:
-                self.fatal(
-                    'Please specify --firefox-ui-branch. Valid values can be found '
-                    'in here https://github.com/mozilla/firefox-ui-tests/branches')
+        if not self.firefox_ui_branch:
+            self.fatal(
+                'Please specify --firefox-ui-branch. Valid values can be found '
+                'in here https://github.com/mozilla/firefox-ui-tests/branches')
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxUITests, self).query_abs_dirs()
 
         dirs = {
             'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
--- a/testing/mozharness/scripts/firefox_ui_updates.py
+++ b/testing/mozharness/scripts/firefox_ui_updates.py
@@ -5,26 +5,28 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """firefox_ui_updates.py
 
 Author: Armen Zambrano G.
 """
 import copy
 import os
+import re
 import urllib
+import urllib2
 import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import INFO
 from mozharness.base.script import PreScriptAction
 from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests
 
+INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz')
 
 class FirefoxUIUpdates(FirefoxUITests):
     # This will be a list containing one item per release based on configs
     # from tools/release/updates/*cfg
     releases = None
     channel = None
     harness_extra_args = [
         [['--update-allow-mar-channel'], {
@@ -40,16 +42,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         [['--update-target-version'], {
             'dest': 'update_target_version',
             'help': 'Version of the updated build.',
         }],
         [['--update-target-buildid'], {
             'dest': 'update_target_buildid',
             'help': 'Build ID of the updated build',
         }],
+        [['--symbols-path=SYMBOLS_PATH'], {
+            'dest': 'symbols_path',
+            'help': 'absolute path to directory containing breakpad '
+                    'symbols, or the url of a zip file containing symbols.',
+        }],
     ]
 
 
     def __init__(self):
         config_options = [
             [['--tools-repo'], {
                 'dest': 'tools_repo',
                 'default': 'http://hg.mozilla.org/build/tools',
@@ -73,52 +80,56 @@ class FirefoxUIUpdates(FirefoxUITests):
                 'default': 1,
                 'help': 'Total chunks to dive the locales into.',
             }],
             [['--dry-run'], {
                 'dest': 'dry_run',
                 'default': False,
                 'help': 'Only show what was going to be tested.',
             }],
+            [["--build-number"], {
+                "dest": "build_number",
+                "help": "Build number of release, eg: 2",
+            }],
             # These are options when we don't use the releng update config file
             [['--installer-url'], {
                 'dest': 'installer_url',
                 'help': 'Point to an installer to download and test against.',
             }],
             [['--installer-path'], {
                 'dest': 'installer_path',
                 'help': 'Point to an installer to test against.',
             }],
         ] + copy.deepcopy(self.harness_extra_args)
 
         super(FirefoxUIUpdates, self).__init__(
             config_options=config_options,
             all_actions=[
-                'purge-builds',
                 'clobber',
                 'checkout',
                 'create-virtualenv',
                 'determine-testing-configuration',
                 'run-tests',
             ],
+            append_env_variables_from_configs=True,
         )
 
         dirs = self.query_abs_dirs()
 
         if self.config.get('tools_tag') is None:
             # We want to make sure that anyone trying to reproduce a job will
             # is using the exact tools tag for reproducibility's sake
             self.fatal('Make sure to specify the --tools-tag')
 
         self.tools_repo = self.config['tools_repo']
         self.tools_tag = self.config['tools_tag']
 
         if self.config.get('update_verify_config'):
             self.updates_config_file = os.path.join(
-                dirs['tools_dir'], 'release', 'updates',
+                dirs['abs_tools_dir'], 'release', 'updates',
                 self.config['update_verify_config']
             )
         else:
             self.fatal('Make sure to specify --update-verify-config. '
                        'See under the directory release/updates in %s.' % self.tools_repo)
 
         self.installer_url = self.config.get('installer_url')
         self.installer_path = self.config.get('installer_path')
@@ -133,17 +144,17 @@ class FirefoxUIUpdates(FirefoxUITests):
 
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxUIUpdates, self).query_abs_dirs()
 
         dirs = {
-            'tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
+            'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
         }
 
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     def checkout(self):
         '''
@@ -153,17 +164,17 @@ class FirefoxUIUpdates(FirefoxUITests):
         We also checkout firefox_ui_tests and update to the right branch
         for it.
         '''
         super(FirefoxUIUpdates, self).checkout()
         dirs = self.query_abs_dirs()
 
         self.vcs_checkout(
             repo=self.tools_repo,
-            dest=dirs['tools_dir'],
+            dest=dirs['abs_tools_dir'],
             revision=self.tools_tag,
             vcs='hgtool'
         )
 
 
     def determine_testing_configuration(self):
         '''
         This method builds a testing matrix either based on an update verification
@@ -189,21 +200,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         of all locales (except for the most recent releases). A quick release has all locales,
         however, it misses the fields 'from' and 'ftp_server_from'.
         Both pairs of information complement each other but differ in such manner.
         '''
         if self.installer_url or self.installer_path:
             return
 
         dirs = self.query_abs_dirs()
-        assert os.path.exists(dirs['tools_dir']), \
+        assert os.path.exists(dirs['abs_tools_dir']), \
             "Without the tools/ checkout we can't use releng's config parser."
 
         # Import the config parser
-        sys.path.insert(1, os.path.join(dirs['tools_dir'], 'lib', 'python'))
+        sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python'))
         from release.updates.verify import UpdateVerifyConfig
 
         uvc = UpdateVerifyConfig()
         uvc.read(self.updates_config_file)
         self.channel = uvc.channel
 
         # Filter out any releases that are less than Gecko 38
         uvc.releases = [r for r in uvc.releases \
@@ -224,43 +235,81 @@ class FirefoxUIUpdates(FirefoxUITests):
         chunked_config = uvc.getChunk(
             chunks=int(self.config['total_chunks']),
             thisChunk=int(self.config['this_chunk'])
         )
 
         self.releases = chunked_config.releases
 
 
+    def _modify_url(self, rel_info):
+        # This is a temporary hack to find crash symbols. It should be replaced
+        # with something that doesn't make wild guesses about where symbol
+        # packages are.
+        # We want this:
+        # https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
+        # https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
+        installer_from = rel_info['from']
+        version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], installer_from)).group(1)
+
+        temp_from = installer_from.replace(version, '%s-candidates/build%s' % (version, self.config["build_number"]), 1).replace('releases', 'candidates')
+
+        return rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US'))
+
+
+    def _query_symbols_url(self, installer_url):
+        for suffix in INSTALLER_SUFFIXES:
+            if installer_url.endswith(suffix):
+                symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
+                continue
+
+        if symbols_url:
+            if not symbols_url.startswith('http'):
+                return symbols_url
+
+            try:
+                # Let's see if the symbols are available
+                return urllib2.urlopen(symbols_url)
+
+            except urllib2.HTTPError, e:
+                self.warning("%s - %s" % (str(e), symbols_url))
+                return None
+        else:
+            self.fatal("Can't figure out symbols_url from installer_url %s!" % installer_url)
+
+
     @PreScriptAction('run-tests')
     def _pre_run_tests(self, action):
         if self.releases is None and not (self.installer_url or self.installer_path):
-            self.critical('You need to call --determine-testing-configuration as well.')
-            exit(1)
+            self.fatal('You need to call --determine-testing-configuration as well.')
 
 
-    def _run_test(self, installer_path, update_channel=None, cleanup=True,
+    def _run_test(self, installer_path, symbols_url=None, update_channel=None, cleanup=True,
                   marionette_port=2828):
         '''
         All required steps for running the tests against an installer.
         '''
-        env = self.query_env()
         dirs = self.query_abs_dirs()
+        env = self.query_env(avoid_host_env=True)
         bin_dir = os.path.dirname(self.query_python_path())
         fx_ui_tests_bin = os.path.join(bin_dir, 'firefox-ui-update')
         gecko_log=os.path.join(dirs['abs_work_dir'], 'gecko.log')
 
         # Build the command
         cmd = [
             fx_ui_tests_bin,
             '--installer', installer_path,
             # Log to stdout until tests are stable.
             '--gecko-log=-',
             '--address=localhost:%s' % marionette_port,
         ]
 
+        if symbols_url:
+            cmd += ['--symbols-path', symbols_url]
+
         for arg in self.harness_extra_args:
             dest = arg[1]['dest']
             if dest in self.config:
                 cmd += [' '.join(arg[0]), self.config[dest]]
 
         if update_channel:
             cmd += ['--update-channel', update_channel]
 
@@ -271,17 +320,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         # Return more output if we fail
         if return_code != 0:
             if os.path.exists(gecko_log):
                 contents = self.read_from_file(gecko_log, verbose=False)
                 self.warning('== Dumping gecko output ==')
                 self.warning(contents)
                 self.warning('== End of gecko output ==')
             else:
-                self.warning('No gecko.log was found: %s' % gecko_log)
+                # We're outputting to stdout with --gecko-log=- so there is not log to
+                # complaing about. Remove the commented line below when changing
+                # this behaviour.
+                # self.warning('No gecko.log was found: %s' % gecko_log)
+                pass
 
         if cleanup:
             for filepath in (installer_path, gecko_log):
                 if os.path.exists(filepath):
                     self.debug('Removing %s' % filepath)
                     os.remove(filepath)
 
         return return_code
@@ -292,57 +345,90 @@ class FirefoxUIUpdates(FirefoxUITests):
 
         if self.installer_url or self.installer_path:
             if self.installer_url:
                 self.installer_path = self.download_file(
                     self.installer_url,
                     parent_dir=dirs['abs_work_dir']
                 )
 
-            return self._run_test(self.installer_path, cleanup=False)
+            symbols_url = self._query_symbols_url(installer_url=self.installer_path)
+
+            return self._run_test(
+                installer_path=self.installer_path,
+                symbols_url=symbols_url,
+                cleanup=False
+            )
 
         else:
+            results = {}
+
             for rel_info in sorted(self.releases, key=lambda release: release['build_id']):
+                build_id = rel_info['build_id']
+                results[build_id] = {}
+
                 self.info('About to run %s %s - %s locales' % (
-                    rel_info['build_id'],
+                    build_id,
                     rel_info['from'],
                     len(rel_info['locales'])
                 ))
 
                 if self.config['dry_run']:
                     continue
 
                 # Each locale gets a fresh port to avoid address in use errors in case of
                 # tests that time out unexpectedly.
                 marionette_port = 2827
                 for locale in rel_info['locales']:
+                    self.info("Running %s %s" % (build_id, locale))
+
+                    # Safe temp hack to determine symbols URL from en-US build1 in the candidates dir
+                    ftp_candidates_installer_url = self._modify_url(rel_info)
+                    symbols_url = self._query_symbols_url(installer_url=ftp_candidates_installer_url)
+
                     # Determine from where to download the file
                     url = '%s/%s' % (
                         rel_info['ftp_server_from'],
                         urllib.quote(rel_info['from'].replace('%locale%', locale))
                     )
                     installer_path = self.download_file(
                         url=url,
                         parent_dir=dirs['abs_work_dir']
                     )
 
                     marionette_port += 1
-                    retcode = self._run_test(installer_path, self.channel,
-                                             marionette_port=marionette_port)
+
+                    retcode = self._run_test(
+                        installer_path=installer_path,
+                        symbols_url=symbols_url,
+                        update_channel=self.channel,
+                        marionette_port=marionette_port)
+
                     if retcode != 0:
                         self.warning('FAIL: firefox-ui-update has failed.' )
                         self.info('You can run the following command on the same machine to reproduce the issue:')
                         self.info('python scripts/firefox_ui_updates.py --cfg generic_releng_config.py '
                                   '--firefox-ui-branch %s --update-verify-config %s '
                                   '--tools-tag %s --installer-url %s '
                                   '--determine-testing-configuration --run-tests '
                                   % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
                         self.info('If you want to run this on your development machine:')
                         self.info('python scripts/firefox_ui_updates.py '
                                   '--firefox-ui-branch %s --update-verify-config %s '
                                   '--tools-tag %s --installer-url %s '
                                   '--cfg developer_config.py '
                                   % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
 
+                    results[build_id][locale] = retcode
+
+            self.info("Firefox UI update tests failed locales:")
+            for build_id in sorted(results.keys()):
+                self.info(build_id)
+                failed_locales = []
+                for locale in sorted(results[build_id].keys()):
+                    if results[build_id][locale] != 0:
+                        failed_locales.append(locale)
+                self.info("  %s" % (', '.join(failed_locales)))
+
 
 if __name__ == '__main__':
     myScript = FirefoxUIUpdates()
     myScript.run_and_exit()