Bug 1154796 - sync in-tree copy of mozharness to hgmo mozharness 239491bc393a, r=ryanvm a=testing
authorJordan Lund <jlund@mozilla.com>
Thu, 16 Jul 2015 11:01:57 -0700
changeset 253078 3d800e3a022e710099df52c2638874d5a26aa991
parent 253077 d8023b434e25beed6cb5d38b57d6bb27aaf888df
child 253079 e89a57634b17cd1a3256b017150b4e064fa92118
push id29060
push userjlund@mozilla.com
push dateThu, 16 Jul 2015 18:16:51 +0000
treeherdermozilla-central@e89a57634b17 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersryanvm, testing
bugs1154796
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1154796 - sync in-tree copy of mozharness to hgmo mozharness 239491bc393a, r=ryanvm a=testing
testing/mozharness/README.txt
testing/mozharness/configs/android/androidarm.py
testing/mozharness/configs/android/androidarm_4_3.py
testing/mozharness/configs/builds/releng_base_mac_cross_builds.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py
testing/mozharness/configs/generic_releng_linux.py
testing/mozharness/configs/generic_releng_linux64.py
testing/mozharness/configs/generic_releng_macosx64.py
testing/mozharness/configs/generic_releng_win32.py
testing/mozharness/configs/generic_releng_win64.py
testing/mozharness/configs/hazards/common.py
testing/mozharness/configs/single_locale/alder.py
testing/mozharness/configs/single_locale/ash.py
testing/mozharness/configs/single_locale/mozilla-aurora.py
testing/mozharness/configs/single_locale/mozilla-central.py
testing/mozharness/mozharness/base/config.py
testing/mozharness/mozharness/base/script.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/mozharness/scripts/b2g_emulator_unittest.py
testing/mozharness/scripts/desktop_l10n.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/firefox_ui_updates.py
testing/mozharness/scripts/hazard_build.py
testing/mozharness/scripts/spidermonkey_build.py
--- a/testing/mozharness/README.txt
+++ b/testing/mozharness/README.txt
@@ -1,11 +1,11 @@
 the contents of this dir (testing/mozharness) represent two parts
 
 1) the old way: mozharness.json is a manifest file that locks or "pins" mozharness to a repository and a revision.
 
 2) the new way: an in-gecko-tree copy of mozharness.
     * hgmo/build/mozharness is still live and the defacto read/write repository
     * continuous integration jobs are based on this copy
     * As we transition to dropping support for hg.m.o/build/mozharness, this copy will continue to be synced
-    * this copy is currently based on: http://hg.mozilla.org/build/mozharness/rev/43f7600b8c80
+    * this copy is currently based on: http://hg.mozilla.org/build/mozharness/rev/239491bc393a
 
 
--- a/testing/mozharness/configs/android/androidarm.py
+++ b/testing/mozharness/configs/android/androidarm.py
@@ -138,16 +138,20 @@ config = {
         "mochitest-15": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=15"],
         },
         "mochitest-16": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=16"],
         },
+        "mochitest-chrome": {
+            "category": "mochitest",
+            "extra_args": ["--chrome"],
+        },
         "mochitest-gl-1": {
             "category": "mochitest-gl",
             "extra_args": ["--this-chunk=1"],
         },
         "mochitest-gl-2": {
             "category": "mochitest-gl",
             "extra_args": ["--this-chunk=2"],
         },
--- a/testing/mozharness/configs/android/androidarm_4_3.py
+++ b/testing/mozharness/configs/android/androidarm_4_3.py
@@ -144,16 +144,20 @@ config = {
         "mochitest-15": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=15"],
         },
         "mochitest-16": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=16"],
         },
+        "mochitest-chrome": {
+            "category": "mochitest",
+            "extra_args": ["--chrome"],
+        },
         "mochitest-gl-1": {
             "category": "mochitest-gl",
             "extra_args": ["--this-chunk=1"],
         },
         "mochitest-gl-2": {
             "category": "mochitest-gl",
             "extra_args": ["--this-chunk=2"],
         },
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_base_mac_cross_builds.py
@@ -0,0 +1,87 @@
+import os
+import sys
+
+config = {
+    #########################################################################
+    ######## MACOSX CROSS GENERIC CONFIG KEYS/VAlUES
+
+    'default_actions': [
+        'clobber',
+        'clone-tools',
+        'checkout-sources',
+        'build',
+        'generate-build-stats',
+        'update',  # decided by query_is_nightly()
+    ],
+    "buildbot_json_path": "buildprops.json",
+    'exes': {
+        'python2.7': sys.executable,
+        'hgtool.py': os.path.join(
+            os.getcwd(), 'build', 'tools', 'buildfarm', 'utils', 'hgtool.py'
+        ),
+        "buildbot": "/tools/buildbot/bin/buildbot",
+    },
+    'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
+    # decides whether we want to use moz_sign_cmd in env
+    'enable_signing': True,
+    'purge_skip': ['info', 'rel-*:45d', 'tb-rel-*:45d'],
+    'purge_basedirs':  [],
+    'enable_ccache': True,
+    'enable_check_test': False,
+    'vcs_share_base': '/builds/hg-shared',
+    'objdir': 'obj-firefox/',
+    'tooltool_script': ["/builds/tooltool.py"],
+    'tooltool_bootstrap': "setup.sh",
+    'enable_count_ctors': False,
+    'enable_talos_sendchange': True,
+    'enable_unittest_sendchange': True,
+    #########################################################################
+
+
+    #########################################################################
+    ###### 64 bit specific ######
+    'base_name': 'OS X 10.7 %(branch)s',
+    'platform': 'macosx64',
+    'stage_platform': 'macosx64',
+    'use_platform_in_symbols_extra_buildid': True,
+    'env': {
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'MOZ_AUTOMATION': '1',
+        'HG_SHARE_BASE_DIR': '/builds/hg-shared',
+        'MOZ_OBJDIR': 'obj-firefox',
+        # SYMBOL_SERVER_HOST is dictated from build_pool_specifics.py
+        'SYMBOL_SERVER_HOST': '%(symbol_server_host)s',
+        'SYMBOL_SERVER_SSH_KEY': "/Users/cltbld/.ssh/ffxbld_rsa",
+        'SYMBOL_SERVER_USER': 'ffxbld',
+        'SYMBOL_SERVER_PATH': '/mnt/netapp/breakpad/symbols_ffx/',
+        'POST_SYMBOL_UPLOAD_CMD': '/usr/local/bin/post-symbol-upload.py',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': '/builds/tooltool_cache',
+        'TOOLTOOL_HOME': '/builds',
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'CCACHE_DIR': '/builds/ccache',
+        'CCACHE_COMPRESS': '1',
+        'CCACHE_UMASK': '002',
+        'LC_ALL': 'C',
+        ## 64 bit specific
+        'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
+/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
+/tools/python27-mercurial/bin:/home/cltbld/bin',
+        ##
+    },
+    'upload_env': {
+        # stage_server is dictated from build_pool_specifics.py
+        'UPLOAD_HOST': '%(stage_server)s',
+        'UPLOAD_USER': '%(stage_username)s',
+        'UPLOAD_SSH_KEY': '/Users/cltbld/.ssh/%(stage_ssh_key)s',
+        'UPLOAD_TO_TEMP': '1',
+    },
+    "check_test_env": {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
+        'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
+    },
+    'purge_minsize': 12,
+    'src_mozconfig': 'browser/config/mozconfigs/macosx64/nightly',
+    'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
+    #########################################################################
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py
@@ -0,0 +1,47 @@
+import os
+
+MOZ_OBJDIR = 'obj-firefox'
+
+config = {
+    'default_actions': [
+        'clobber',
+        'clone-tools',
+        'checkout-sources',
+        'setup-mock',
+        'build',
+        'upload-files',
+        'sendchange',
+        # 'check-test',
+        # 'generate-build-stats',
+        # 'update',
+    ],
+    'stage_platform': 'linux64-tsan',
+    'purge_minsize': 12,
+    'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
+tsan.manifest",
+    'platform_supports_post_upload_to_latest': False,
+    'enable_signing': False,
+    'enable_talos_sendchange': False,
+    #### 64 bit build specific #####
+    'env': {
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'MOZ_AUTOMATION': '1',
+        'DISPLAY': ':2',
+        'HG_SHARE_BASE_DIR': '/builds/hg-shared',
+        'MOZ_OBJDIR': 'obj-firefox',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': '/builds/tooltool_cache',
+        'TOOLTOOL_HOME': '/builds',
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'CCACHE_DIR': '/builds/ccache',
+        'CCACHE_COMPRESS': '1',
+        'CCACHE_UMASK': '002',
+        'LC_ALL': 'C',
+        ## 64 bit specific
+        'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
+/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
+/tools/python27-mercurial/bin:/home/cltbld/bin',
+    },
+    'src_mozconfig': 'browser/config/mozconfigs/linux64/opt-tsan',
+    #######################
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_linux.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_linux64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_macosx64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/osx64/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_win32.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win32/minidump_stackwalk',
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/generic_releng_win64.py
@@ -0,0 +1,5 @@
+config = {
+    'env': {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win64/minidump_stackwalk',
+    }
+}
--- a/testing/mozharness/configs/hazards/common.py
+++ b/testing/mozharness/configs/hazards/common.py
@@ -33,17 +33,19 @@ config = {
         "dest": "tools"
     }],
 
     "upload_remote_baseuri": 'https://ftp-ssl.mozilla.org/',
 
     'tools_dir': "/tools",
     'compiler_manifest': "build/gcc.manifest",
     'b2g_compiler_manifest': "build/gcc-b2g.manifest",
+    'compiler_setup': "setup.sh.gcc",
     'sixgill_manifest': "build/sixgill.manifest",
+    'sixgill_setup': "setup.sh.sixgill",
 
     # Mock.
     "mock_packages": [
         "autoconf213", "mozilla-python27-mercurial", "ccache",
         "zip", "zlib-devel", "glibc-static",
         "openssh-clients", "mpfr", "wget", "rsync",
 
         # For building the JS shell
--- a/testing/mozharness/configs/single_locale/alder.py
+++ b/testing/mozharness/configs/single_locale/alder.py
@@ -22,19 +22,17 @@ config = {
     "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
     "update_channel": "nightly",
     "latest_mar_dir": '/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n',
 
     # l10n
     "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
 
     # mar
-    "enable_partials": True,
     "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-    "previous_mar_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n",
 
     # repositories
     "mozilla_dir": "alder",
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "revision": "default",
         "dest": "tools",
--- a/testing/mozharness/configs/single_locale/ash.py
+++ b/testing/mozharness/configs/single_locale/ash.py
@@ -22,19 +22,17 @@ config = {
     "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
     "update_channel": "nightly",
     "latest_mar_dir": '/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n',
 
     # l10n
     "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
 
     # mar
-    "enable_partials": True,
     "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-    "previous_mar_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n",
 
     # repositories
     "mozilla_dir": "ash",
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "revision": "default",
         "dest": "tools",
--- a/testing/mozharness/configs/single_locale/mozilla-aurora.py
+++ b/testing/mozharness/configs/single_locale/mozilla-aurora.py
@@ -4,19 +4,17 @@ config = {
     "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora/",
     "update_channel": "aurora",
     "latest_mar_dir": '/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora-l10n',
 
     # l10n
     "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-aurora",
 
     # mar
-    "enable_partials": True,
     "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora/mar-tools/%(platform)s",
-    "previous_mar_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora-l10n",
 
     # repositories
     "mozilla_dir": "mozilla-aurora",
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "revision": "default",
         "dest": "tools",
--- a/testing/mozharness/configs/single_locale/mozilla-central.py
+++ b/testing/mozharness/configs/single_locale/mozilla-central.py
@@ -4,19 +4,17 @@ config = {
     "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
     "update_channel": "nightly",
     "latest_mar_dir": '/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n',
 
     # l10n
     "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
 
     # mar
-    "enable_partials": True,
     "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-    "previous_mar_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central-l10n",
 
     # repositories
     "mozilla_dir": "mozilla-central",
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "revision": "default",
         "dest": "tools",
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -206,22 +206,26 @@ def download_config_file(url, file_name)
 
 # BaseConfig {{{1
 class BaseConfig(object):
     """Basic config setting/getting.
     """
     def __init__(self, config=None, initial_config_file=None, config_options=None,
                  all_actions=None, default_actions=None,
                  volatile_config=None, option_args=None,
-                 require_config_file=False, usage="usage: %prog [options]"):
+                 require_config_file=False,
+                 append_env_variables_from_configs=False,
+                 usage="usage: %prog [options]"):
         self._config = {}
         self.all_cfg_files_and_dicts = []
         self.actions = []
         self.config_lock = False
         self.require_config_file = require_config_file
+        # It allows to append env variables from multiple config files
+        self.append_env_variables_from_configs = append_env_variables_from_configs
 
         if all_actions:
             self.all_actions = all_actions[:]
         else:
             self.all_actions = ['clobber', 'build']
         if default_actions:
             self.default_actions = default_actions[:]
         else:
@@ -472,18 +476,28 @@ class BaseConfig(object):
             # config file name and its assoctiated dict
             # eg ('builds/branch_specifics.py', {'foo': 'bar'})
             # let's store this to self for things like --interpret-config-files
             self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
                 # append opt_config to allow them to overwrite previous configs
                 options.config_files + options.opt_config_files, options=options
             ))
             config = {}
-            for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
-                config.update(c_dict)
+            if self.append_env_variables_from_configs:
+                # We only append values from various configs for the 'env' entry
+                # For everything else we follow the standard behaviour
+                for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
+                    for v in c_dict.keys():
+                        if v == 'env' and v in config:
+                            config[v].update(c_dict[v])
+                        else:
+                            config[v] = c_dict[v]
+            else:
+                for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
+                    config.update(c_dict)
             # assign or update self._config depending on if it exists or not
             #    NOTE self._config will be passed to ReadOnlyConfig's init -- a
             #    dict subclass with immutable locking capabilities -- and serve
             #    as the keys/values that make up that instance. Ultimately,
             #    this becomes self.config during BaseScript's init
             self.set_config(config)
         for key in defaults.keys():
             value = getattr(options, key)
--- a/testing/mozharness/mozharness/base/script.py
+++ b/testing/mozharness/mozharness/base/script.py
@@ -851,17 +851,18 @@ class ScriptMixin(PlatformMixin):
                              sleeptime, level=log_level)
                     time.sleep(sleeptime)
                     sleeptime = sleeptime * 2
                     if sleeptime > max_sleeptime:
                         sleeptime = max_sleeptime
 
     def query_env(self, partial_env=None, replace_dict=None,
                   purge_env=(),
-                  set_self_env=None, log_level=DEBUG):
+                  set_self_env=None, log_level=DEBUG,
+                  avoid_host_env=False):
         """ Environment query/generation method.
         The default, self.query_env(), will look for self.config['env']
         and replace any special strings in there ( %(PATH)s ).
         It will then store it as self.env for speeding things up later.
 
         If you specify partial_env, partial_env will be used instead of
         self.config['env'], and we don't save self.env as it's a one-off.
 
@@ -873,29 +874,34 @@ class ScriptMixin(PlatformMixin):
                 environment variables.
             purge_env (list): environment names to delete from the final
                 environment dictionary.
             set_self_env (boolean, optional): whether or not the environment
                 variables dictionary should be copied to `self`.
                 Defaults to True.
             log_level (str, optional): log level name to use on normal operation.
                 Defaults to `DEBUG`.
+            avoid_host_env (boolean, optional): if set to True, we will not use
+                any environment variables set on the host except PATH.
+                Defaults to False.
 
         Returns:
             dict: environment variables names with their values.
         """
         if partial_env is None:
             if self.env is not None:
                 return self.env
             partial_env = self.config.get('env', None)
             if partial_env is None:
                 partial_env = {}
             if set_self_env is None:
                 set_self_env = True
-        env = os.environ.copy()
+
+        env = {'PATH': os.environ['PATH']} if avoid_host_env else os.environ.copy()
+
         default_replace_dict = self.query_abs_dirs()
         default_replace_dict['PATH'] = os.environ['PATH']
         if not replace_dict:
             replace_dict = default_replace_dict
         else:
             for key in default_replace_dict:
                 if key not in replace_dict:
                     replace_dict[key] = default_replace_dict[key]
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -304,16 +304,17 @@ class BuildOptionParser(object):
     # add to this list and you can automagically do things like
     # --custom-build-variant-cfg asan
     # and the script will pull up the appropriate path for the config
     # against the current platform and bits.
     # *It will warn and fail if there is not a config for the current
     # platform/bits
     build_variants = {
         'asan': 'builds/releng_sub_%s_configs/%s_asan.py',
+        'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py',
         'b2g-debug': 'b2g/releng_sub_%s_configs/%s_debug.py',
         'debug': 'builds/releng_sub_%s_configs/%s_debug.py',
         'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py',
         'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py',
         'mulet': 'builds/releng_sub_%s_configs/%s_mulet.py',
         'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
         'graphene': 'builds/releng_sub_%s_configs/%s_graphene.py',
         'horizon': 'builds/releng_sub_%s_configs/%s_horizon.py',
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
@@ -60,24 +60,22 @@ class FirefoxUITests(VCSToolsScript, Vir
             config_options=self.config_options,
             require_config_file=True,
             config=default_config,
             all_actions=all_actions,
             **kwargs
         )
 
         self.firefox_ui_repo = self.config['firefox_ui_repo']
+        self.firefox_ui_branch = self.config.get('firefox_ui_branch')
 
-        if 'checkout' in self.actions:
-            try:
-                self.firefox_ui_branch = self.config['firefox_ui_branch']
-            except:
-                self.fatal(
-                    'Please specify --firefox-ui-branch. Valid values can be found '
-                    'in here https://github.com/mozilla/firefox-ui-tests/branches')
+        if not self.firefox_ui_branch:
+            self.fatal(
+                'Please specify --firefox-ui-branch. Valid values can be found '
+                'in here https://github.com/mozilla/firefox-ui-tests/branches')
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxUITests, self).query_abs_dirs()
 
         dirs = {
             'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -382,16 +382,17 @@ 2. running via buildbot and running the 
     def _download_test_packages(self, suite_categories, target_unzip_dirs):
         # Some platforms define more suite categories/names than others.
         # This is a difference in the convention of the configs more than
         # to how these tests are run, so we pave over these differences here.
         aliases = {
             'robocop': 'mochitest',
             'mochitest-chrome': 'mochitest',
             'mochitest-gl': 'mochitest',
+            'webapprt': 'mochitest',
             'jsreftest': 'reftest',
             'crashtest': 'reftest',
         }
         suite_categories = [aliases.get(name, name) for name in suite_categories]
 
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
--- a/testing/mozharness/scripts/b2g_emulator_unittest.py
+++ b/testing/mozharness/scripts/b2g_emulator_unittest.py
@@ -138,16 +138,17 @@ class B2GEmulatorTest(TestingMixin, VCSM
         )
 
         # these are necessary since self.config is read only
         c = self.config
         self.adb_path = c.get('adb_path', self._query_adb())
         self.installer_url = c.get('installer_url')
         self.installer_path = c.get('installer_path')
         self.test_url = c.get('test_url')
+        self.test_packages_url = c.get('test_packages_url')
         self.test_manifest = c.get('test_manifest')
         self.busybox_path = None
 
     # TODO detect required config items and fail if not set
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -2,17 +2,16 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """desktop_l10n.py
 
 This script manages Desktop repacks for nightly builds.
-In this version, a single partial is supported.
 """
 import os
 import re
 import sys
 import shlex
 import logging
 
 import subprocess
@@ -40,17 +39,17 @@ from mozharness.base.log import FATAL
 
 try:
     import simplejson as json
     assert json
 except ImportError:
     import json
 
 
-# needed by summarize
+# needed by _map
 SUCCESS = 0
 FAILURE = 1
 
 # when running get_output_form_command, pymake has some extra output
 # that needs to be filtered out
 PyMakeIgnoreList = [
     re.compile(r'''.*make\.py(?:\[\d+\])?: Entering directory'''),
     re.compile(r'''.*make\.py(?:\[\d+\])?: Leaving directory'''),
@@ -127,22 +126,16 @@ class DesktopSingleLocale(LocalesMixin, 
          "help": "Override the user repo path for all repos"}
     ], [
         ['--release-config-file', ],
         {"action": "store",
          "dest": "release_config_file",
          "type": "string",
          "help": "Specify the release config file to use"}
     ], [
-        ['--keystore', ],
-        {"action": "store",
-         "dest": "keystore",
-         "type": "string",
-         "help": "Specify the location of the signing keystore"}
-    ], [
         ['--this-chunk', ],
         {"action": "store",
          "dest": "this_locale_chunk",
          "type": "int",
          "help": "Specify which chunk of locales to run"}
     ], [
         ['--total-chunks', ],
         {"action": "store",
@@ -170,25 +163,20 @@ class DesktopSingleLocale(LocalesMixin, 
                 "funsize-props",
                 "submit-to-balrog",
                 "summary",
             ],
             'config': {
                 "buildbot_json_path": "buildprops.json",
                 "ignore_locales": ["en-US"],
                 "locales_dir": "browser/locales",
-                "previous_mar_dir": "previous",
-                "current_mar_dir": "current",
                 "update_mar_dir": "dist/update",
-                "previous_mar_filename": "previous.mar",
-                "current_work_mar_dir": "current.work",
                 "buildid_section": "App",
                 "buildid_option": "BuildID",
                 "application_ini": "application.ini",
-                "unpack_script": "tools/update-packaging/unwrap_full_update.pl",
                 "log_name": "single_locale",
                 "clobber_file": 'CLOBBER',
                 "appName": "Firefox",
                 "hashType": "sha512",
                 "taskcluster_credentials_file": "oauth.txt",
                 'virtualenv_modules': [
                     'requests==2.2.1',
                     'PyHawk-with-a-single-extra-commit==0.1.5',
@@ -212,17 +200,16 @@ class DesktopSingleLocale(LocalesMixin, 
         self.bootstrap_env = None
         self.upload_env = None
         self.revision = None
         self.version = None
         self.upload_urls = {}
         self.locales_property = {}
         self.l10n_dir = None
         self.package_urls = {}
-        self.partials = {}
         self.pushdate = None
         # Each locale adds its list of files to upload_files - some will be
         # duplicates (like the mar binaries), so we use a set to prune those
         # when uploading to taskcluster.
         self.upload_files = set()
 
         if 'mock_target' in self.config:
             self.enable_mock()
@@ -381,17 +368,17 @@ class DesktopSingleLocale(LocalesMixin, 
 
         if 'upload_env_extra' in config:
             for extra in config['upload_env_extra']:
                 upload_env[extra] = config['upload_env_extra'][extra]
         self.upload_env = upload_env
         return self.upload_env
 
     def query_l10n_env(self):
-        l10n_env = self._query_upload_env()
+        l10n_env = self._query_upload_env().copy()
         # both upload_env and bootstrap_env define MOZ_SIGN_CMD
         # the one from upload_env is taken from os.environ, the one from
         # bootstrap_env is set with query_moz_sign_cmd()
         # we need to use the value provided my query_moz_sign_cmd or make upload
         # will fail (signtool.py path is wrong)
         l10n_env.update(self.query_bootstrap_env())
         return l10n_env
 
@@ -462,40 +449,30 @@ class DesktopSingleLocale(LocalesMixin, 
                     discard = True
                     continue
             if not discard:
                 output.append(line.strip())
         output = " ".join(output).strip()
         self.info('echo-variable-%s: %s' % (variable, output))
         return output
 
-    def query_base_package_name(self, locale):
-        """Gets the package name from the objdir.
-        Only valid after setup is run.
-        """
-        # optimization:
-        # replace locale with %(locale)s
-        # and store its values.
-        args = ['AB_CD=%s' % locale]
-        return self._query_make_variable('PACKAGE', make_args=args)
-
     def query_version(self):
         """Gets the version from the objdir.
         Only valid after setup is run."""
         if self.version:
             return self.version
         config = self.config
         if config.get('release_config_file'):
             release_config = self.query_release_config()
             self.version = release_config['version']
         else:
             self.version = self._query_make_variable("MOZ_APP_VERSION")
         return self.version
 
-    def summarize(self, func, items):
+    def _map(self, func, items):
         """runs func for any item in items, calls the add_failure() for each
            error. It assumes that function returns 0 when successful.
            returns a two element tuple with (success_count, total_count)"""
         success_count = 0
         total_count = len(items)
         name = func.__name__
         for item in items:
             result = func(item)
@@ -565,28 +542,21 @@ class DesktopSingleLocale(LocalesMixin, 
         self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
                                 tag_override=config.get('tag_override'))
         self.pull_locale_source()
 
     def setup(self):
         """setup step"""
         dirs = self.query_abs_dirs()
         self._run_tooltool()
-        # create dirs
-        self._create_base_dirs()
-        # copy the mozconfig file
         self._copy_mozconfig()
-        # run mach conigure
         self._mach_configure()
         self._run_make_in_config_dir()
-        # download the en-us binary
         self.make_wget_en_US()
-        # ...and unpack it
-        self.make_unpack()
-        # get the revision
+        self.make_unpack_en_US()
         revision = self._query_revision()
         if not revision:
             self.fatal("Can't determine revision!")
         #  TODO do this through VCSMixin instead of hardcoding hg
         #  self.update(dest=dirs["abs_mozilla_dir"], revision=revision)
         hg = self.query_exe("hg")
         self.run_command([hg, "update", "-r", revision],
                          cwd=dirs["abs_mozilla_dir"],
@@ -599,17 +569,17 @@ class DesktopSingleLocale(LocalesMixin, 
         _clobber_file = self._clobber_file()
         if os.path.exists(_clobber_file):
             self._touch_file(_clobber_file)
         # and again...
         # thanks to the last hg update, we can be on different firefox 'version'
         # than the one on default,
         self._mach_configure()
         self._run_make_in_config_dir()
-        self.make_wget_en_US()
+        self.download_mar_tools()
 
     def _run_make_in_config_dir(self):
         """this step creates nsinstall, needed my make_wget_en_US()
         """
         dirs = self.query_abs_dirs()
         config_dir = os.path.join(dirs['abs_objdir'], 'config')
         env = self.query_bootstrap_env()
         return self._make(target=['export'], cwd=config_dir, env=env)
@@ -699,29 +669,17 @@ class DesktopSingleLocale(LocalesMixin, 
         """runs make and returns the output of the command"""
         make = self._get_make_executable()
         return self.get_output_from_command(make + target,
                                             cwd=cwd,
                                             env=env,
                                             silent=True,
                                             halt_on_failure=halt_on_failure)
 
-    def make_export(self, buildid):
-        """calls make export <buildid>"""
-        #  is it really needed ???
-        if buildid is None:
-            self.info('buildid is set to None, skipping make export')
-            return
-        dirs = self.query_abs_dirs()
-        cwd = dirs['abs_locales_dir']
-        env = self.query_bootstrap_env()
-        target = ["export", 'MOZ_BUILD_DATE=%s' % str(buildid)]
-        return self._make(target=target, cwd=cwd, env=env)
-
-    def make_unpack(self):
+    def make_unpack_en_US(self):
         """wrapper for make unpack"""
         config = self.config
         dirs = self.query_abs_dirs()
         env = self.query_bootstrap_env()
         cwd = os.path.join(dirs['abs_objdir'], config['locales_dir'])
         return self._make(target=["unpack"], cwd=cwd, env=env)
 
     def make_wget_en_US(self):
@@ -803,215 +761,41 @@ class DesktopSingleLocale(LocalesMixin, 
         env['L10NBASEDIR'] = self.l10n_dir
         dirs = self.query_abs_dirs()
         cwd = os.path.join(dirs['abs_locales_dir'])
         target = ["installers-%s" % locale,
                   "LOCALE_MERGEDIR=%s" % env["LOCALE_MERGEDIR"], ]
         return self._make(target=target, cwd=cwd,
                           env=env, halt_on_failure=False)
 
-    def generate_complete_mar(self, locale):
-        """creates a complete mar file"""
-        self.info('generating complete mar for locale %s' % (locale))
-        config = self.config
-        dirs = self.query_abs_dirs()
-        self._create_mar_dirs()
-        self.download_mar_tools()
-        package_basedir = os.path.join(dirs['abs_objdir'],
-                                       config['package_base_dir'])
-        dist_dir = os.path.join(dirs['abs_objdir'], 'dist')
-        env = self.query_l10n_env()
-        cmd = os.path.join(dirs['abs_objdir'], config['update_packaging_dir'])
-        cmd = ['-C', cmd, 'full-update', 'AB_CD=%s' % locale,
-               'PACKAGE_BASE_DIR=%s' % package_basedir,
-               'DIST=%s' % dist_dir,
-               'MOZ_PKG_PRETTYNAMES=']
-        return_code = self._make(target=cmd,
-                                 cwd=dirs['abs_mozilla_dir'],
-                                 env=env)
-        return return_code
-
-    def _copy_complete_mar(self, locale):
-        """copies the file generated by generate_complete_mar() into the right
-           place"""
-        # complete mar file is created in obj-l10n/dist/update
-        # but we need it in obj-l10n-dist/current, let's copy it
-        current_mar_file = self._current_mar_filename(locale)
-        src_file = self.localized_marfile(locale)
-        dst_file = os.path.join(self._current_mar_dir(), current_mar_file)
-        if os.path.exists(dst_file):
-            self.info('removing %s' % (dst_file))
-            os.remove(dst_file)
-        # copyfile returns None on success but we need 0 if the operation was
-        # successful
-        if self.copyfile(src_file, dst_file) is None:
-            # success
-            return SUCCESS
-        return FAILURE
-
     def repack_locale(self, locale):
         """wraps the logic for comapare locale, make installers and generate
-           partials"""
-        # get mar tools
-        self.download_mar_tools()
-        # remove current/previous/... directories
-        self._delete_mar_dirs()
-        self._create_mar_dirs()
+           complete updates."""
 
         if self.run_compare_locales(locale) != SUCCESS:
             self.error("compare locale %s failed" % (locale))
             return FAILURE
 
-        # compare locale succeded, let's run make installers
+        # compare locale succeeded, run make installers
         if self.make_installers(locale) != SUCCESS:
             self.error("make installers-%s failed" % (locale))
             return FAILURE
 
-        if self._requires_generate_mar('complete', locale):
-            if self.generate_complete_mar(locale) != SUCCESS:
-                self.error("generate complete %s mar failed" % (locale))
-                return FAILURE
-        # copy the complete mar file where generate_partial_updates expects it
-        if self._copy_complete_mar(locale) != SUCCESS:
-            self.error("copy_complete_mar failed!")
-            return FAILURE
-
-        if self._requires_generate_mar('partial', locale):
-            if self.generate_partial_updates(locale) != 0:
-                self.error("generate partials %s failed" % (locale))
-                return FAILURE
         if self.get_upload_files(locale):
             self.error("failed to get list of files to upload for locale %s" % (locale))
             return FAILURE
         # now try to upload the artifacts
         if self.make_upload(locale):
             self.error("make upload for locale %s failed!" % (locale))
             return FAILURE
         return SUCCESS
 
-    def _requires_generate_mar(self, mar_type, locale):
-        # Bug 1136750 - Partial mar is generated but not uploaded
-        # do not use mozharness for complete/partial updates, testing
-        # a full cycle of intree only updates generation
-        return False
-
-        generate = True
-        if mar_type == 'complete':
-            complete_filename = self.localized_marfile(locale)
-            if os.path.exists(complete_filename):
-                self.info('complete mar, already exists: %s' % (complete_filename))
-                generate = False
-            else:
-                self.info('complete mar, does not exist: %s' % (complete_filename))
-        elif mar_type == 'partial':
-            if not self.has_partials():
-                self.info('partials are disabled: enable_partials == False')
-                generate = False
-            else:
-                partial_filename = self._query_partial_mar_filename(locale)
-                if os.path.exists(partial_filename):
-                    self.info('partial mar, already exists: %s' % (partial_filename))
-                    generate = False
-                else:
-                    self.info('partial mar, does not exist: %s' % (partial_filename))
-        else:
-            self.fatal('unknown mar_type: %s' % mar_type)
-        return generate
-
-    def has_partials(self):
-        """returns True if partials are enabled, False elsewhere"""
-        config = self.config
-        return config["enable_partials"]
-
     def repack(self):
         """creates the repacks and udpates"""
-        self.summarize(self.repack_locale, self.query_locales())
-
-    def localized_marfile(self, locale):
-        """returns the localized mar file name"""
-        config = self.config
-        version = self.query_version()
-        localized_mar = config['localized_mar'] % {'version': version,
-                                                   'locale': locale}
-        localized_mar = os.path.join(self._mar_dir('update_mar_dir'),
-                                     localized_mar)
-        return localized_mar
-
-    def generate_partial_updates(self, locale):
-        """create partial updates for locale"""
-        # clean up any left overs from previous locales
-        # remove current/ current.work/ previous/ directories
-        self.info('creating partial update for locale: %s' % (locale))
-        # and recreate current/ previous/
-        self._create_mar_dirs()
-        # download mar and mbsdiff executables, we need them later...
-        self.download_mar_tools()
-
-        # unpack current mar file
-        current_marfile = self._current_mar_filename(locale)
-        current_mar_dir = self._current_mar_dir()
-        result = self._unpack_mar(current_marfile, current_mar_dir)
-        if result != SUCCESS:
-            self.error('failed to unpack %s to %s' % (current_marfile,
-                                                      current_mar_dir))
-            return result
-        # current mar file unpacked, remove it
-        self.rmtree(current_marfile)
-        # partial filename
-        previous_mar_dir = self._previous_mar_dir()
-        previous_mar_buildid = self.get_buildid_from_mar_dir(previous_mar_dir)
-        partial_filename = self._query_partial_mar_filename(locale)
-        if locale not in self.package_urls:
-            self.package_urls[locale] = {}
-        self.package_urls[locale]['partial_filename'] = partial_filename
-        self.package_urls[locale]['previous_buildid'] = previous_mar_buildid
-        self._delete_pgc_files()
-        result = self.do_incremental_update(previous_mar_dir, current_mar_dir,
-                                            partial_filename)
-        if result == 0:
-            # incremental updates succeded
-            # prepare partialInfo for balrog submission
-            partialInfo = {}
-            p_marfile = self._query_partial_mar_filename(locale)
-            partialInfo['from_buildid'] = previous_mar_buildid
-            partialInfo['size'] = self.query_filesize(p_marfile)
-            partialInfo['hash'] = self.query_sha512sum(p_marfile)
-            # url will be available only after make upload
-            # self._query_partial_mar_url(locale)
-            # and of course we need to generate partials befrore uploading them
-            partialInfo['url'] = None
-            if locale not in self.partials:
-                self.partials[locale] = []
-
-            # append partialInfo
-            self.partials[locale].append(partialInfo)
-        return result
-
-    def _partial_filename(self, locale):
-        config = self.config
-        version = self.query_version()
-        # download the previous partial, if needed
-        self._create_mar_dirs()
-        # download mar and mbsdiff executables
-        self.download_mar_tools()
-        # get the previous mar file
-        previous_marfile = self._get_previous_mar(locale)
-        # and unpack it
-        previous_mar_dir = self._previous_mar_dir()
-        result = self._unpack_mar(previous_marfile, previous_mar_dir)
-        if result != SUCCESS:
-            self.error('failed to unpack %s to %s' % (previous_marfile,
-                                                      previous_mar_dir))
-            return result
-        previous_mar_buildid = self.get_buildid_from_mar_dir(previous_mar_dir)
-        current_mar_buildid = self._query_buildid()
-        return config['partial_mar'] % {'version': version,
-                                        'locale': locale,
-                                        'from_buildid': current_mar_buildid,
-                                        'to_buildid': previous_mar_buildid}
+        self._map(self.repack_locale, self.query_locales())
 
     def _query_objdir(self):
         """returns objdir name from configuration"""
         return self.config['objdir']
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
@@ -1071,17 +855,17 @@ class DesktopSingleLocale(LocalesMixin, 
         self.set_buildbot_property("hashType", hashType)
         self.set_buildbot_property("platform", platform)
         # values common to the current repacks
         self.set_buildbot_property("buildid", self._query_buildid())
         self.set_buildbot_property("appVersion", self.query_version())
 
         # submit complete mar to balrog
         # clean up buildbot_properties
-        self.summarize(self.submit_repack_to_balrog, self.query_locales())
+        self._map(self.submit_repack_to_balrog, self.query_locales())
 
     def submit_repack_to_balrog(self, locale):
         """submit a single locale to balrog"""
         # check if locale has been uploaded, if not just return a FAILURE
         if locale not in self.package_urls:
             self.error("%s is not present in package_urls. Did you run make upload?" % locale)
             return FAILURE
 
@@ -1112,29 +896,16 @@ class DesktopSingleLocale(LocalesMixin, 
             result = self.submit_balrog_updates()
             self.info("balrog return code: %s" % (result))
             if result == 0:
                 ret = SUCCESS
         except Exception as error:
             self.error("submit repack to balrog failed: %s" % (str(error)))
         return ret
 
-    def _get_partialInfo(self, locale):
-        """we can have 0, 1 or many partials, this method returns the partialInfo
-           needed by balrog submitter"""
-
-        if locale not in self.partials or not self.has_partials():
-            return []
-
-        # we have only a single partial for now
-        # MakeUploadOutputParser can match a single parser
-        partial_url = self.package_urls[locale]["partialMarUrl"]
-        self.partials[locale][0]["url"] = partial_url
-        return self.partials[locale]
-
     def _query_complete_mar_filename(self, locale):
         """returns the full path to a localized complete mar file"""
         config = self.config
         version = self.query_version()
         complete_mar_name = config['localized_mar'] % {'version': version,
                                                        'locale': locale}
         return os.path.join(self._update_mar_dir(), complete_mar_name)
 
@@ -1146,159 +917,30 @@ class DesktopSingleLocale(LocalesMixin, 
         if "completeMarUrl" in self.package_urls[locale]:
             return self.package_urls[locale]["completeMarUrl"]
         # url = self.config.get("update", {}).get("mar_base_url")
         # if url:
         #    url += os.path.basename(self.query_marfile_path())
         #    return url.format(branch=self.query_branch())
         self.fatal("Couldn't find complete mar url in config or package_urls")
 
-    def _query_partial_mar_url(self, locale):
-        """returns partial mar url"""
-        try:
-            return self.package_urls[locale]["partialMarUrl"]
-        except KeyError:
-            msg = "Couldn't find package_urls: %s %s" % (locale, self.package_urls)
-            self.error("package_urls: %s" % (self.package_urls))
-            self.fatal(msg)
-
-    def _query_partial_mar_filename(self, locale):
-        """returns the full path to a partial, it returns a valid path only
-           after make upload"""
-        partial_mar_name = self._partial_filename(locale)
-        return os.path.join(self._update_mar_dir(), partial_mar_name)
-
-    def _query_previous_mar_buildid(self, locale):
-        """return the partial mar buildid,
-        this method returns a valid buildid only after generate partials,
-        it raises an exception when buildid is not available
-        """
-        try:
-            return self.package_urls[locale]["previous_buildid"]
-        except KeyError:
-            self.error("no previous mar buildid")
-            raise
-
-    def _delete_pgc_files(self):
-        """deletes pgc files"""
-        for directory in (self._previous_mar_dir(),
-                          self._current_mar_dir()):
-            for pcg_file in self._pgc_files(directory):
-                self.info("removing %s" % pcg_file)
-                self.rmtree(pcg_file)
-
-    def _current_mar_url(self, locale):
-        """returns current mar url"""
-        config = self.config
-        base_url = config['current_mar_url']
-        return "/".join((base_url, self._current_mar_name(locale)))
-
-    def _previous_mar_url(self, locale):
-        """returns the url for previous mar"""
-        config = self.config
-        base_url = config['previous_mar_url']
-        return "/".join((base_url, self._localized_mar_name(locale)))
-
-    def _get_previous_mar(self, locale):
-        """downloads the previous mar file"""
-        self.mkdir_p(self._previous_mar_dir())
-        self.download_file(self._previous_mar_url(locale),
-                           self._previous_mar_filename(locale))
-        return self._previous_mar_filename(locale)
-
-    def _current_mar_name(self, locale):
-        """returns current mar file name"""
-        config = self.config
-        version = self.query_version()
-        return config["current_mar_filename"] % {'version': version,
-                                                 'locale': locale, }
-
-    def _localized_mar_name(self, locale):
-        """returns localized mar name"""
-        config = self.config
-        version = self.query_version()
-        return config["localized_mar"] % {'version': version, 'locale': locale}
-
-    def _previous_mar_filename(self, locale):
-        """returns the complete path to previous.mar"""
-        config = self.config
-        return os.path.join(self._previous_mar_dir(),
-                            config['previous_mar_filename'])
-
-    def _current_mar_filename(self, locale):
-        """returns the complete path to current.mar"""
-        return os.path.join(self._current_mar_dir(), self._current_mar_name(locale))
-
-    def _create_mar_dirs(self):
-        """creates mar directories: previous/ current/"""
-        for directory in (self._previous_mar_dir(),
-                          self._current_mar_dir()):
-            self.info("creating: %s" % directory)
-            self.mkdir_p(directory)
-
-    def _delete_mar_dirs(self):
-        """delete mar directories: previous, current"""
-        for directory in (self._previous_mar_dir(),
-                          self._current_mar_dir(),
-                          self._current_work_mar_dir()):
-            self.info("deleting: %s" % directory)
-            if os.path.exists(directory):
-                self.rmtree(directory)
-
-    def _unpack_script(self):
-        """unpack script full path"""
-        config = self.config
-        dirs = self.query_abs_dirs()
-        return os.path.join(dirs['abs_mozilla_dir'], config['unpack_script'])
-
-    def _previous_mar_dir(self):
-        """returns the full path of the previous/ directory"""
-        return self._mar_dir('previous_mar_dir')
-
-    def _abs_dist_dir(self):
-        """returns the full path to abs_objdir/dst"""
-        dirs = self.query_abs_dirs()
-        return os.path.join(dirs['abs_objdir'], 'dist')
-
     def _update_mar_dir(self):
         """returns the full path of the update/ directory"""
         return self._mar_dir('update_mar_dir')
 
-    def _current_mar_dir(self):
-        """returns the full path of the current/ directory"""
-        return self._mar_dir('current_mar_dir')
-
-    def _current_work_mar_dir(self):
-        """returns the full path to current.work"""
-        return self._mar_dir('current_work_mar_dir')
-
     def _mar_binaries(self):
         """returns a tuple with mar and mbsdiff paths"""
         config = self.config
         return (config['mar'], config['mbsdiff'])
 
     def _mar_dir(self, dirname):
         """returns the full path of dirname;
             dirname is an entry in configuration"""
-        config = self.config
-        return os.path.join(self._get_objdir(), config.get(dirname))
-
-    def _get_objdir(self):
-        """returns full path to objdir"""
         dirs = self.query_abs_dirs()
-        return dirs['abs_objdir']
-
-    def _pgc_files(self, basedir):
-        """returns a list of .pcf files in basedir"""
-        pgc_files = []
-        for dirpath, dirnames, filenames in os.walk(basedir):
-            for pgc in filenames:
-                if pgc.endswith('.pgc'):
-                    pgc_files.append(os.path.join(dirpath, pgc))
-        return pgc_files
+        return os.path.join(dirs['abs_objdir'], self.config[dirname])
 
     # TODO: replace with ToolToolMixin
     def _get_tooltool_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
             fn = self.config['tooltool_authentication_file']
         elif self._is_windows():
@@ -1331,26 +973,19 @@ class DesktopSingleLocale(LocalesMixin, 
         ]
         cmd.extend(config['tooltool_script'])
         auth_file = self._get_tooltool_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
 
-    def _create_base_dirs(self):
-        config = self.config
-        dirs = self.query_abs_dirs()
-        for make_dir in config.get('make_dirs', []):
-            dirname = os.path.join(dirs['abs_objdir'], make_dir)
-            self.mkdir_p(dirname)
-
     def funsize_props(self):
-        """writes funsize info into buildprops.json"""
-        # see bug
+        """Set buildbot properties required to trigger funsize tasks
+         responsible to generate partial updates for successfully generated locales"""
         funsize_info = {
             'locales': self.query_locales(),
             'branch': self.config['branch'],
             'appName': self.config['appName'],
             'platform': self.config['platform'],
         }
         self.info('funsize info: %s' % funsize_info)
         self.set_buildbot_property('funsize_info', json.dumps(funsize_info),
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -28,17 +28,17 @@ from mozharness.mozilla.blob_upload impo
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.buildbot import TBPL_WARNING
 
-SUITE_CATEGORIES = ['cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
+SUITE_CATEGORIES = ['cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt']
 
 
 # DesktopUnittest {{{1
 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
     config_options = [
         [['--mochitest-suite', ], {
             "action": "extend",
             "dest": "specified_mochitest_suites",
--- a/testing/mozharness/scripts/firefox_ui_updates.py
+++ b/testing/mozharness/scripts/firefox_ui_updates.py
@@ -5,26 +5,28 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """firefox_ui_updates.py
 
 Author: Armen Zambrano G.
 """
 import copy
 import os
+import re
 import urllib
+import urllib2
 import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import INFO
 from mozharness.base.script import PreScriptAction
 from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests
 
+INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz')
 
 class FirefoxUIUpdates(FirefoxUITests):
     # This will be a list containing one item per release based on configs
     # from tools/release/updates/*cfg
     releases = None
     channel = None
     harness_extra_args = [
         [['--update-allow-mar-channel'], {
@@ -40,16 +42,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         [['--update-target-version'], {
             'dest': 'update_target_version',
             'help': 'Version of the updated build.',
         }],
         [['--update-target-buildid'], {
             'dest': 'update_target_buildid',
             'help': 'Build ID of the updated build',
         }],
+        [['--symbols-path=SYMBOLS_PATH'], {
+            'dest': 'symbols_path',
+            'help': 'absolute path to directory containing breakpad '
+                    'symbols, or the url of a zip file containing symbols.',
+        }],
     ]
 
 
     def __init__(self):
         config_options = [
             [['--tools-repo'], {
                 'dest': 'tools_repo',
                 'default': 'http://hg.mozilla.org/build/tools',
@@ -73,16 +80,20 @@ class FirefoxUIUpdates(FirefoxUITests):
                 'default': 1,
                 'help': 'Total chunks to dive the locales into.',
             }],
             [['--dry-run'], {
                 'dest': 'dry_run',
                 'default': False,
                 'help': 'Only show what was going to be tested.',
             }],
+            [["--build-number"], {
+                "dest": "build_number",
+                "help": "Build number of release, eg: 2",
+            }],
             # These are options when we don't use the releng update config file
             [['--installer-url'], {
                 'dest': 'installer_url',
                 'help': 'Point to an installer to download and test against.',
             }],
             [['--installer-path'], {
                 'dest': 'installer_path',
                 'help': 'Point to an installer to test against.',
@@ -94,31 +105,32 @@ class FirefoxUIUpdates(FirefoxUITests):
             all_actions=[
                 'purge-builds',
                 'clobber',
                 'checkout',
                 'create-virtualenv',
                 'determine-testing-configuration',
                 'run-tests',
             ],
+            append_env_variables_from_configs=True,
         )
 
         dirs = self.query_abs_dirs()
 
         if self.config.get('tools_tag') is None:
             # We want to make sure that anyone trying to reproduce a job will
             # is using the exact tools tag for reproducibility's sake
             self.fatal('Make sure to specify the --tools-tag')
 
         self.tools_repo = self.config['tools_repo']
         self.tools_tag = self.config['tools_tag']
 
         if self.config.get('update_verify_config'):
             self.updates_config_file = os.path.join(
-                dirs['tools_dir'], 'release', 'updates',
+                dirs['abs_tools_dir'], 'release', 'updates',
                 self.config['update_verify_config']
             )
         else:
             self.fatal('Make sure to specify --update-verify-config. '
                        'See under the directory release/updates in %s.' % self.tools_repo)
 
         self.installer_url = self.config.get('installer_url')
         self.installer_path = self.config.get('installer_path')
@@ -133,17 +145,17 @@ class FirefoxUIUpdates(FirefoxUITests):
 
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxUIUpdates, self).query_abs_dirs()
 
         dirs = {
-            'tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
+            'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
         }
 
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     def checkout(self):
         '''
@@ -153,17 +165,17 @@ class FirefoxUIUpdates(FirefoxUITests):
         We also checkout firefox_ui_tests and update to the right branch
         for it.
         '''
         super(FirefoxUIUpdates, self).checkout()
         dirs = self.query_abs_dirs()
 
         self.vcs_checkout(
             repo=self.tools_repo,
-            dest=dirs['tools_dir'],
+            dest=dirs['abs_tools_dir'],
             revision=self.tools_tag,
             vcs='hgtool'
         )
 
 
     def determine_testing_configuration(self):
         '''
         This method builds a testing matrix either based on an update verification
@@ -189,21 +201,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         of all locales (except for the most recent releases). A quick release has all locales,
         however, it misses the fields 'from' and 'ftp_server_from'.
         Both pairs of information complement each other but differ in such manner.
         '''
         if self.installer_url or self.installer_path:
             return
 
         dirs = self.query_abs_dirs()
-        assert os.path.exists(dirs['tools_dir']), \
+        assert os.path.exists(dirs['abs_tools_dir']), \
             "Without the tools/ checkout we can't use releng's config parser."
 
         # Import the config parser
-        sys.path.insert(1, os.path.join(dirs['tools_dir'], 'lib', 'python'))
+        sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python'))
         from release.updates.verify import UpdateVerifyConfig
 
         uvc = UpdateVerifyConfig()
         uvc.read(self.updates_config_file)
         self.channel = uvc.channel
 
         # Filter out any releases that are less than Gecko 38
         uvc.releases = [r for r in uvc.releases \
@@ -224,43 +236,81 @@ class FirefoxUIUpdates(FirefoxUITests):
         chunked_config = uvc.getChunk(
             chunks=int(self.config['total_chunks']),
             thisChunk=int(self.config['this_chunk'])
         )
 
         self.releases = chunked_config.releases
 
 
+    def _modify_url(self, rel_info):
+        # This is a temporary hack to find crash symbols. It should be replaced
+        # with something that doesn't make wild guesses about where symbol
+        # packages are.
+        # We want this:
+        # https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
+        # https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
+        installer_from = rel_info['from']
+        version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], installer_from)).group(1)
+
+        temp_from = installer_from.replace(version, '%s-candidates/build%s' % (version, self.config["build_number"]), 1).replace('releases', 'candidates')
+
+        return rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US'))
+
+
+    def _query_symbols_url(self, installer_url):
+        for suffix in INSTALLER_SUFFIXES:
+            if installer_url.endswith(suffix):
+                symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
+                continue
+
+        if symbols_url:
+            if not symbols_url.startswith('http'):
+                return symbols_url
+
+            try:
+                # Let's see if the symbols are available
+                return urllib2.urlopen(symbols_url)
+
+            except urllib2.HTTPError, e:
+                self.warning("%s - %s" % (str(e), symbols_url))
+                return None
+        else:
+            self.fatal("Can't figure out symbols_url from installer_url %s!" % installer_url)
+
+
     @PreScriptAction('run-tests')
     def _pre_run_tests(self, action):
         if self.releases is None and not (self.installer_url or self.installer_path):
-            self.critical('You need to call --determine-testing-configuration as well.')
-            exit(1)
+            self.fatal('You need to call --determine-testing-configuration as well.')
 
 
-    def _run_test(self, installer_path, update_channel=None, cleanup=True,
+    def _run_test(self, installer_path, symbols_url=None, update_channel=None, cleanup=True,
                   marionette_port=2828):
         '''
         All required steps for running the tests against an installer.
         '''
-        env = self.query_env()
         dirs = self.query_abs_dirs()
+        env = self.query_env(avoid_host_env=True)
         bin_dir = os.path.dirname(self.query_python_path())
         fx_ui_tests_bin = os.path.join(bin_dir, 'firefox-ui-update')
         gecko_log=os.path.join(dirs['abs_work_dir'], 'gecko.log')
 
         # Build the command
         cmd = [
             fx_ui_tests_bin,
             '--installer', installer_path,
             # Log to stdout until tests are stable.
             '--gecko-log=-',
             '--address=localhost:%s' % marionette_port,
         ]
 
+        if symbols_url:
+            cmd += ['--symbols-path', symbols_url]
+
         for arg in self.harness_extra_args:
             dest = arg[1]['dest']
             if dest in self.config:
                 cmd += [' '.join(arg[0]), self.config[dest]]
 
         if update_channel:
             cmd += ['--update-channel', update_channel]
 
@@ -271,17 +321,21 @@ class FirefoxUIUpdates(FirefoxUITests):
         # Return more output if we fail
         if return_code != 0:
             if os.path.exists(gecko_log):
                 contents = self.read_from_file(gecko_log, verbose=False)
                 self.warning('== Dumping gecko output ==')
                 self.warning(contents)
                 self.warning('== End of gecko output ==')
             else:
-                self.warning('No gecko.log was found: %s' % gecko_log)
+                # We're outputting to stdout with --gecko-log=- so there is not log to
+                # complaing about. Remove the commented line below when changing
+                # this behaviour.
+                # self.warning('No gecko.log was found: %s' % gecko_log)
+                pass
 
         if cleanup:
             for filepath in (installer_path, gecko_log):
                 if os.path.exists(filepath):
                     self.debug('Removing %s' % filepath)
                     os.remove(filepath)
 
         return return_code
@@ -292,57 +346,90 @@ class FirefoxUIUpdates(FirefoxUITests):
 
         if self.installer_url or self.installer_path:
             if self.installer_url:
                 self.installer_path = self.download_file(
                     self.installer_url,
                     parent_dir=dirs['abs_work_dir']
                 )
 
-            return self._run_test(self.installer_path, cleanup=False)
+            symbols_url = self._query_symbols_url(installer_url=self.installer_path)
+
+            return self._run_test(
+                installer_path=self.installer_path,
+                symbols_url=symbols_url,
+                cleanup=False
+            )
 
         else:
+            results = {}
+
             for rel_info in sorted(self.releases, key=lambda release: release['build_id']):
+                build_id = rel_info['build_id']
+                results[build_id] = {}
+
                 self.info('About to run %s %s - %s locales' % (
-                    rel_info['build_id'],
+                    build_id,
                     rel_info['from'],
                     len(rel_info['locales'])
                 ))
 
                 if self.config['dry_run']:
                     continue
 
                 # Each locale gets a fresh port to avoid address in use errors in case of
                 # tests that time out unexpectedly.
                 marionette_port = 2827
                 for locale in rel_info['locales']:
+                    self.info("Running %s %s" % (build_id, locale))
+
+                    # Safe temp hack to determine symbols URL from en-US build1 in the candidates dir
+                    ftp_candidates_installer_url = self._modify_url(rel_info)
+                    symbols_url = self._query_symbols_url(installer_url=ftp_candidates_installer_url)
+
                     # Determine from where to download the file
                     url = '%s/%s' % (
                         rel_info['ftp_server_from'],
                         urllib.quote(rel_info['from'].replace('%locale%', locale))
                     )
                     installer_path = self.download_file(
                         url=url,
                         parent_dir=dirs['abs_work_dir']
                     )
 
                     marionette_port += 1
-                    retcode = self._run_test(installer_path, self.channel,
-                                             marionette_port=marionette_port)
+
+                    retcode = self._run_test(
+                        installer_path=installer_path,
+                        symbols_url=symbols_url,
+                        update_channel=self.channel,
+                        marionette_port=marionette_port)
+
                     if retcode != 0:
                         self.warning('FAIL: firefox-ui-update has failed.' )
                         self.info('You can run the following command on the same machine to reproduce the issue:')
                         self.info('python scripts/firefox_ui_updates.py --cfg generic_releng_config.py '
                                   '--firefox-ui-branch %s --update-verify-config %s '
                                   '--tools-tag %s --installer-url %s '
                                   '--determine-testing-configuration --run-tests '
                                   % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
                         self.info('If you want to run this on your development machine:')
                         self.info('python scripts/firefox_ui_updates.py '
                                   '--firefox-ui-branch %s --update-verify-config %s '
                                   '--tools-tag %s --installer-url %s '
                                   '--cfg developer_config.py '
                                   % (self.firefox_ui_branch, self.updates_config_file, self.tools_tag, url))
 
+                    results[build_id][locale] = retcode
+
+            self.info("Firefox UI update tests failed locales:")
+            for build_id in sorted(results.keys()):
+                self.info(build_id)
+                failed_locales = []
+                for locale in sorted(results[build_id].keys()):
+                    if results[build_id][locale] != 0:
+                        failed_locales.append(locale)
+                self.info("  %s" % (', '.join(failed_locales)))
+
 
 if __name__ == '__main__':
     myScript = FirefoxUIUpdates()
     myScript.run_and_exit()
--- a/testing/mozharness/scripts/hazard_build.py
+++ b/testing/mozharness/scripts/hazard_build.py
@@ -267,24 +267,24 @@ class B2GHazardBuild(PurgeMixin, B2GBuil
         )
 
     def checkout_sources(self):
         self.make_source_dir()
         super(B2GHazardBuild, self).checkout_sources()
 
     def get_blobs(self):
         dirs = self.query_abs_dirs()
-        self.tooltool_fetch(self.query_compiler_manifest(),
-                            output_dir=dirs['abs_work_dir'])
-        self.tooltool_fetch(self.query_sixgill_manifest(),
-                            output_dir=dirs['abs_work_dir'])
+        self.tooltool_fetch(self.query_compiler_manifest(), "sh " + self.config['compiler_setup'],
+                            dirs['abs_work_dir'])
+        self.tooltool_fetch(self.query_sixgill_manifest(), "sh " + self.config['sixgill_setup'],
+                            dirs['abs_work_dir'])
         if not os.path.exists(dirs['target_compiler_base']):
             self.mkdir_p(dirs['target_compiler_base'])
-        self.tooltool_fetch(self.query_b2g_compiler_manifest(),
-                            output_dir=dirs['target_compiler_base'])
+        self.tooltool_fetch(self.query_b2g_compiler_manifest(), "sh " + self.config['compiler_setup'],
+                            dirs['target_compiler_base'])
 
     def clobber_shell(self):
         dirs = self.query_abs_dirs()
         self.rmtree(dirs['shell_objdir'])
 
     def configure_shell(self):
         self.enable_mock()
         dirs = self.query_abs_dirs()
--- a/testing/mozharness/scripts/spidermonkey_build.py
+++ b/testing/mozharness/scripts/spidermonkey_build.py
@@ -382,20 +382,20 @@ class SpidermonkeyBuild(MockMixin,
             self.do_checkout_source()
         except Exception as e:
             self.fatal("checkout failed: " + str(e), exit_code=RETRY)
 
     def get_blobs(self):
         work_dir = self.query_abs_dirs()['abs_work_dir']
         if not os.path.exists(work_dir):
             self.mkdir_p(work_dir)
-        self.tooltool_fetch(self.query_compiler_manifest(),
-                            output_dir=work_dir)
-        self.tooltool_fetch(self.query_sixgill_manifest(),
-                            output_dir=work_dir)
+        self.tooltool_fetch(self.query_compiler_manifest(), "sh " + self.config['compiler_setup'],
+                            work_dir)
+        self.tooltool_fetch(self.query_sixgill_manifest(), "sh " + self.config['sixgill_setup'],
+                            work_dir)
 
     def clobber_shell(self):
         self.analysis.clobber_shell(self)
 
     def configure_shell(self):
         self.enable_mock()
 
         try: