Bug 1380605 - Rebased patch from OpenH264 v1.6 (bug 1286533), r=catlee
authorJustin Wood <Callek@gmail.com>
Wed, 13 Jul 2016 09:05:44 -0400
changeset 429134 252f32919ce2a7ab1f5891189525bb31b4e27f79
parent 429133 6b1112ce0727a6ff4a8c1e239685a165bdbb0d89
child 429135 25413c572c6e618128d37f898da5261c4f9b5d3c
push id7761
push userjlund@mozilla.com
push dateFri, 15 Sep 2017 00:19:52 +0000
treeherdermozilla-beta@c38455951db4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscatlee
bugs1380605, 1286533, 64022
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1380605 - Rebased patch from OpenH264 v1.6 (bug 1286533), r=catlee Imported from https://reviewboard.mozilla.org/r/64022/diff/4#index_header MozReview-Commit-ID: B3NiWFvr2oR
testing/mozharness/configs/openh264/android-arm.py
testing/mozharness/configs/openh264/android-x86.py
testing/mozharness/configs/openh264/linux32.py
testing/mozharness/configs/openh264/linux64.py
testing/mozharness/configs/openh264/macosx32.py
testing/mozharness/configs/openh264/macosx64.py
testing/mozharness/configs/openh264/tooltool-manifests/android.manifest
testing/mozharness/configs/openh264/tooltool-manifests/linux.manifest
testing/mozharness/configs/openh264/tooltool-manifests/osx.manifest
testing/mozharness/configs/openh264/tooltool-manifests/win.manifest
testing/mozharness/configs/openh264/win32.py
testing/mozharness/configs/openh264/win64.py
testing/mozharness/external_tools/packagesymbols.py
testing/mozharness/mozharness/base/script.py
testing/mozharness/scripts/openh264_build.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/android-arm.py
@@ -0,0 +1,39 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'default_actions': [
+        'get-tooltool',
+        'checkout-sources',
+        'build',
+        # 'test',  # can't run android tests on linux hosts
+        'package',
+    ],
+
+    'tooltool_manifest_file': "android.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'arm',
+    'use_mock': True,
+    'mock_target': 'mozilla-centos6-x86_64',
+    'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+                      'libstdc++-devel.i686', 'zip', 'yasm',
+                      'mozilla-python27'],
+    'mock_files': [
+        ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+        ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+        ('/tools/tooltool.py', '/builds/tooltool.py'),
+    ],
+    'operating_system': 'android',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/android-x86.py
@@ -0,0 +1,40 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'default_actions': [
+        'get-tooltool',
+        'checkout-sources',
+        'build',
+        # 'test',  # can't run android tests on linux hosts
+        'package',
+    ],
+
+    'tooltool_manifest_file': "android.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'avoid_avx2': True,
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'x86',
+    'use_mock': True,
+    'mock_target': 'mozilla-centos6-x86_64',
+    'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+                      'libstdc++-devel.i686', 'zip', 'yasm',
+                      'mozilla-python27'],
+    'mock_files': [
+        ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+        ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+        ('/tools/tooltool.py', '/builds/tooltool.py'),
+    ],
+    'operating_system': 'android',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/linux32.py
@@ -0,0 +1,31 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'tooltool_manifest_file': "linux.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'x86',
+    'use_mock': True,
+    'avoid_avx2': True,
+    'mock_target': 'mozilla-centos6-x86_64',
+    'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+                      'libstdc++-devel.i686', 'zip', 'yasm',
+                      'mozilla-python27'],
+    'mock_files': [
+        ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+        ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+    ],
+    'operating_system': 'linux',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/linux64.py
@@ -0,0 +1,31 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'tooltool_manifest_file': "linux.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'x64',
+    'use_mock': True,
+    'avoid_avx2': True,
+    'mock_target': 'mozilla-centos6-x86_64',
+    'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+                      'libstdc++-devel.i686', 'zip', 'yasm',
+                      'mozilla-python27'],
+    'mock_files': [
+        ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+        ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+    ],
+    'operating_system': 'linux',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/macosx32.py
@@ -0,0 +1,21 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'tooltool_manifest_file': "osx.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'x86',
+    'use_yasm': True,
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/macosx64.py
@@ -0,0 +1,21 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'tooltool_manifest_file': "osx.manifest",
+    'tooltool_cache': "/builds/tooltool_cache",
+    'exes': {
+        'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+        'tooltool.py': "/builds/tooltool.py",
+        'python2.7': "/tools/python27/bin/python2.7",
+    },
+    'dump_syms_binary': 'dump_syms',
+    'arch': 'x64',
+    'use_yasm': True,
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/android.manifest
@@ -0,0 +1,17 @@
+[
+{
+"size": 357533004,
+"visibility": "internal",
+"digest": "9b24b6db45ca4f6418d18119e3d2dd48e2a352e977ebf4fd0b2a8c4103fd0735eacb0f7360ff3a2eda6777fdd1029325ca23b8a4c34542b3b1fa487ff633647c",
+"algorithm": "sha512",
+"filename": "android-ndk.tar.bz2",
+"unpack": true
+},
+{
+"size": 4086983, 
+"visibility": "public", 
+"digest": "f9b8922384b489cf4969b1d26d4fef93db5882dacadd9e679e728164220b237ce170e94f45e6a6ede1055f4d06097f994e5d0185f42cf90b9e3dc57dfc2d0005", 
+"algorithm": "sha512", 
+"filename": "dump_syms"
+}
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/linux.manifest
@@ -0,0 +1,9 @@
+[
+{
+"size": 4086983, 
+"visibility": "public", 
+"digest": "f9b8922384b489cf4969b1d26d4fef93db5882dacadd9e679e728164220b237ce170e94f45e6a6ede1055f4d06097f994e5d0185f42cf90b9e3dc57dfc2d0005", 
+"algorithm": "sha512", 
+"filename": "dump_syms"
+}
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/osx.manifest
@@ -0,0 +1,10 @@
+[
+{
+"size": 393368, 
+"visibility": "public", 
+"digest": "dcbd679a57ac3a1bc20df45cd8770b462d2c430d1c5fb85366c02904422824ed4193a66d936fa9d21fb49190f5915f9b24aad16e3eb8b7c9a90d09c93067982f", 
+"algorithm": "sha512", 
+"filename": "dump_syms"
+}
+]
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/win.manifest
@@ -0,0 +1,10 @@
+[
+{
+"size": 51200, 
+"visibility": "public", 
+"digest": "3071d2c22325c6f952d99491005dce85887ebd6a7ec52a91af157cf6f1ef4453234489d1269f7f92c739e08fd8a747d1ea2d6a6cf13cc9818a2f99302b4eb970", 
+"algorithm": "sha512", 
+"filename": "dump_syms_vc1800.exe"
+}
+]
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/win32.py
@@ -0,0 +1,37 @@
+import sys
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+VSPATH = 'C:/tools/vs2013'
+config = {
+   'tooltool_manifest_file': "win.manifest",
+   'exes': {
+       'gittool.py': [sys.executable, os.path.join(external_tools_path, 'gittool.py')],
+       'python2.7': 'c:\\mozilla-build\\python27\\python2.7.exe',
+       'tooltool.py': [sys.executable, "c:\\mozilla-build\\tooltool.py"],
+   },
+   'dump_syms_binary': 'dump_syms_vc1800.exe',
+   'arch': 'x86',
+   'use_yasm': True,
+   'operating_system': 'msvc',
+   'partial_env': {
+       'PATH': '%s;%s;%s;%s;%s' % (
+           'c:/Program Files (x86)/Windows Kits/8.1/bin/x86;{_VSPATH}/Common7/IDE;{_VSPATH}/VC/BIN/amd64_x86;{_VSPATH}/VC/BIN/amd64;{_VSPATH}/Common7/Tools;{_VSPATH}/VC/VCPackages;c:/mozilla-build/moztools'.format(_VSPATH=VSPATH),
+           'c:/windows/Microsoft.NET/Framework/v3.5;c:/windows/Microsoft.NET/Framework/v4.0.30319',
+           os.environ['PATH'],
+           'C:\\mozilla-build\\Git\\bin',
+           'C:\\mozilla-build\\svn-win32-1.6.3\\bin',
+       ),
+       'WIN32_REDIST_DIR': '{_VSPATH}/VC/redist/x86/Microsoft.VC120.CRT'.format(_VSPATH=VSPATH),
+       'INCLUDE': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\shared;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\um;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl\\wrappers;{_VSPATH}\\vc\\include;{_VSPATH}\\vc\\atlmfc\\include;c:\\tools\\sdks\\dx10\\include'.format(_VSPATH=VSPATH),
+       'LIB': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x86;{_VSPATH}\\vc\\lib;{_VSPATH}\\vc\\atlmfc\\lib;c:\\tools\\sdks\\dx10\\lib'.format(_VSPATH=VSPATH),
+       'LIBPATH': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x86;{_VSPATH}\\vc\\lib;{_VSPATH}\\vc\\atlmfc\\lib;c:\\tools\\sdks\\dx10\\lib'.format(_VSPATH=VSPATH),
+       'WINDOWSSDKDIR': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\',
+   },
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/win64.py
@@ -0,0 +1,37 @@
+import sys
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+VSPATH = 'C:/tools/vs2013'
+config = {
+   'tooltool_manifest_file': "win.manifest",
+   'exes': {
+        'gittool.py': [sys.executable, os.path.join(external_tools_path, 'gittool.py')],
+        'python2.7': 'c:\\mozilla-build\\python27\\python2.7.exe',
+        'tooltool.py': [sys.executable, "c:\\mozilla-build\\tooltool.py"],
+   },
+   'dump_syms_binary': 'dump_syms_vc1800.exe',
+   'arch': 'x64',
+   'use_yasm': True,
+   'operating_system': 'msvc',
+   'partial_env': {
+       'PATH': '%s;%s;%s;%s;%s' % (
+           'c:/Program Files (x86)/Windows Kits/8.1/bin/x64;{_VSPATH}/Common7/IDE;{_VSPATH}/VC/BIN/amd64;{_VSPATH}/VC/BIN/x86_amd64;{_VSPATH}/VC/BIN;{_VSPATH}/Common7/Tools;{_VSPATH}/VC/VCPackages;c:/mozilla-build/moztools-x64'.format(_VSPATH=VSPATH),
+           'c:/windows/Microsoft.NET/Framework64/v3.5;c:/windows/Microsoft.NET/Framework64/v4.0.30319',
+           os.environ['PATH'],
+           'C:\\mozilla-build\\Git\\bin',
+           'C:\\mozilla-build\\svn-win32-1.6.3\\bin',
+       ),
+       'WIN32_REDIST_DIR': '{_VSPATH}/VC/redist/x64/Microsoft.VC120.CRT'.format(_VSPATH=VSPATH),
+       'INCLUDE': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\shared;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\um;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl\\wrappers;{_VSPATH}\\vc\\include;{_VSPATH}\\vc\\atlmfc\\include;c:\\tools\\sdks\\dx10\\include'.format(_VSPATH=VSPATH),
+       'LIB': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x64;{_VSPATH}\\vc\\lib\\amd64;{_VSPATH}\\vc\\atlmfc\\lib\\amd64;c:\\tools\\sdks\\dx10\\lib\\x64'.format(_VSPATH=VSPATH),
+       'LIBPATH': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x64;{_VSPATH}\\vc\\lib\\amd64;{_VSPATH}\\vc\\atlmfc\\lib\\amd64;c:\\tools\\sdks\\dx10\\lib\\x64'.format(_VSPATH=VSPATH),
+       'WINDOWSSDKDIR': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\',
+   },
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/external_tools/packagesymbols.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import os
+import subprocess
+import sys
+import zipfile
+
+
+class ProcError(Exception):
+    def __init__(self, returncode, stderr):
+        self.returncode = returncode
+        self.stderr = stderr
+
+
+def check_output(command):
+    proc = subprocess.Popen(command,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    stdout, stderr = proc.communicate()
+    if proc.returncode != 0:
+        raise ProcError(proc.returncode, stderr)
+    return stdout
+
+
+def process_file(dump_syms, path):
+    try:
+        stdout = check_output([dump_syms, path])
+    except ProcError as e:
+        print('Error: running "%s %s": %s' % (dump_syms, path, e.stderr))
+        return None, None, None
+    bits = stdout.splitlines()[0].split(' ', 4)
+    if len(bits) != 5:
+        return None, None, None
+    _, platform, cpu_arch, debug_id, debug_file = bits
+    if debug_file.lower().endswith('.pdb'):
+        sym_file = debug_file[:-4] + '.sym'
+    else:
+        sym_file = debug_file + '.sym'
+    filename = os.path.join(debug_file, debug_id, sym_file)
+    debug_filename = os.path.join(debug_file, debug_id, debug_file)
+    return filename, stdout, debug_filename
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('dump_syms', help='Path to dump_syms binary')
+    parser.add_argument('files', nargs='+',
+                        help='Path to files to dump symbols from')
+    parser.add_argument('--symbol-zip', default='symbols.zip',
+                        help='Name of zip file to put dumped symbols in')
+    parser.add_argument('--no-binaries',
+                        action='store_true',
+                        default=False,
+                        help='Don\'t store binaries in zip file')
+    args = parser.parse_args()
+    count = 0
+    with zipfile.ZipFile(args.symbol_zip, 'w', zipfile.ZIP_DEFLATED) as zf:
+        for f in args.files:
+            filename, contents, debug_filename = process_file(args.dump_syms, f)
+            if not (filename and contents):
+                print('Error dumping symbols')
+                sys.exit(1)
+            zf.writestr(filename, contents)
+            count += 1
+            if not args.no_binaries:
+                zf.write(f, debug_filename)
+                count += 1
+    print('Added %d files to %s' % (count, args.symbol_zip))
+
+if __name__ == '__main__':
+    main()
--- a/testing/mozharness/mozharness/base/script.py
+++ b/testing/mozharness/mozharness/base/script.py
@@ -1443,17 +1443,17 @@ class ScriptMixin(PlatformMixin):
                 self.return_code = fatal_exit_code
                 self.fatal("Halting on failure while running %s" % command,
                            exit_code=fatal_exit_code)
         if return_type == 'num_errors':
             return parser.num_errors
         return returncode
 
     def get_output_from_command(self, command, cwd=None,
-                                halt_on_failure=False, env=None,
+                                halt_on_failure=False, env=None, partial_env=None,
                                 silent=False, log_level=INFO,
                                 tmpfile_base_path='tmpfile',
                                 return_type='output', save_tmpfiles=False,
                                 throw_exception=False, fatal_exit_code=2,
                                 ignore_errors=False, success_codes=None):
         """Similar to run_command, but where run_command is an
         os.system(command) analog, get_output_from_command is a `command`
         analog.
@@ -1475,16 +1475,18 @@ class ScriptMixin(PlatformMixin):
             command (str | list): command or list of commands to
               execute and log.
             cwd (str, optional): directory path from where to execute the
               command. Defaults to `None`.
             halt_on_failure (bool, optional): whether or not to redefine the
               log level as `FATAL` on error. Defaults to False.
             env (dict, optional): key-value of environment values to use to
               run the command. Defaults to None.
+            partial_env (dict, optional): key-value of environment values to
+              replace from the current environment values. Defaults to None.
             silent (bool, optional): whether or not to output the stdout of
               executing the command. Defaults to False.
             log_level (str, optional): log level name to use on normal execution.
               Defaults to `INFO`.
             tmpfile_base_path (str, optional): base path of the file to which
               the output will be writen to. Defaults to 'tmpfile'.
             return_type (str, optional): if equal to 'output' then the complete
               output of the executed command is returned, otherwise the written
@@ -1543,18 +1545,26 @@ class ScriptMixin(PlatformMixin):
         except IOError:
             level = ERROR
             if halt_on_failure:
                 level = FATAL
             self.log("Can't open %s for writing!" % tmp_stderr_filename +
                      self.exception(), level=level)
             return None
         shell = True
-        if isinstance(command, list):
+        if isinstance(command, list) or isinstance(command, tuple):
             shell = False
+
+        if env is None:
+            if partial_env:
+                self.info("Using partial env: %s" % pprint.pformat(partial_env))
+                env = self.query_env(partial_env=partial_env)
+        else:
+            self.info("Using env: %s" % pprint.pformat(env))
+
         p = subprocess.Popen(command, shell=shell, stdout=tmp_stdout,
                              cwd=cwd, stderr=tmp_stderr, env=env)
         # XXX: changed from self.debug to self.log due to this error:
         #      TypeError: debug() takes exactly 1 argument (2 given)
         self.log("Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), level=DEBUG)
         p.wait()
         tmp_stdout.close()
         tmp_stderr.close()
--- a/testing/mozharness/scripts/openh264_build.py
+++ b/testing/mozharness/scripts/openh264_build.py
@@ -2,42 +2,55 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 import sys
 import os
 import glob
+import re
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 # import the guts
+import mozharness
 from mozharness.base.vcs.vcsbase import VCSScript
 from mozharness.base.log import ERROR
 from mozharness.base.transfer import TransferMixin
 from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.tooltool import TooltoolMixin
 
 
-class OpenH264Build(MockMixin, TransferMixin, VCSScript):
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+
+class OpenH264Build(MockMixin, TransferMixin, VCSScript, TooltoolMixin):
     all_actions = [
         'clobber',
+        'get-tooltool',
         'checkout-sources',
         'build',
         'test',
         'package',
+        'dump-symbols',
         'upload',
     ]
 
     default_actions = [
+        'get-tooltool',
         'checkout-sources',
         'build',
         'test',
         'package',
+        'dump-symbols',
     ]
 
     config_options = [
         [["--repo"], {
             "dest": "repo",
             "help": "OpenH264 repository to use",
             "default": "https://github.com/cisco/openh264.git"
         }],
@@ -46,26 +59,19 @@ class OpenH264Build(MockMixin, TransferM
             "help": "revision to checkout",
             "default": "master"
         }],
         [["--debug"], {
             "dest": "debug_build",
             "action": "store_true",
             "help": "Do a debug build",
         }],
-        [["--64"], {
-            "dest": "64bit",
-            "action": "store_true",
-            "help": "Do a 64-bit build",
-            "default": True,
-        }],
-        [["--32"], {
-            "dest": "64bit",
-            "action": "store_false",
-            "help": "Do a 32-bit build",
+        [["--arch"], {
+            "dest": "arch",
+            "help": "Arch type to use (x64, x86, or arm)",
         }],
         [["--os"], {
             "dest": "operating_system",
             "help": "Specify the operating system to build for",
         }],
         [["--use-mock"], {
             "dest": "use_mock",
             "help": "use mock to set up build environment",
@@ -73,28 +79,31 @@ class OpenH264Build(MockMixin, TransferM
             "default": False,
         }],
         [["--use-yasm"], {
             "dest": "use_yasm",
             "help": "use yasm instead of nasm",
             "action": "store_true",
             "default": False,
         }],
+        [["--avoid-avx2"], {
+            "dest": "avoid_avx2",
+            "help": "Pass HAVE_AVX2='false' through to Make to support older nasm",
+            "action": "store_true",
+            "default": False,
+        }]
     ]
 
     def __init__(self, require_config_file=False, config={},
                  all_actions=all_actions,
                  default_actions=default_actions):
 
         # Default configuration
         default_config = {
             'debug_build': False,
-            'mock_target': 'mozilla-centos6-x86_64',
-            'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686', 'libstdc++-devel.i686', 'zip', 'yasm'],
-            'mock_files': [],
             'upload_ssh_key': os.path.expanduser("~/.ssh/ffxbld_rsa"),
             'upload_ssh_user': 'ffxbld',
             'upload_ssh_host': 'stage.mozilla.org',
             'upload_path_base': '/home/ffxbld/openh264',
             'use_yasm': False,
         }
         default_config.update(config)
 
@@ -106,47 +115,85 @@ class OpenH264Build(MockMixin, TransferM
             all_actions=all_actions,
             default_actions=default_actions,
         )
 
         if self.config['use_mock']:
             self.setup_mock()
             self.enable_mock()
 
+    def get_tooltool(self):
+        c = self.config
+        if not c.get('tooltool_manifest_file'):
+            self.info("Skipping tooltool fetching since no tooltool manifest")
+            return
+        dirs = self.query_abs_dirs()
+        self.mkdir_p(dirs['abs_work_dir'])
+        manifest = os.path.join(dirs['base_work_dir'],
+                                'scripts', 'configs',
+                                'openh264', 'tooltool-manifests',
+                                c['tooltool_manifest_file'])
+        self.info("Getting tooltool files from manifest (%s)" % manifest)
+        try:
+            self.tooltool_fetch(
+                manifest=manifest,
+                output_dir=dirs['abs_work_dir'],
+                cache=c.get('tooltool_cache')
+            )
+        except KeyError:
+            self.error('missing a required key.')
+
     def query_package_name(self):
-        if self.config['64bit']:
+        if self.config['arch'] == "x64":
             bits = '64'
         else:
             bits = '32'
 
         version = self.config['revision']
 
         if sys.platform == 'linux2':
             if self.config.get('operating_system') == 'android':
-                return 'openh264-android-{version}.zip'.format(version=version, bits=bits)
+                if self.config.get('arch') == 'x86':
+                    return 'openh264-android-{arch}-{version}.zip'.format(
+                        version=version, bits=bits, arch=self.config['arch']
+                    )
+                else:
+                    return 'openh264-android-{arch}-{version}.zip'.format(
+                        version=version, bits=bits, arch='arm'
+                    )
             else:
                 return 'openh264-linux{bits}-{version}.zip'.format(version=version, bits=bits)
         elif sys.platform == 'darwin':
             return 'openh264-macosx{bits}-{version}.zip'.format(version=version, bits=bits)
         elif sys.platform == 'win32':
             return 'openh264-win{bits}-{version}.zip'.format(version=version, bits=bits)
         self.fatal("can't determine platform")
 
     def query_make_params(self):
+        dirs = self.query_abs_dirs()
         retval = []
         if self.config['debug_build']:
             retval.append('BUILDTYPE=Debug')
 
-        if self.config['64bit']:
+        if self.config['avoid_avx2']:
+            retval.append('HAVE_AVX2=false')
+
+        if self.config['arch'] == 'x64':
             retval.append('ENABLE64BIT=Yes')
         else:
             retval.append('ENABLE64BIT=No')
 
         if "operating_system" in self.config:
             retval.append("OS=%s" % self.config['operating_system'])
+            if self.config["operating_system"] == "android":
+                if self.config['arch'] == 'x86':
+                    retval.append("ARCH=x86")
+                retval.append('TARGET=invalid')
+                retval.append('NDKLEVEL=9')
+                retval.append('NDKROOT=%s/android-ndk' % dirs['abs_work_dir'])
 
         if self.config['use_yasm']:
             retval.append('ASM=yasm')
 
         return retval
 
     def query_upload_ssh_key(self):
         return self.config['upload_ssh_key']
@@ -155,21 +202,27 @@ class OpenH264Build(MockMixin, TransferM
         return self.config['upload_ssh_host']
 
     def query_upload_ssh_user(self):
         return self.config['upload_ssh_user']
 
     def query_upload_ssh_path(self):
         return "%s/%s" % (self.config['upload_path_base'], self.config['revision'])
 
-    def run_make(self, target):
+    def run_make(self, target, capture_output=False):
         cmd = ['make', target] + self.query_make_params()
         dirs = self.query_abs_dirs()
         repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
-        return self.run_command(cmd, cwd=repo_dir)
+        env = self.config.get('env')
+        partial_env = self.config.get('partial_env')
+        kwargs = dict(cwd=repo_dir, env=env, partial_env=partial_env)
+        if capture_output:
+            return self.get_output_from_command(cmd, **kwargs)
+        else:
+            return self.run_command(cmd, **kwargs)
 
     def checkout_sources(self):
         repo = self.config['repo']
         rev = self.config['revision']
 
         dirs = self.query_abs_dirs()
         repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
 
@@ -212,23 +265,61 @@ class OpenH264Build(MockMixin, TransferM
 
     def package(self):
         dirs = self.query_abs_dirs()
         srcdir = os.path.join(dirs['abs_work_dir'], 'src')
         package_name = self.query_package_name()
         package_file = os.path.join(dirs['abs_work_dir'], package_name)
         if os.path.exists(package_file):
             os.unlink(package_file)
-        to_package = [os.path.basename(f) for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*"))]
+        to_package = []
+        for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")):
+            if not re.search(
+                    "(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)",
+                    f):
+                # Don't package unnecessary zip bloat
+                # Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1
+                self.log("Skipping packaging of {package}".format(package=f))
+                continue
+            to_package.append(os.path.basename(f))
+        self.log("Packaging files %s" % to_package)
         cmd = ['zip', package_file] + to_package
         retval = self.run_command(cmd, cwd=srcdir)
         if retval != 0:
             self.fatal("couldn't make package")
         self.copy_to_upload_dir(package_file)
 
+    def dump_symbols(self):
+        dirs = self.query_abs_dirs()
+        c = self.config
+        srcdir = os.path.join(dirs['abs_work_dir'], 'src')
+        package_name = self.run_make('echo-plugin-name', capture_output=True)
+        if not package_name:
+            self.fatal("failure running make")
+        zip_package_name = self.query_package_name()
+        if not zip_package_name[-4:] == ".zip":
+            self.fatal("Unexpected zip_package_name")
+        symbol_package_name = "{base}.symbols.zip".format(base=zip_package_name[:-4])
+        symbol_zip_path = os.path.join(dirs['abs_upload_dir'], symbol_package_name)
+        repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
+        env = self.config.get('env')
+        partial_env = self.config.get('partial_env')
+        kwargs = dict(cwd=repo_dir, env=env, partial_env=partial_env)
+        dump_syms = os.path.join(dirs['abs_work_dir'], c['dump_syms_binary'])
+        self.chmod(dump_syms, 0755)
+        python = self.query_exe('python2.7')
+        cmd = [python, os.path.join(external_tools_path, 'packagesymbols.py'),
+               '--symbol-zip', symbol_zip_path,
+               dump_syms, os.path.join(srcdir, package_name)]
+        if self.config['use_mock']:
+            self.disable_mock()
+        self.run_command(cmd, **kwargs)
+        if self.config['use_mock']:
+            self.enable_mock()
+
     def upload(self):
         if self.config['use_mock']:
             self.disable_mock()
         dirs = self.query_abs_dirs()
         self.rsync_upload_directory(
             dirs['abs_upload_dir'],
             self.query_upload_ssh_key(),
             self.query_upload_ssh_user(),