Bug 1284800: Create new update script for libyuv r=sotaro,rillian
authorRandell Jesup <rjesup@jesup.org>
Fri, 24 Feb 2017 14:01:40 -0500
changeset 344728 946da1e13208
parent 344727 654b04ab210a
child 344729 c560b9eed8b1
push id87432
push userrjesup@wgate.com
push date2017-02-24 19:03 +0000
treeherdermozilla-inbound@b21f4e412b66 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssotaro, rillian
bugs1284800
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1284800: Create new update script for libyuv r=sotaro,rillian MozReview-Commit-ID: 3MK84gHSrLg
media/libyuv/README_MOZILLA
media/libyuv/add_H444ToARGB.patch
media/libyuv/allow_disabling_asm_avx2.patch
media/libyuv/fix_build_errors.patch
media/libyuv/make_mjpeg_printfs_optional.patch
media/libyuv/update.py
media/libyuv/update_gyp.patch
new file mode 100644
--- /dev/null
+++ b/media/libyuv/README_MOZILLA
@@ -0,0 +1,12 @@
+YUV-format image scaling and conversion library
+
+The source in this directory was copied from upstream by running the
+update.py script from media/libyuv. Any changes made relative to upstream
+should be reflected in that script, e.g. by applying patch files after the
+copy step.
+
+The upstream repository is https://chromium.googlesource.com/libyuv/libyuv
+Updates before this file was added were done manually; the last import was
+r1602 (svn).
+
+The git commit ID last used to import was 054ec37f8e31e6131c8f19eb74395d29009d6604
new file mode 100644
--- /dev/null
+++ b/media/libyuv/add_H444ToARGB.patch
@@ -0,0 +1,76 @@
+# HG changeset patch
+# User Randell Jesup <rjesup@jesup.org>
+# Parent  fb2b299f7dfd2bfaaea3f9a626f4022a85f24f34
+Bug 1341543: add_H444ToARGB() to libyuv r=sotaro
+
+MozReview-Commit-ID: 5NfATfZtWz
+
+diff --git a/media/libyuv/libyuv/include/libyuv/convert_argb.h b/media/libyuv/libyuv/include/libyuv/convert_argb.h
+--- a/media/libyuv/libyuv/include/libyuv/convert_argb.h
++++ b/media/libyuv/libyuv/include/libyuv/convert_argb.h
+@@ -84,16 +84,28 @@ int I444ToARGB(const uint8* src_y,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+                uint8* dst_argb,
+                int dst_stride_argb,
+                int width,
+                int height);
+ 
++LIBYUV_API
++int H444ToARGB(const uint8* src_y,
++               int src_stride_y,
++               const uint8* src_u,
++               int src_stride_u,
++               const uint8* src_v,
++               int src_stride_v,
++               uint8* dst_argb,
++               int dst_stride_argb,
++               int width,
++               int height);
++
+ // Convert J444 to ARGB.
+ LIBYUV_API
+ int J444ToARGB(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+diff --git a/media/libyuv/libyuv/source/convert_argb.cc b/media/libyuv/libyuv/source/convert_argb.cc
+--- a/media/libyuv/libyuv/source/convert_argb.cc
++++ b/media/libyuv/libyuv/source/convert_argb.cc
+@@ -524,16 +524,33 @@ int I444ToARGB(const uint8* src_y,
+                int dst_stride_argb,
+                int width,
+                int height) {
+   return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+                           src_stride_v, dst_argb, dst_stride_argb,
+                           &kYuvI601Constants, width, height);
+ }
+ 
++// Convert H444 to ARGB.
++LIBYUV_API
++int H444ToARGB(const uint8* src_y,
++               int src_stride_y,
++               const uint8* src_u,
++               int src_stride_u,
++               const uint8* src_v,
++               int src_stride_v,
++               uint8* dst_argb,
++               int dst_stride_argb,
++               int width,
++               int height) {
++  return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
++                          src_stride_v, dst_argb, dst_stride_argb,
++                          &kYuvH709Constants, width, height);
++}
++
+ // Convert I444 to ABGR.
+ LIBYUV_API
+ int I444ToABGR(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
new file mode 100644
--- /dev/null
+++ b/media/libyuv/allow_disabling_asm_avx2.patch
@@ -0,0 +1,97 @@
+# HG changeset patch
+# Parent  7591140de51c6147662ee708391fba138398316d
+Bug 1341543: Allow disabling of inline ASM and AVX2 in libyyuv r=sotaro
+
+diff --git a/media/libyuv/libyuv/include/libyuv/row.h b/media/libyuv/libyuv/include/libyuv/row.h
+--- a/media/libyuv/libyuv/include/libyuv/row.h
++++ b/media/libyuv/libyuv/include/libyuv/row.h
+@@ -181,17 +181,17 @@ extern "C" {
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_SSSE3
+ #endif
+ #endif
+ 
+ // The following are available on all x86 platforms, but
+ // require VS2012, clang 3.4 or gcc 4.7.
+ // The code supports NaCL but requires a new compiler and validator.
+-#if !defined(LIBYUV_DISABLE_X86) &&                          \
++#if !defined(LIBYUV_DISABLE_AVX2) && !defined(LIBYUV_DISABLE_X86) &&                          \
+     (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
+      defined(GCC_HAS_AVX2))
+ #define HAS_ARGBCOPYALPHAROW_AVX2
+ #define HAS_ARGBCOPYYTOALPHAROW_AVX2
+ #define HAS_ARGBEXTRACTALPHAROW_AVX2
+ #define HAS_ARGBMIRRORROW_AVX2
+ #define HAS_ARGBPOLYNOMIALROW_AVX2
+ #define HAS_ARGBSHUFFLEROW_AVX2
+diff --git a/media/libyuv/libyuv/libyuv.gyp b/media/libyuv/libyuv/libyuv.gyp
+--- a/media/libyuv/libyuv/libyuv.gyp
++++ b/media/libyuv/libyuv/libyuv.gyp
+@@ -21,16 +21,18 @@
+     # Can be enabled if your jpeg has GYP support.
+     'libyuv_disable_jpeg%': 1,
+     # 'chromium_code' treats libyuv as internal and increases warning level.
+     'chromium_code': 1,
+     # clang compiler default variable usable by other apps that include libyuv.
+     'clang%': 0,
+     # Link-Time Optimizations.
+     'use_lto%': 0,
++    'yuv_disable_asm%': 0,
++    'yuv_disable_avx2%': 0,
+     'mips_msa%': 0,  # Default to msa off.
+     'build_neon': 0,
+     'build_msa': 0,
+     'conditions': [
+        ['(target_arch == "armv7" or target_arch == "armv7s" or \
+        (target_arch == "arm" and arm_version >= 7) or target_arch == "arm64")\
+        and (arm_neon == 1 or arm_neon_optional == 1)', {
+          'build_neon': 1,
+@@ -95,16 +97,29 @@
+             }],
+           ],
+         }],
+         ['build_msa != 0', {
+           'defines': [
+             'LIBYUV_MSA',
+           ],
+         }],
++        [ 'yuv_disable_asm != 0', {
++          'defines': [
++            # Enable the following 3 macros to turn off assembly for specified CPU.
++            'LIBYUV_DISABLE_X86',
++            'LIBYUV_DISABLE_NEON',
++            'LIBYUV_DISABLE_DSPR2',
++          ],
++        }],
++        [ 'yuv_disable_avx2 == 1', {
++          'defines': [
++            'LIBYUV_DISABLE_AVX2',
++          ]
++        }],
+         ['build_with_mozilla == 1', {
+           'defines': [
+             'HAVE_JPEG'
+           ],
+           'cflags_mozilla': [
+             '$(MOZ_JPEG_CFLAGS)',
+           ],
+         }],
+@@ -133,16 +148,18 @@
+           ],
+         }],
+       ], #conditions
+       'defines': [
+         # Enable the following 3 macros to turn off assembly for specified CPU.
+         # 'LIBYUV_DISABLE_X86',
+         # 'LIBYUV_DISABLE_NEON',
+         # 'LIBYUV_DISABLE_DSPR2',
++        # This disables AVX2 (Haswell) support, overriding compiler checks
++        # 'LIBYUV_DISABLE_AVX2',
+         # Enable the following macro to build libyuv as a shared library (dll).
+         # 'LIBYUV_USING_SHARED_LIBRARY',
+         # TODO(fbarchard): Make these into gyp defines.
+       ],
+       'include_dirs': [
+         'include',
+         '.',
+       ],
new file mode 100644
--- /dev/null
+++ b/media/libyuv/fix_build_errors.patch
@@ -0,0 +1,94 @@
+# HG changeset patch
+# Parent  034c99a4c2010c8d60ff52daf0a183ac1fd05777
+Bug 1341543: Fix build errors for libyuv in mozilla tree r=sotaro
+
+diff --git a/media/libyuv/libyuv/include/libyuv/basic_types.h b/media/libyuv/libyuv/include/libyuv/basic_types.h
+--- a/media/libyuv/libyuv/include/libyuv/basic_types.h
++++ b/media/libyuv/libyuv/include/libyuv/basic_types.h
+@@ -8,22 +8,36 @@
+  *  be found in the AUTHORS file in the root of the source tree.
+  */
+ 
+ #ifndef INCLUDE_LIBYUV_BASIC_TYPES_H_
+ #define INCLUDE_LIBYUV_BASIC_TYPES_H_
+ 
+ #include <stddef.h>  // for NULL, size_t
+ 
+-#if defined(_MSC_VER) && (_MSC_VER < 1600)
+-#include <sys/types.h>  // for uintptr_t on x86
+-#else
++#if !defined(_MSC_VER) && (_MSC_VER < 1600)
+ #include <stdint.h>  // for uintptr_t
+ #endif
+ 
++typedef uint64_t uint64;
++typedef int64_t  int64;
++#if defined(_MSC_VER)
++// nsprpub/pr/include/obsolete/protypes.h defines these weirdly
++typedef long int32;
++typedef unsigned long uint32;
++#else
++typedef uint32_t uint32;
++typedef int32_t  int32;
++#endif
++typedef uint16_t uint16;
++typedef int16_t  int16;
++typedef uint8_t  uint8;
++typedef int8_t   int8;
++#define INT_TYPES_DEFINED 1
++
+ #ifndef GG_LONGLONG
+ #ifndef INT_TYPES_DEFINED
+ #define INT_TYPES_DEFINED
+ #ifdef COMPILER_MSVC
+ typedef unsigned __int64 uint64;
+ typedef __int64 int64;
+ #ifndef INT64_C
+ #define INT64_C(x) x##I64
+diff --git a/media/libyuv/libyuv/include/libyuv/row.h b/media/libyuv/libyuv/include/libyuv/row.h
+--- a/media/libyuv/libyuv/include/libyuv/row.h
++++ b/media/libyuv/libyuv/include/libyuv/row.h
+@@ -169,18 +169,19 @@ extern "C" {
+ #define HAS_SOBELROW_SSE2
+ #define HAS_SOBELTOPLANEROW_SSE2
+ #define HAS_SOBELXROW_SSE2
+ #define HAS_SOBELXYROW_SSE2
+ #define HAS_SOBELYROW_SSE2
+ 
+ // The following functions fail on gcc/clang 32 bit with fpic and framepointer.
+ // caveat: clangcl uses row_win.cc which works.
+-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+-    defined(_MSC_VER)
++#if !defined(MOZ_PROFILING) && \
++  (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) ||  \
++   defined(_MSC_VER))
+ // TODO(fbarchard): fix build error on android_full_debug=1
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_SSSE3
+ #endif
+ #endif
+ 
+ // The following are available on all x86 platforms, but
+ // require VS2012, clang 3.4 or gcc 4.7.
+@@ -231,18 +232,19 @@ extern "C" {
+ // Effects:
+ #define HAS_ARGBADDROW_AVX2
+ #define HAS_ARGBATTENUATEROW_AVX2
+ #define HAS_ARGBMULTIPLYROW_AVX2
+ #define HAS_ARGBSUBTRACTROW_AVX2
+ #define HAS_ARGBUNATTENUATEROW_AVX2
+ #define HAS_BLENDPLANEROW_AVX2
+ 
+-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+-    defined(_MSC_VER)
++#if !defined(MOZ_PROFILING) && \
++  (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) ||  \
++   defined(_MSC_VER))
+ // TODO(fbarchard): fix build error on android_full_debug=1
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_AVX2
+ #endif
+ #endif
+ 
+ // The following are available for AVX2 Visual C and clangcl 32 bit:
+ // TODO(fbarchard): Port to gcc.
new file mode 100644
--- /dev/null
+++ b/media/libyuv/make_mjpeg_printfs_optional.patch
@@ -0,0 +1,52 @@
+# HG changeset patch
+# Parent  b079f87bb602629bb8b51979d79f825be7aef1ae
+Bug 1341513: Make disable MJPEG fprintf warnings build-time option r=sotaro
+
+diff --git a/media/libyuv/libyuv/source/mjpeg_decoder.cc b/media/libyuv/libyuv/source/mjpeg_decoder.cc
+--- a/media/libyuv/libyuv/source/mjpeg_decoder.cc
++++ b/media/libyuv/libyuv/source/mjpeg_decoder.cc
+@@ -73,18 +73,20 @@ MJpegDecoder::MJpegDecoder()
+       databuf_strides_(NULL) {
+   decompress_struct_ = new jpeg_decompress_struct;
+   source_mgr_ = new jpeg_source_mgr;
+ #ifdef HAVE_SETJMP
+   error_mgr_ = new SetJmpErrorMgr;
+   decompress_struct_->err = jpeg_std_error(&error_mgr_->base);
+   // Override standard exit()-based error handler.
+   error_mgr_->base.error_exit = &ErrorHandler;
++#ifndef DEBUG_MJPEG
+   error_mgr_->base.output_message = &OutputHandler;
+ #endif
++#endif
+   decompress_struct_->client_data = NULL;
+   source_mgr_->init_source = &init_source;
+   source_mgr_->fill_input_buffer = &fill_input_buffer;
+   source_mgr_->skip_input_data = &skip_input_data;
+   source_mgr_->resync_to_restart = &jpeg_resync_to_restart;
+   source_mgr_->term_source = &term_source;
+   jpeg_create_decompress(decompress_struct_);
+   decompress_struct_->src = source_mgr_;
+@@ -450,21 +452,22 @@ void ErrorHandler(j_common_ptr cinfo) {
+ #endif
+ 
+   SetJmpErrorMgr* mgr = reinterpret_cast<SetJmpErrorMgr*>(cinfo->err);
+   // This rewinds the call stack to the point of the corresponding setjmp()
+   // and causes it to return (for a second time) with value 1.
+   longjmp(mgr->setjmp_buffer, 1);
+ }
+ 
++#ifndef DEBUG_MJPEG
+ // Suppress fprintf warnings.
+ void OutputHandler(j_common_ptr cinfo) {
+   (void)cinfo;
+ }
+-
++#endif
+ #endif  // HAVE_SETJMP
+ 
+ void MJpegDecoder::AllocOutputBuffers(int num_outbufs) {
+   if (num_outbufs != num_outbufs_) {
+     // We could perhaps optimize this case to resize the output buffers without
+     // necessarily having to delete and recreate each one, but it's not worth
+     // it.
+     DestroyOutputBuffers();
new file mode 100644
--- /dev/null
+++ b/media/libyuv/update.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import argparse
+import os
+import re
+import shutil
+import sys
+import subprocess
+import tarfile
+import urllib
+from pprint import pprint
+from StringIO import StringIO
+
+def prepare_upstream(prefix, commit=None):
+    upstream_url = 'https://chromium.googlesource.com/libyuv/libyuv'
+    shutil.rmtree(os.path.join(base, 'libyuv/'))
+    print(upstream_url + '/+archive/' + commit + '.tar.gz')
+    urllib.urlretrieve(upstream_url + '/+archive/' + commit + '.tar.gz', 'libyuv.tar.gz')
+    tarfile.open('libyuv.tar.gz').extractall(path='libyuv')
+    os.remove(os.path.join(base, 'libyuv.tar.gz'))
+    os.chdir(base)
+    return commit
+
+def cleanup_upstream():
+    os.remove(os.path.join(base, 'libyuv/.gitignore'))
+
+def apply_patches():
+    # Patch to update gyp build files
+    os.system("patch -p3 < update_gyp.patch")
+    # Patch to fix build errors
+    os.system("patch -p3 < fix_build_errors.patch")
+    # Patch to make mjpeg printfs optional at build time
+    os.system("patch -p3 < make_mjpeg_printfs_optional.patch")
+    # Patch to allow disabling of inline ASM and AVX2 code
+    os.system("patch -p3 < allow_disabling_asm_avx2.patch")
+    # Patch to add H444ToARGB() variant
+    os.system("patch -p3 < add_H444ToARGB.patch")
+
+def update_readme(commit):
+    with open('README_MOZILLA') as f:
+        readme = f.read()
+
+    if 'The git commit ID last used to import was ' in readme:
+        new_readme = re.sub('The git commit ID last used to import was [v\.a-f0-9]+',
+            'The git commit ID last used to import was %s' % commit, readme)
+    else:
+        new_readme = "%s\n\nThe git commit ID last used to import was %s\n" % (readme, commit)
+
+    if readme != new_readme:
+        with open('README_MOZILLA', 'w') as f:
+            f.write(new_readme)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='''Update libyuv''')
+    parser.add_argument('--debug', dest='debug', action="store_true")
+    parser.add_argument('--commit', dest='commit', type=str, default='master')
+
+    args = parser.parse_args()
+
+    commit = args.commit
+    DEBUG = args.debug
+
+    base = os.path.abspath(os.curdir)
+    prefix = os.path.join(base, 'libyuv/')
+
+    commit = prepare_upstream(prefix, commit)
+
+    apply_patches()
+    update_readme(commit)
+
+    print('Patches applied; run "hg addremove --similarity 70 libyuv" before committing changes')
+
+    cleanup_upstream()
new file mode 100644
--- /dev/null
+++ b/media/libyuv/update_gyp.patch
@@ -0,0 +1,83 @@
+# HG changeset patch
+# Parent  034c99a4c2010c8d60ff52daf0a183ac1fd05777
+Bug 1341543: Update libyuv gyp file with mozilla changes r=sotaro
+
+diff --git a/media/libyuv/libyuv/libyuv.gyp b/media/libyuv/libyuv/libyuv.gyp
+--- a/media/libyuv/libyuv/libyuv.gyp
++++ b/media/libyuv/libyuv/libyuv.gyp
+@@ -47,17 +47,17 @@
+     {
+       'target_name': 'libyuv',
+       # Change type to 'shared_library' to build .so or .dll files.
+       'type': 'static_library',
+       'variables': {
+         'optimize': 'max',  # enable O2 and ltcg.
+       },
+       # Allows libyuv.a redistributable library without external dependencies.
+-      'standalone_static_library': 1,
++      # 'standalone_static_library': 1,
+       'conditions': [
+        # Disable -Wunused-parameter
+         ['clang == 1', {
+           'cflags': [
+             '-Wno-unused-parameter',
+          ],
+         }],
+         ['build_neon != 0', {
+@@ -65,39 +65,55 @@
+             'LIBYUV_NEON',
+           ],
+           'cflags!': [
+             '-mfpu=vfp',
+             '-mfpu=vfpv3',
+             '-mfpu=vfpv3-d16',
+             # '-mthumb',  # arm32 not thumb
+           ],
++          'cflags_mozilla!': [
++            '-mfpu=vfp',
++            '-mfpu=vfpv3',
++            '-mfpu=vfpv3-d16',
++          ],
+           'conditions': [
+             # Disable LTO in libyuv_neon target due to gcc 4.9 compiler bug.
+             ['clang == 0 and use_lto == 1', {
+               'cflags!': [
+                 '-flto',
+                 '-ffat-lto-objects',
+               ],
+             }],
+             # arm64 does not need -mfpu=neon option as neon is not optional
+             ['target_arch != "arm64"', {
+               'cflags': [
+                 '-mfpu=neon',
+                 # '-marm',  # arm32 not thumb
+               ],
++              'cflags_mozilla': [
++                '-mfpu=neon',
++              ],
+             }],
+           ],
+         }],
+         ['build_msa != 0', {
+           'defines': [
+             'LIBYUV_MSA',
+           ],
+         }],
+-        ['OS != "ios" and libyuv_disable_jpeg != 1', {
++        ['build_with_mozilla == 1', {
++          'defines': [
++            'HAVE_JPEG'
++          ],
++          'cflags_mozilla': [
++            '$(MOZ_JPEG_CFLAGS)',
++          ],
++        }],
++        ['OS != "ios" and libyuv_disable_jpeg != 1 and build_with_mozilla != 1', {
+           'defines': [
+             'HAVE_JPEG'
+           ],
+           'conditions': [
+             # Caveat system jpeg support may not support motion jpeg
+             [ 'use_system_libjpeg == 1', {
+               'dependencies': [
+                  '<(DEPTH)/third_party/libjpeg/libjpeg.gyp:libjpeg',