Bug 1284800: Create new update script for libyuv r=sotaro,rillian
authorRandell Jesup <rjesup@jesup.org>
Fri, 24 Feb 2017 14:01:40 -0500
changeset 373833 946da1e13208023d553eeba96f6797f6f638278f
parent 373832 654b04ab210a0217d58602f89be5cf132aa57f78
child 373834 c560b9eed8b12f88686453cbb8bf52c93a31636b
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssotaro, rillian
bugs1284800
milestone54.0a1
Bug 1284800: Create new update script for libyuv r=sotaro,rillian MozReview-Commit-ID: 3MK84gHSrLg
media/libyuv/README_MOZILLA
media/libyuv/add_H444ToARGB.patch
media/libyuv/allow_disabling_asm_avx2.patch
media/libyuv/fix_build_errors.patch
media/libyuv/make_mjpeg_printfs_optional.patch
media/libyuv/update.py
media/libyuv/update_gyp.patch
new file mode 100644
--- /dev/null
+++ b/media/libyuv/README_MOZILLA
@@ -0,0 +1,12 @@
+YUV-format image scaling and conversion library
+
+The source in this directory was copied from upstream by running the
+update.py script from media/libyuv. Any changes made relative to upstream
+should be reflected in that script, e.g. by applying patch files after the
+copy step.
+
+The upstream repository is https://chromium.googlesource.com/libyuv/libyuv
+Updates before this file was added were done manually; the last import was
+r1602 (svn).
+
+The git commit ID last used to import was 054ec37f8e31e6131c8f19eb74395d29009d6604
new file mode 100644
--- /dev/null
+++ b/media/libyuv/add_H444ToARGB.patch
@@ -0,0 +1,76 @@
+# HG changeset patch
+# User Randell Jesup <rjesup@jesup.org>
+# Parent  fb2b299f7dfd2bfaaea3f9a626f4022a85f24f34
+Bug 1341543: add_H444ToARGB() to libyuv r=sotaro
+
+MozReview-Commit-ID: 5NfATfZtWz
+
+diff --git a/media/libyuv/libyuv/include/libyuv/convert_argb.h b/media/libyuv/libyuv/include/libyuv/convert_argb.h
+--- a/media/libyuv/libyuv/include/libyuv/convert_argb.h
++++ b/media/libyuv/libyuv/include/libyuv/convert_argb.h
+@@ -84,16 +84,28 @@ int I444ToARGB(const uint8* src_y,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+                uint8* dst_argb,
+                int dst_stride_argb,
+                int width,
+                int height);
+ 
++LIBYUV_API
++int H444ToARGB(const uint8* src_y,
++               int src_stride_y,
++               const uint8* src_u,
++               int src_stride_u,
++               const uint8* src_v,
++               int src_stride_v,
++               uint8* dst_argb,
++               int dst_stride_argb,
++               int width,
++               int height);
++
+ // Convert J444 to ARGB.
+ LIBYUV_API
+ int J444ToARGB(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
+diff --git a/media/libyuv/libyuv/source/convert_argb.cc b/media/libyuv/libyuv/source/convert_argb.cc
+--- a/media/libyuv/libyuv/source/convert_argb.cc
++++ b/media/libyuv/libyuv/source/convert_argb.cc
+@@ -524,16 +524,33 @@ int I444ToARGB(const uint8* src_y,
+                int dst_stride_argb,
+                int width,
+                int height) {
+   return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+                           src_stride_v, dst_argb, dst_stride_argb,
+                           &kYuvI601Constants, width, height);
+ }
+ 
++// Convert H444 to ARGB.
++LIBYUV_API
++int H444ToARGB(const uint8* src_y,
++               int src_stride_y,
++               const uint8* src_u,
++               int src_stride_u,
++               const uint8* src_v,
++               int src_stride_v,
++               uint8* dst_argb,
++               int dst_stride_argb,
++               int width,
++               int height) {
++  return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
++                          src_stride_v, dst_argb, dst_stride_argb,
++                          &kYuvH709Constants, width, height);
++}
++
+ // Convert I444 to ABGR.
+ LIBYUV_API
+ int I444ToABGR(const uint8* src_y,
+                int src_stride_y,
+                const uint8* src_u,
+                int src_stride_u,
+                const uint8* src_v,
+                int src_stride_v,
new file mode 100644
--- /dev/null
+++ b/media/libyuv/allow_disabling_asm_avx2.patch
@@ -0,0 +1,97 @@
+# HG changeset patch
+# Parent  7591140de51c6147662ee708391fba138398316d
+Bug 1341543: Allow disabling of inline ASM and AVX2 in libyyuv r=sotaro
+
+diff --git a/media/libyuv/libyuv/include/libyuv/row.h b/media/libyuv/libyuv/include/libyuv/row.h
+--- a/media/libyuv/libyuv/include/libyuv/row.h
++++ b/media/libyuv/libyuv/include/libyuv/row.h
+@@ -181,17 +181,17 @@ extern "C" {
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_SSSE3
+ #endif
+ #endif
+ 
+ // The following are available on all x86 platforms, but
+ // require VS2012, clang 3.4 or gcc 4.7.
+ // The code supports NaCL but requires a new compiler and validator.
+-#if !defined(LIBYUV_DISABLE_X86) &&                          \
++#if !defined(LIBYUV_DISABLE_AVX2) && !defined(LIBYUV_DISABLE_X86) &&                          \
+     (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
+      defined(GCC_HAS_AVX2))
+ #define HAS_ARGBCOPYALPHAROW_AVX2
+ #define HAS_ARGBCOPYYTOALPHAROW_AVX2
+ #define HAS_ARGBEXTRACTALPHAROW_AVX2
+ #define HAS_ARGBMIRRORROW_AVX2
+ #define HAS_ARGBPOLYNOMIALROW_AVX2
+ #define HAS_ARGBSHUFFLEROW_AVX2
+diff --git a/media/libyuv/libyuv/libyuv.gyp b/media/libyuv/libyuv/libyuv.gyp
+--- a/media/libyuv/libyuv/libyuv.gyp
++++ b/media/libyuv/libyuv/libyuv.gyp
+@@ -21,16 +21,18 @@
+     # Can be enabled if your jpeg has GYP support.
+     'libyuv_disable_jpeg%': 1,
+     # 'chromium_code' treats libyuv as internal and increases warning level.
+     'chromium_code': 1,
+     # clang compiler default variable usable by other apps that include libyuv.
+     'clang%': 0,
+     # Link-Time Optimizations.
+     'use_lto%': 0,
++    'yuv_disable_asm%': 0,
++    'yuv_disable_avx2%': 0,
+     'mips_msa%': 0,  # Default to msa off.
+     'build_neon': 0,
+     'build_msa': 0,
+     'conditions': [
+        ['(target_arch == "armv7" or target_arch == "armv7s" or \
+        (target_arch == "arm" and arm_version >= 7) or target_arch == "arm64")\
+        and (arm_neon == 1 or arm_neon_optional == 1)', {
+          'build_neon': 1,
+@@ -95,16 +97,29 @@
+             }],
+           ],
+         }],
+         ['build_msa != 0', {
+           'defines': [
+             'LIBYUV_MSA',
+           ],
+         }],
++        [ 'yuv_disable_asm != 0', {
++          'defines': [
++            # Enable the following 3 macros to turn off assembly for specified CPU.
++            'LIBYUV_DISABLE_X86',
++            'LIBYUV_DISABLE_NEON',
++            'LIBYUV_DISABLE_DSPR2',
++          ],
++        }],
++        [ 'yuv_disable_avx2 == 1', {
++          'defines': [
++            'LIBYUV_DISABLE_AVX2',
++          ]
++        }],
+         ['build_with_mozilla == 1', {
+           'defines': [
+             'HAVE_JPEG'
+           ],
+           'cflags_mozilla': [
+             '$(MOZ_JPEG_CFLAGS)',
+           ],
+         }],
+@@ -133,16 +148,18 @@
+           ],
+         }],
+       ], #conditions
+       'defines': [
+         # Enable the following 3 macros to turn off assembly for specified CPU.
+         # 'LIBYUV_DISABLE_X86',
+         # 'LIBYUV_DISABLE_NEON',
+         # 'LIBYUV_DISABLE_DSPR2',
++        # This disables AVX2 (Haswell) support, overriding compiler checks
++        # 'LIBYUV_DISABLE_AVX2',
+         # Enable the following macro to build libyuv as a shared library (dll).
+         # 'LIBYUV_USING_SHARED_LIBRARY',
+         # TODO(fbarchard): Make these into gyp defines.
+       ],
+       'include_dirs': [
+         'include',
+         '.',
+       ],
new file mode 100644
--- /dev/null
+++ b/media/libyuv/fix_build_errors.patch
@@ -0,0 +1,94 @@
+# HG changeset patch
+# Parent  034c99a4c2010c8d60ff52daf0a183ac1fd05777
+Bug 1341543: Fix build errors for libyuv in mozilla tree r=sotaro
+
+diff --git a/media/libyuv/libyuv/include/libyuv/basic_types.h b/media/libyuv/libyuv/include/libyuv/basic_types.h
+--- a/media/libyuv/libyuv/include/libyuv/basic_types.h
++++ b/media/libyuv/libyuv/include/libyuv/basic_types.h
+@@ -8,22 +8,36 @@
+  *  be found in the AUTHORS file in the root of the source tree.
+  */
+ 
+ #ifndef INCLUDE_LIBYUV_BASIC_TYPES_H_
+ #define INCLUDE_LIBYUV_BASIC_TYPES_H_
+ 
+ #include <stddef.h>  // for NULL, size_t
+ 
+-#if defined(_MSC_VER) && (_MSC_VER < 1600)
+-#include <sys/types.h>  // for uintptr_t on x86
+-#else
++#if !defined(_MSC_VER) && (_MSC_VER < 1600)
+ #include <stdint.h>  // for uintptr_t
+ #endif
+ 
++typedef uint64_t uint64;
++typedef int64_t  int64;
++#if defined(_MSC_VER)
++// nsprpub/pr/include/obsolete/protypes.h defines these weirdly
++typedef long int32;
++typedef unsigned long uint32;
++#else
++typedef uint32_t uint32;
++typedef int32_t  int32;
++#endif
++typedef uint16_t uint16;
++typedef int16_t  int16;
++typedef uint8_t  uint8;
++typedef int8_t   int8;
++#define INT_TYPES_DEFINED 1
++
+ #ifndef GG_LONGLONG
+ #ifndef INT_TYPES_DEFINED
+ #define INT_TYPES_DEFINED
+ #ifdef COMPILER_MSVC
+ typedef unsigned __int64 uint64;
+ typedef __int64 int64;
+ #ifndef INT64_C
+ #define INT64_C(x) x##I64
+diff --git a/media/libyuv/libyuv/include/libyuv/row.h b/media/libyuv/libyuv/include/libyuv/row.h
+--- a/media/libyuv/libyuv/include/libyuv/row.h
++++ b/media/libyuv/libyuv/include/libyuv/row.h
+@@ -169,18 +169,19 @@ extern "C" {
+ #define HAS_SOBELROW_SSE2
+ #define HAS_SOBELTOPLANEROW_SSE2
+ #define HAS_SOBELXROW_SSE2
+ #define HAS_SOBELXYROW_SSE2
+ #define HAS_SOBELYROW_SSE2
+ 
+ // The following functions fail on gcc/clang 32 bit with fpic and framepointer.
+ // caveat: clangcl uses row_win.cc which works.
+-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+-    defined(_MSC_VER)
++#if !defined(MOZ_PROFILING) && \
++  (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) ||  \
++   defined(_MSC_VER))
+ // TODO(fbarchard): fix build error on android_full_debug=1
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_SSSE3
+ #endif
+ #endif
+ 
+ // The following are available on all x86 platforms, but
+ // require VS2012, clang 3.4 or gcc 4.7.
+@@ -231,18 +232,19 @@ extern "C" {
+ // Effects:
+ #define HAS_ARGBADDROW_AVX2
+ #define HAS_ARGBATTENUATEROW_AVX2
+ #define HAS_ARGBMULTIPLYROW_AVX2
+ #define HAS_ARGBSUBTRACTROW_AVX2
+ #define HAS_ARGBUNATTENUATEROW_AVX2
+ #define HAS_BLENDPLANEROW_AVX2
+ 
+-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+-    defined(_MSC_VER)
++#if !defined(MOZ_PROFILING) && \
++  (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) ||  \
++   defined(_MSC_VER))
+ // TODO(fbarchard): fix build error on android_full_debug=1
+ // https://code.google.com/p/libyuv/issues/detail?id=517
+ #define HAS_I422ALPHATOARGBROW_AVX2
+ #endif
+ #endif
+ 
+ // The following are available for AVX2 Visual C and clangcl 32 bit:
+ // TODO(fbarchard): Port to gcc.
new file mode 100644
--- /dev/null
+++ b/media/libyuv/make_mjpeg_printfs_optional.patch
@@ -0,0 +1,52 @@
+# HG changeset patch
+# Parent  b079f87bb602629bb8b51979d79f825be7aef1ae
+Bug 1341513: Make disable MJPEG fprintf warnings build-time option r=sotaro
+
+diff --git a/media/libyuv/libyuv/source/mjpeg_decoder.cc b/media/libyuv/libyuv/source/mjpeg_decoder.cc
+--- a/media/libyuv/libyuv/source/mjpeg_decoder.cc
++++ b/media/libyuv/libyuv/source/mjpeg_decoder.cc
+@@ -73,18 +73,20 @@ MJpegDecoder::MJpegDecoder()
+       databuf_strides_(NULL) {
+   decompress_struct_ = new jpeg_decompress_struct;
+   source_mgr_ = new jpeg_source_mgr;
+ #ifdef HAVE_SETJMP
+   error_mgr_ = new SetJmpErrorMgr;
+   decompress_struct_->err = jpeg_std_error(&error_mgr_->base);
+   // Override standard exit()-based error handler.
+   error_mgr_->base.error_exit = &ErrorHandler;
++#ifndef DEBUG_MJPEG
+   error_mgr_->base.output_message = &OutputHandler;
+ #endif
++#endif
+   decompress_struct_->client_data = NULL;
+   source_mgr_->init_source = &init_source;
+   source_mgr_->fill_input_buffer = &fill_input_buffer;
+   source_mgr_->skip_input_data = &skip_input_data;
+   source_mgr_->resync_to_restart = &jpeg_resync_to_restart;
+   source_mgr_->term_source = &term_source;
+   jpeg_create_decompress(decompress_struct_);
+   decompress_struct_->src = source_mgr_;
+@@ -450,21 +452,22 @@ void ErrorHandler(j_common_ptr cinfo) {
+ #endif
+ 
+   SetJmpErrorMgr* mgr = reinterpret_cast<SetJmpErrorMgr*>(cinfo->err);
+   // This rewinds the call stack to the point of the corresponding setjmp()
+   // and causes it to return (for a second time) with value 1.
+   longjmp(mgr->setjmp_buffer, 1);
+ }
+ 
++#ifndef DEBUG_MJPEG
+ // Suppress fprintf warnings.
+ void OutputHandler(j_common_ptr cinfo) {
+   (void)cinfo;
+ }
+-
++#endif
+ #endif  // HAVE_SETJMP
+ 
+ void MJpegDecoder::AllocOutputBuffers(int num_outbufs) {
+   if (num_outbufs != num_outbufs_) {
+     // We could perhaps optimize this case to resize the output buffers without
+     // necessarily having to delete and recreate each one, but it's not worth
+     // it.
+     DestroyOutputBuffers();
new file mode 100644
--- /dev/null
+++ b/media/libyuv/update.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import argparse
+import os
+import re
+import shutil
+import sys
+import subprocess
+import tarfile
+import urllib
+from pprint import pprint
+from StringIO import StringIO
+
+def prepare_upstream(prefix, commit=None):
+    upstream_url = 'https://chromium.googlesource.com/libyuv/libyuv'
+    shutil.rmtree(os.path.join(base, 'libyuv/'))
+    print(upstream_url + '/+archive/' + commit + '.tar.gz')
+    urllib.urlretrieve(upstream_url + '/+archive/' + commit + '.tar.gz', 'libyuv.tar.gz')
+    tarfile.open('libyuv.tar.gz').extractall(path='libyuv')
+    os.remove(os.path.join(base, 'libyuv.tar.gz'))
+    os.chdir(base)
+    return commit
+
+def cleanup_upstream():
+    os.remove(os.path.join(base, 'libyuv/.gitignore'))
+
+def apply_patches():
+    # Patch to update gyp build files
+    os.system("patch -p3 < update_gyp.patch")
+    # Patch to fix build errors
+    os.system("patch -p3 < fix_build_errors.patch")
+    # Patch to make mjpeg printfs optional at build time
+    os.system("patch -p3 < make_mjpeg_printfs_optional.patch")
+    # Patch to allow disabling of inline ASM and AVX2 code
+    os.system("patch -p3 < allow_disabling_asm_avx2.patch")
+    # Patch to add H444ToARGB() variant
+    os.system("patch -p3 < add_H444ToARGB.patch")
+
+def update_readme(commit):
+    with open('README_MOZILLA') as f:
+        readme = f.read()
+
+    if 'The git commit ID last used to import was ' in readme:
+        new_readme = re.sub('The git commit ID last used to import was [v\.a-f0-9]+',
+            'The git commit ID last used to import was %s' % commit, readme)
+    else:
+        new_readme = "%s\n\nThe git commit ID last used to import was %s\n" % (readme, commit)
+
+    if readme != new_readme:
+        with open('README_MOZILLA', 'w') as f:
+            f.write(new_readme)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='''Update libyuv''')
+    parser.add_argument('--debug', dest='debug', action="store_true")
+    parser.add_argument('--commit', dest='commit', type=str, default='master')
+
+    args = parser.parse_args()
+
+    commit = args.commit
+    DEBUG = args.debug
+
+    base = os.path.abspath(os.curdir)
+    prefix = os.path.join(base, 'libyuv/')
+
+    commit = prepare_upstream(prefix, commit)
+
+    apply_patches()
+    update_readme(commit)
+
+    print('Patches applied; run "hg addremove --similarity 70 libyuv" before committing changes')
+
+    cleanup_upstream()
new file mode 100644
--- /dev/null
+++ b/media/libyuv/update_gyp.patch
@@ -0,0 +1,83 @@
+# HG changeset patch
+# Parent  034c99a4c2010c8d60ff52daf0a183ac1fd05777
+Bug 1341543: Update libyuv gyp file with mozilla changes r=sotaro
+
+diff --git a/media/libyuv/libyuv/libyuv.gyp b/media/libyuv/libyuv/libyuv.gyp
+--- a/media/libyuv/libyuv/libyuv.gyp
++++ b/media/libyuv/libyuv/libyuv.gyp
+@@ -47,17 +47,17 @@
+     {
+       'target_name': 'libyuv',
+       # Change type to 'shared_library' to build .so or .dll files.
+       'type': 'static_library',
+       'variables': {
+         'optimize': 'max',  # enable O2 and ltcg.
+       },
+       # Allows libyuv.a redistributable library without external dependencies.
+-      'standalone_static_library': 1,
++      # 'standalone_static_library': 1,
+       'conditions': [
+        # Disable -Wunused-parameter
+         ['clang == 1', {
+           'cflags': [
+             '-Wno-unused-parameter',
+          ],
+         }],
+         ['build_neon != 0', {
+@@ -65,39 +65,55 @@
+             'LIBYUV_NEON',
+           ],
+           'cflags!': [
+             '-mfpu=vfp',
+             '-mfpu=vfpv3',
+             '-mfpu=vfpv3-d16',
+             # '-mthumb',  # arm32 not thumb
+           ],
++          'cflags_mozilla!': [
++            '-mfpu=vfp',
++            '-mfpu=vfpv3',
++            '-mfpu=vfpv3-d16',
++          ],
+           'conditions': [
+             # Disable LTO in libyuv_neon target due to gcc 4.9 compiler bug.
+             ['clang == 0 and use_lto == 1', {
+               'cflags!': [
+                 '-flto',
+                 '-ffat-lto-objects',
+               ],
+             }],
+             # arm64 does not need -mfpu=neon option as neon is not optional
+             ['target_arch != "arm64"', {
+               'cflags': [
+                 '-mfpu=neon',
+                 # '-marm',  # arm32 not thumb
+               ],
++              'cflags_mozilla': [
++                '-mfpu=neon',
++              ],
+             }],
+           ],
+         }],
+         ['build_msa != 0', {
+           'defines': [
+             'LIBYUV_MSA',
+           ],
+         }],
+-        ['OS != "ios" and libyuv_disable_jpeg != 1', {
++        ['build_with_mozilla == 1', {
++          'defines': [
++            'HAVE_JPEG'
++          ],
++          'cflags_mozilla': [
++            '$(MOZ_JPEG_CFLAGS)',
++          ],
++        }],
++        ['OS != "ios" and libyuv_disable_jpeg != 1 and build_with_mozilla != 1', {
+           'defines': [
+             'HAVE_JPEG'
+           ],
+           'conditions': [
+             # Caveat system jpeg support may not support motion jpeg
+             [ 'use_system_libjpeg == 1', {
+               'dependencies': [
+                  '<(DEPTH)/third_party/libjpeg/libjpeg.gyp:libjpeg',