Bug 940708 - Part 2: Build webrtc in unified mode; r=gps
☠☠ backed out by 2b7739306a76 ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 18 Dec 2013 15:40:03 +0900
changeset 161593 d4aec392a165f049fb455a1936cecca91dcf140c
parent 161592 d7c9b80ab65fe155d4a0ed70e435f766567812ba
child 161594 aedcac877ab02061245a30b94b6d4f42277121da
push id25887
push userkwierso@gmail.com
push dateSat, 21 Dec 2013 02:42:17 +0000
treeherdermozilla-central@90c67da3f827 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs940708
milestone29.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 940708 - Part 2: Build webrtc in unified mode; r=gps
media/webrtc/moz.build
python/mozbuild/mozbuild/frontend/gyp_reader.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/frontend/sandbox_symbols.py
--- a/media/webrtc/moz.build
+++ b/media/webrtc/moz.build
@@ -1,35 +1,211 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 include('/build/gyp.mozbuild')
 
+webrtc_non_unified_sources = [
+    'trunk/webrtc/common_audio/vad/vad_core.c',                                  # Because of name clash in the kInitCheck variable
+    'trunk/webrtc/common_audio/vad/webrtc_vad.c',                                # Because of name clash in the kInitCheck variable
+    'trunk/webrtc/modules/audio_coding/codecs/g722/g722_decode.c',               # Because of name clash in the saturate function
+    'trunk/webrtc/modules/audio_coding/codecs/g722/g722_encode.c',               # Because of name clash in the saturate function
+    'trunk/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter.c',   # Because of name clash in the kDampFilter variable
+    'trunk/webrtc/modules/audio_coding/codecs/isac/fix/source/pitch_filter_c.c', # Because of name clash in the kDampFilter variable
+    'trunk/webrtc/modules/audio_coding/neteq4/audio_vector.cc',                  # Because of explicit template specializations
+    'trunk/webrtc/modules/audio_device/opensl/opensles_input.cc',                # Because of name clash in the kOption variable
+    'trunk/webrtc/modules/audio_device/opensl/opensles_output.cc',               # Because of name clash in the kOption variable
+    'trunk/webrtc/modules/audio_device/win/audio_device_core_win.cc',            # Because of ordering assumptions in strsafe.h
+    'trunk/webrtc/modules/audio_processing/aec/aec_core.c',                      # Because of name clash in the ComfortNoise function
+    'trunk/webrtc/modules/audio_processing/aecm/aecm_core.c',                    # Because of name clash in the ComfortNoise function
+    'trunk/webrtc/modules/audio_processing/aecm/echo_control_mobile.c',          # Because of name clash in the kInitCheck variable
+    'trunk/webrtc/modules/audio_processing/agc/analog_agc.c',                    # Because of name clash in the kInitCheck variable
+    'trunk/webrtc/modules/audio_processing/echo_cancellation_impl.cc',           # Because of name clash in the MapError function
+    'trunk/webrtc/modules/audio_processing/echo_control_mobile_impl.cc',         # Because of name clash in the MapError function
+    'trunk/webrtc/modules/audio_processing/gain_control_impl.cc',                # Because of name clash in the Handle typedef
+    'trunk/webrtc/modules/audio_processing/high_pass_filter_impl.cc',            # Because of name clash in the Handle typedef
+    'trunk/webrtc/modules/audio_processing/noise_suppression_impl.cc',           # Because of name clash in the Handle typedef
+    'trunk/webrtc/modules/video_capture/mac/qtkit/video_capture_qtkit.mm',       # Because of name clash in the nsAutoreleasePool class
+    'trunk/webrtc/modules/video_capture/mac/qtkit/video_capture_qtkit_info.mm',  # Because of name clash in the nsAutoreleasePool class
+    'trunk/webrtc/modules/video_capture/windows/device_info_ds.cc',              # Because of the MEDIASUBTYPE_HDYC variable
+    'trunk/webrtc/modules/video_capture/windows/sink_filter_ds.cc',              # Because of the MEDIASUBTYPE_HDYC variable
+]
+
 GYP_DIRS += ['trunk']
 
 GYP_DIRS['trunk'].input = 'trunk/peerconnection.gyp'
 GYP_DIRS['trunk'].variables = gyp_vars
 GYP_DIRS['trunk'].sandbox_vars['FINAL_LIBRARY'] = 'webrtc'
+GYP_DIRS['trunk'].non_unified_sources += webrtc_non_unified_sources
 
 if CONFIG['MOZ_WEBRTC_SIGNALING']:
     GYP_DIRS += ['signaling']
     GYP_DIRS['signaling'].input = 'signaling/signaling.gyp'
     GYP_DIRS['signaling'].variables = gyp_vars.copy()
     GYP_DIRS['signaling'].variables.update(
         build_for_test=0
     )
     GYP_DIRS['signaling'].sandbox_vars['FINAL_LIBRARY'] = 'xul'
+    signaling_non_unified_sources = [
+        'signaling/src/sipcc/core/gsm/fim.c',        # Because of name clash in the logTag variable
+        'signaling/src/sipcc/core/gsm/gsm_sdp.c',    # Because of name clash in the logTag variable
+        'signaling/src/sipcc/core/gsm/lsm.c',        # Because of name clash in the logTag variable
+        'signaling/src/sipcc/core/sdp/sdp_base64.c', # Because of name clash with the macro PADDING
+    ]
+    # These files cannot be built in unified mode because they force NSPR logging.
+    signaling_non_unified_sources_2 = [
+        'signaling/src/callcontrol/CallControlManagerImpl.cpp',
+        'signaling/src/common/browser_logging/CSFLog.cpp',
+        'signaling/src/media-conduit/AudioConduit.cpp',
+        'signaling/src/media-conduit/VideoConduit.cpp',
+        'signaling/src/media/CSFAudioControlWrapper.cpp',
+        'signaling/src/media/CSFVideoControlWrapper.cpp',
+        'signaling/src/media/VcmSIPCCBinding.cpp',
+        'signaling/src/mediapipeline/MediaPipeline.cpp',
+        'signaling/src/mediapipeline/SrtpFlow.cpp',
+        'signaling/src/peerconnection/MediaStreamList.cpp',
+        'signaling/src/peerconnection/PeerConnectionCtx.cpp',
+        'signaling/src/peerconnection/PeerConnectionImpl.cpp',
+        'signaling/src/peerconnection/PeerConnectionMedia.cpp',
+        'signaling/src/sipcc/core/ccapp/call_logger.c',
+        'signaling/src/sipcc/core/ccapp/capability_set.c',
+        'signaling/src/sipcc/core/ccapp/cc_call_feature.c',
+        'signaling/src/sipcc/core/ccapp/cc_config.c',
+        'signaling/src/sipcc/core/ccapp/cc_device_feature.c',
+        'signaling/src/sipcc/core/ccapp/cc_device_manager.c',
+        'signaling/src/sipcc/core/ccapp/cc_info.c',
+        'signaling/src/sipcc/core/ccapp/cc_service.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_call.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_call_info.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_config.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_device.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_device_info.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_feature_info.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_line_info.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_service.c',
+        'signaling/src/sipcc/core/ccapp/ccapi_snapshot.c',
+        'signaling/src/sipcc/core/ccapp/ccapp_task.c',
+        'signaling/src/sipcc/core/ccapp/ccprovider.c',
+        'signaling/src/sipcc/core/ccapp/conf_roster.c',
+        'signaling/src/sipcc/core/common/cfgfile_utils.c',
+        'signaling/src/sipcc/core/common/config_api.c',
+        'signaling/src/sipcc/core/common/config_parser.c',
+        'signaling/src/sipcc/core/common/init.c',
+        'signaling/src/sipcc/core/common/logger.c',
+        'signaling/src/sipcc/core/common/misc.c',
+        'signaling/src/sipcc/core/common/plat.c',
+        'signaling/src/sipcc/core/common/platform_api.c',
+        'signaling/src/sipcc/core/common/prot_configmgr.c',
+        'signaling/src/sipcc/core/common/resource_manager.c',
+        'signaling/src/sipcc/core/common/subscription_handler.c',
+        'signaling/src/sipcc/core/common/ui.c',
+        'signaling/src/sipcc/core/gsm/ccapi.c',
+        'signaling/src/sipcc/core/gsm/ccapi_strings.c',
+        'signaling/src/sipcc/core/gsm/dcsm.c',
+        'signaling/src/sipcc/core/gsm/fsm.c',
+        'signaling/src/sipcc/core/gsm/fsmb2bcnf.c',
+        'signaling/src/sipcc/core/gsm/fsmcac.c',
+        'signaling/src/sipcc/core/gsm/fsmcnf.c',
+        'signaling/src/sipcc/core/gsm/fsmdef.c',
+        'signaling/src/sipcc/core/gsm/fsmxfr.c',
+        'signaling/src/sipcc/core/gsm/gsm.c',
+        'signaling/src/sipcc/core/gsm/gsm_sdp_crypto.c',
+        'signaling/src/sipcc/core/gsm/media_cap_tbl.c',
+        'signaling/src/sipcc/core/gsm/sm.c',
+        'signaling/src/sipcc/core/gsm/subapi.c',
+        'signaling/src/sipcc/core/sdp/ccsdp.c',
+        'signaling/src/sipcc/core/sdp/sdp_access.c',
+        'signaling/src/sipcc/core/sdp/sdp_attr.c',
+        'signaling/src/sipcc/core/sdp/sdp_attr_access.c',
+        'signaling/src/sipcc/core/sdp/sdp_config.c',
+        'signaling/src/sipcc/core/sdp/sdp_main.c',
+        'signaling/src/sipcc/core/sdp/sdp_services_unix.c',
+        'signaling/src/sipcc/core/sdp/sdp_token.c',
+        'signaling/src/sipcc/core/sdp/sdp_utils.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_callinfo.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_cc.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_common_util.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_core.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_debug.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_info.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_messaging.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_platform.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_platform_tcp.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_platform_timers.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_platform_tls.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_platform_udp.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_pmh.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_publish.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_register.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_reldev.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_sdp.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_spi_utils.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_subsmanager.c',
+        'signaling/src/sipcc/core/sipstack/ccsip_task.c',
+        'signaling/src/sipcc/core/sipstack/httpish.c',
+        'signaling/src/sipcc/core/sipstack/sip_common_regmgr.c',
+        'signaling/src/sipcc/core/sipstack/sip_common_transport.c',
+        'signaling/src/sipcc/core/sipstack/sip_csps_transport.c',
+        'signaling/src/sipcc/core/sipstack/sip_interface_regmgr.c',
+        'signaling/src/sipcc/core/sipstack/sip_platform_task.c',
+        'signaling/src/sipcc/core/sipstack/sip_platform_win32_task.c',
+        'signaling/src/sipcc/core/src-common/configapp.c',
+        'signaling/src/sipcc/core/src-common/dialplan.c',
+        'signaling/src/sipcc/core/src-common/dialplanint.c',
+        'signaling/src/sipcc/core/src-common/digcalc.c',
+        'signaling/src/sipcc/core/src-common/kpml_common_util.c',
+        'signaling/src/sipcc/core/src-common/kpmlmap.c',
+        'signaling/src/sipcc/core/src-common/misc_apps_task.c',
+        'signaling/src/sipcc/core/src-common/pres_sub_not_handler.c',
+        'signaling/src/sipcc/core/src-common/publish_int.c',
+        'signaling/src/sipcc/core/src-common/string_lib.c',
+        'signaling/src/sipcc/core/src-common/util_ios_queue.c',
+        'signaling/src/sipcc/cpr/android/cpr_android_init.c',
+        'signaling/src/sipcc/cpr/android/cpr_android_stdio.c',
+        'signaling/src/sipcc/cpr/android/cpr_android_timers_using_select.c',
+        'signaling/src/sipcc/cpr/darwin/cpr_darwin_init.c',
+        'signaling/src/sipcc/cpr/darwin/cpr_darwin_stdio.c',
+        'signaling/src/sipcc/cpr/darwin/cpr_darwin_timers_using_select.c',
+        'signaling/src/sipcc/cpr/linux/cpr_linux_init.c',
+        'signaling/src/sipcc/cpr/linux/cpr_linux_ipc.c',
+        'signaling/src/sipcc/cpr/linux/cpr_linux_stdio.c',
+        'signaling/src/sipcc/cpr/linux/cpr_linux_timers_using_select.c',
+        'signaling/src/sipcc/cpr/win32/cpr_win_stdio.c',
+        'signaling/src/sipcc/cpr/win32/cpr_win_timers.c',
+        'signaling/src/sipcc/plat/common/dns_utils.c',
+        'signaling/src/sipcc/plat/csf2g/reset_api.c',
+        'signaling/src/sipcc/plat/win32/dns_utils.c',
+        'signaling/src/sipcc/plat/win32/mystub.c',
+        'signaling/src/sipcc/plat/win32/plat_api_stub.c',
+        'signaling/src/sipcc/plat/win32/plat_api_win.c',
+        'signaling/src/softphonewrapper/CC_SIPCCCall.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCCallInfo.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCCallServerInfo.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCDevice.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCDeviceInfo.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCFeatureInfo.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCLine.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCLineInfo.cpp',
+        'signaling/src/softphonewrapper/CC_SIPCCService.cpp',
+        'signaling/src/softphonewrapper/ccapi_plat_api_impl.cpp',
+    ]
+
+    GYP_DIRS['signaling'].non_unified_sources += signaling_non_unified_sources
+    GYP_DIRS['signaling'].non_unified_sources += signaling_non_unified_sources_2
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'gonk':
     GYP_DIRS += ['trunk/testing']
     GYP_DIRS['trunk/testing'].input = 'trunk/testing/gtest.gyp'
     GYP_DIRS['trunk/testing'].variables = gyp_vars
+    GYP_DIRS['trunk/testing'].non_unified_sources += webrtc_non_unified_sources
 
     if CONFIG['MOZ_WEBRTC_SIGNALING']:
         GYP_DIRS += ['signalingtest']
         GYP_DIRS['signalingtest'].input = 'signaling/signaling.gyp'
         GYP_DIRS['signalingtest'].variables = gyp_vars.copy()
         GYP_DIRS['signalingtest'].variables.update(
             build_for_test=1
         )
+        GYP_DIRS['signalingtest'].non_unified_sources += signaling_non_unified_sources
+        GYP_DIRS['signalingtest'].non_unified_sources += signaling_non_unified_sources_2
--- a/python/mozbuild/mozbuild/frontend/gyp_reader.py
+++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py
@@ -9,16 +9,17 @@ import time
 import os
 import mozpack.path as mozpath
 from mozpack.files import FileFinder
 from .sandbox import (
     alphabetical_sorted,
     GlobalNamespace,
 )
 from .sandbox_symbols import VARIABLES
+from .reader import SandboxValidationError
 
 # Define this module as gyp.generator.mozbuild so that gyp can use it
 # as a generator under the name "mozbuild".
 sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__]
 
 # build/gyp_chromium does this:
 #   script_dir = os.path.dirname(os.path.realpath(__file__))
 #   chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
@@ -66,27 +67,28 @@ class GypSandbox(GlobalNamespace):
 
 
 def encode(value):
     if isinstance(value, unicode):
         return value.encode('utf-8')
     return value
 
 
-def read_from_gyp(config, path, output, vars):
+def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
     """Read a gyp configuration and emits GypSandboxes for the backend to
     process.
 
     config is a ConfigEnvironment, path is the path to a root gyp configuration
     file, output is the base path under which the objdir for the various gyp
     dependencies will be, and vars a dict of variables to pass to the gyp
     processor.
     """
 
     time_start = time.time()
+    all_sources = set()
 
     # gyp expects plain str instead of unicode. The frontend code gives us
     # unicode strings, so convert them.
     path = encode(path)
     str_vars = dict((name, encode(value)) for name, value in vars.items())
 
     params = {
         b'parallel': False,
@@ -155,21 +157,27 @@ def read_from_gyp(config, path, output, 
             sandbox['FORCE_STATIC_LIB'] = True
             # Remove leading 'lib' from the target_name if any, and use as
             # library name.
             name = spec['target_name']
             if name.startswith('lib'):
                 name = name[3:]
             # The sandbox expects an unicode string.
             sandbox['LIBRARY_NAME'] = name.decode('utf-8')
+            # gyp files contain headers in sources lists.
+            sources = set(mozpath.normpath(mozpath.join(sandbox['SRCDIR'], f))
+                for f in spec.get('sources', [])
+                if mozpath.splitext(f)[-1] != '.h')
+
+            unified_sources = sources - non_unified_sources
+            sources -= unified_sources
+            all_sources |= sources
             # The sandbox expects alphabetical order when adding sources
-            sources = alphabetical_sorted(spec.get('sources', []))
-            # gyp files contain headers in sources lists.
-            sandbox['SOURCES'] = \
-                [f for f in sources if mozpath.splitext(f)[-1] != '.h']
+            sandbox['SOURCES'] = alphabetical_sorted(sources)
+            sandbox['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
 
             for define in target_conf.get('defines', []):
                 if '=' in define:
                     name, value = define.split('=', 1)
                     sandbox['DEFINES'][name] = value
                 else:
                     sandbox['DEFINES'][define] = True
 
@@ -183,8 +191,12 @@ def read_from_gyp(config, path, output, 
             # Ignore other types than static_library because we don't have
             # anything using them, and we're not testing them. They can be
             # added when that becomes necessary.
             raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
 
         sandbox.execution_time = time.time() - time_start
         yield sandbox
         time_start = time.time()
+#    remainder = non_unified_sources - all_sources
+#    if remainder:
+#        raise SandboxValidationError('%s defined as non_unified_source, but is '
+#            'not defined as a source' % ', '.join(remainder))
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -754,30 +754,38 @@ class BuildReader(object):
 
         curdir = mozpath.dirname(path)
 
         gyp_sandboxes = []
         for target_dir in sandbox['GYP_DIRS']:
             gyp_dir = sandbox['GYP_DIRS'][target_dir]
             for v in ('input', 'variables'):
                 if not getattr(gyp_dir, v):
-                    raise Exception('Missing value for GYP_DIRS["%s"].%s' %
-                        (target_dir, v))
+                    raise SandboxValidationError('Missing value for '
+                        'GYP_DIRS["%s"].%s' % (target_dir, v))
 
             # The make backend assumes sandboxes for sub-directories are
             # emitted after their parent, so accumulate the gyp sandboxes.
             # We could emit the parent sandbox before processing gyp
             # configuration, but we need to add the gyp objdirs to that sandbox
             # first.
             from .gyp_reader import read_from_gyp
+            non_unified_sources = set()
+            for s in gyp_dir.non_unified_sources:
+                source = mozpath.normpath(mozpath.join(curdir, s))
+                if not os.path.exists(source):
+                    raise SandboxValidationError('Cannot find %s referenced '
+                        'from %s' % (source, path))
+                non_unified_sources.add(source)
             for gyp_sandbox in read_from_gyp(self.config,
                                              mozpath.join(curdir, gyp_dir.input),
                                              mozpath.join(sandbox['OBJDIR'],
                                                           target_dir),
-                                             gyp_dir.variables):
+                                             gyp_dir.variables,
+                                             non_unified_sources = non_unified_sources):
                 gyp_sandbox.update(gyp_dir.sandbox_vars)
                 gyp_sandboxes.append(gyp_sandbox)
 
         # Add the gyp subdirectories to DIRS. We don't care about trying to
         # place some of them in PARALLEL_DIRS because they're only going to be
         # relevant for the compile and libs tiers. The compile tier is already
         # parallelized, and the libs tier is always serialized, and will remain
         # so until the library linking operations are moved out of it, at which
--- a/python/mozbuild/mozbuild/frontend/sandbox_symbols.py
+++ b/python/mozbuild/mozbuild/frontend/sandbox_symbols.py
@@ -576,30 +576,34 @@ VARIABLES = {
         result is dist/xpi-stage/$(XPI_NAME). If DIST_SUBDIR is present, then
         the $(DIST_SUBDIR) directory of the otherwise default value is used.
         """, 'libs'),
 
     'GYP_DIRS': (StrictOrderingOnAppendListWithFlagsFactory({
             'variables': dict,
             'input': unicode,
             'sandbox_vars': dict,
+            'non_unified_sources': StrictOrderingOnAppendList,
         }), list,
         """Defines a list of object directories handled by gyp configurations.
 
         Elements of this list give the relative object directory. For each
         element of the list, GYP_DIRS may be accessed as a dictionary
         (GYP_DIRS[foo]). The object this returns has attributes that need to be
         set to further specify gyp processing:
             - input, gives the path to the root gyp configuration file for that
               object directory.
             - variables, a dictionary containing variables and values to pass
               to the gyp processor.
             - sandbox_vars, a dictionary containing variables and values to
               pass to the mozbuild processor on top of those derived from gyp
               configuration.
+            - non_unified_sources, a list containing sources files, relative to
+              the current moz.build, that should be excluded from source file
+              unification.
 
         Typical use looks like:
             GYP_DIRS += ['foo', 'bar']
             GYP_DIRS['foo'].input = 'foo/foo.gyp'
             GYP_DIRS['foo'].variables = {
                 'foo': 'bar',
                 (...)
             }