author | Wes Kocher <wkocher@mozilla.com> |
Wed, 28 Dec 2016 16:08:44 -0800 | |
changeset 327474 | 143bb4b9249e528e658f6ccc449991794b8675f8 |
parent 327473 | b882b98ee9ddba735a0a3a57623e0b9d9fc7742a (current diff) |
parent 327472 | fce32b66735c5e444f6e2025789ffab1aa1663e3 (diff) |
child 327475 | 35b4baa410113617d75672bc32476c14253064b9 |
child 327512 | 9d2a7eed81cd8f14ecdae2bf8f19cf1cb017a169 |
child 327513 | ff2f15ead830ee5aab1ba295343011d2c9f0f4a9 |
child 342082 | b0853f9cf0b1e7fbd54b5e9a57c20d859e8ccc06 |
push id | 35500 |
push user | kwierso@gmail.com |
push date | Thu, 29 Dec 2016 00:20:28 +0000 |
treeherder | autoland@35b4baa41011 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | merge |
milestone | 53.0a1 |
first release with | nightly linux32
143bb4b9249e
/
53.0a1
/
20161229030206
/
files
nightly linux64
143bb4b9249e
/
53.0a1
/
20161229030206
/
files
nightly mac
143bb4b9249e
/
53.0a1
/
20161229030206
/
files
nightly win32
143bb4b9249e
/
53.0a1
/
20161229030206
/
files
nightly win64
143bb4b9249e
/
53.0a1
/
20161229030206
/
files
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
releases | nightly linux32
53.0a1
/
20161229030206
/
pushlog to previous
nightly linux64
53.0a1
/
20161229030206
/
pushlog to previous
nightly mac
53.0a1
/
20161229030206
/
pushlog to previous
nightly win32
53.0a1
/
20161229030206
/
pushlog to previous
nightly win64
53.0a1
/
20161229030206
/
pushlog to previous
|
--- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -50,23 +50,16 @@ tasks: The task that creates all of the other tasks in the task graph workerType: "gecko-decision" provisionerId: "aws-provisioner-v1" tags: createdForUser: {{owner}} - scopes: - # Bug 1269443: cache scopes, etc. must be listed explicitly - - "docker-worker:cache:level-{{level}}-*" - - "docker-worker:cache:tooltool-cache" - # mozilla-taskcluster will append the appropriate assume:repo:<repo> - # scope here. - routes: - "index.gecko.v2.{{project}}.latest.firefox.decision" - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}" - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}" payload: env: # checkout-gecko uses these to check out the source; the inputs
--- a/CLOBBER +++ b/CLOBBER @@ -17,9 +17,9 @@ # # Modifying this file will now automatically clobber the buildbot machines \o/ # # Are you updating CLOBBER because you think it's needed for your WebIDL # changes to stick? As of bug 928195, this shouldn't be necessary! Please # don't change CLOBBER for WebIDL changes any more. -Bug 1322938 needs a clobber for test_lowDiskSpace.html on Android +Bug 1250356 Setting CLOBBER out caution landing a huge patchset with gyp changes
--- a/browser/base/content/sanitize.js +++ b/browser/base/content/sanitize.js @@ -152,19 +152,17 @@ Sanitizer.prototype = { seenError = true; console.error("Error sanitizing " + name, ex); }; // Array of objects in form { name, promise }. // `name` is the item's name and `promise` may be a promise, if the // sanitization is asynchronous, or the function return value, otherwise. let handles = []; - for (let itemName of itemsToClear) { - // Workaround for bug 449811. - let name = itemName; + for (let name of itemsToClear) { let item = this.items[name]; try { // Catch errors here, so later we can just loop through these. handles.push({ name, promise: item.clear(range) .then(() => progress[name] = "cleared", ex => annotateError(name, ex)) });
--- a/browser/components/migration/MigrationUtils.jsm +++ b/browser/components/migration/MigrationUtils.jsm @@ -289,31 +289,27 @@ this.MigratorPrototype = { let notify = function(aMsg, aItemType) { Services.obs.notifyObservers(null, aMsg, aItemType); }; for (let resourceType of Object.keys(MigrationUtils._importQuantities)) { MigrationUtils._importQuantities[resourceType] = 0; } notify("Migration:Started"); - for (let [key, value] of resourcesGroupedByItems) { - // Workaround bug 449811. - let migrationType = key, itemResources = value; + for (let [migrationType, itemResources] of resourcesGroupedByItems) { notify("Migration:ItemBeforeMigrate", migrationType); let itemSuccess = false; for (let res of itemResources) { - // Workaround bug 449811. - let resource = res; - maybeStartTelemetryStopwatch(migrationType, resource); + maybeStartTelemetryStopwatch(migrationType, res); let completeDeferred = PromiseUtils.defer(); let resourceDone = function(aSuccess) { - maybeStopTelemetryStopwatch(migrationType, resource); - itemResources.delete(resource); + maybeStopTelemetryStopwatch(migrationType, res); + itemResources.delete(res); itemSuccess |= aSuccess; if (itemResources.size == 0) { notify(itemSuccess ? "Migration:ItemAfterMigrate" : "Migration:ItemError", migrationType); resourcesGroupedByItems.delete(migrationType); if (resourcesGroupedByItems.size == 0) { collectQuantityTelemetry(); @@ -321,17 +317,17 @@ this.MigratorPrototype = { } } completeDeferred.resolve(); }; // If migrate throws, an error occurred, and the callback // (itemMayBeDone) might haven't been called. try { - resource.migrate(resourceDone); + res.migrate(resourceDone); } catch (ex) { Cu.reportError(ex); resourceDone(false); } // Certain resources must be ran sequentially or they could fail, // for example bookmarks and history (See bug 1272652).
--- a/browser/components/originattributes/test/browser/browser.ini +++ b/browser/components/originattributes/test/browser/browser.ini @@ -64,8 +64,9 @@ support-files = [browser_firstPartyIsolation.js] [browser_localStorageIsolation.js] [browser_blobURLIsolation.js] [browser_imageCacheIsolation.js] [browser_sharedworker.js] [browser_httpauth.js] [browser_clientAuth.js] [browser_cacheAPI.js] +[browser_permissions.js]
new file mode 100644 --- /dev/null +++ b/browser/components/originattributes/test/browser/browser_permissions.js @@ -0,0 +1,44 @@ +/** + * Bug 1282655 - Test if site permissions are universal across origin attributes. + * + * This test is testing the cookie "permission" for a specific URI. + */ + +const TEST_PAGE = "http://example.net"; +const uri = Services.io.newURI(TEST_PAGE, null, null); + +function disableCookies() { + Services.cookies.removeAll(); + Services.perms.add(uri, "cookie", Services.perms.DENY_ACTION); +} + +function ensureCookieNotSet(aBrowser) { + ContentTask.spawn(aBrowser, null, function*() { + content.document.cookie = "key=value"; + is(content.document.cookie, "", "Setting/reading cookies should be disabled" + + " for this domain for all origin attribute combinations."); + }); +} + +IsolationTestTools.runTests(TEST_PAGE, ensureCookieNotSet, () => true, + disableCookies); + +function enableCookies() { + Services.cookies.removeAll(); + Services.perms.add(uri, "cookie", Services.perms.ALLOW_ACTION); +} + +function ensureCookieSet(aBrowser) { + ContentTask.spawn(aBrowser, null, function() { + content.document.cookie = "key=value"; + is(content.document.cookie, "key=value", "Setting/reading cookies should be" + + " enabled for this domain for all origin attribute combinations."); + }); +} + +IsolationTestTools.runTests(TEST_PAGE, ensureCookieSet, () => true, + enableCookies); + +registerCleanupFunction(() => { + Services.cookies.removeAll(); +});
--- a/build/gyp.mozbuild +++ b/build/gyp.mozbuild @@ -34,16 +34,18 @@ gyp_vars.update({ 'build_json': 0, 'build_icu': 0, 'build_opus': 0, 'libyuv_dir': '/media/libyuv', 'yuv_disable_avx2': 0 if CONFIG['HAVE_X86_AVX2'] else 1, # don't use openssl 'use_openssl': 0, + 'debug': 1 if CONFIG['DEBUG'] else 0, + 'use_x11': 1 if CONFIG['MOZ_X11'] else 0, 'use_glib': 1 if CONFIG['GLIB_LIBS'] else 0, # turn off mandatory use of NEON and instead use NEON detection 'arm_neon': 0, 'arm_neon_optional': 1, 'moz_widget_toolkit_gonk': 0,
--- a/devtools/client/responsive.html/browser/tunnel.js +++ b/devtools/client/responsive.html/browser/tunnel.js @@ -483,23 +483,21 @@ MessageManagerTunnel.prototype = { if (!this.inner.frameLoader) { return null; } return this.inner.frameLoader.messageManager; }, init() { for (let method of this.PASS_THROUGH_METHODS) { - // Workaround bug 449811 to ensure a fresh binding each time through the loop - let _method = method; - this[_method] = (...args) => { + this[method] = (...args) => { if (!this.outerParentMM) { return null; } - return this.outerParentMM[_method](...args); + return this.outerParentMM[method](...args); }; } for (let name of this.INNER_TO_OUTER_MESSAGES) { this.innerParentMM.addMessageListener(name, this); this.tunneledMessageNames.add(name); } @@ -533,23 +531,21 @@ MessageManagerTunnel.prototype = { for (let name of this.tunneledMessageNames) { this.innerParentMM.removeMessageListener(name, this); } // Some objects may have cached this tunnel as the messageManager for a frame. To // ensure it keeps working after tunnel close, rewrite the overidden methods as pass // through methods. for (let method of this.OVERRIDDEN_METHODS) { - // Workaround bug 449811 to ensure a fresh binding each time through the loop - let _method = method; - this[_method] = (...args) => { + this[method] = (...args) => { if (!this.outerParentMM) { return null; } - return this.outerParentMM[_method](...args); + return this.outerParentMM[method](...args); }; } }, observe(subject, topic, data) { if (topic != "message-manager-close") { return; }
--- a/dom/base/Element.cpp +++ b/dom/base/Element.cpp @@ -2141,27 +2141,30 @@ Element::DispatchClickEvent(nsPresContex NS_PRECONDITION(aSourceEvent, "Must have source event"); NS_PRECONDITION(aStatus, "Null out param?"); WidgetMouseEvent event(aSourceEvent->IsTrusted(), eMouseClick, aSourceEvent->mWidget, WidgetMouseEvent::eReal); event.mRefPoint = aSourceEvent->mRefPoint; uint32_t clickCount = 1; float pressure = 0; + uint32_t pointerId = 0; // Use the default value here. uint16_t inputSource = 0; WidgetMouseEvent* sourceMouseEvent = aSourceEvent->AsMouseEvent(); if (sourceMouseEvent) { clickCount = sourceMouseEvent->mClickCount; pressure = sourceMouseEvent->pressure; + pointerId = sourceMouseEvent->pointerId; inputSource = sourceMouseEvent->inputSource; } else if (aSourceEvent->mClass == eKeyboardEventClass) { inputSource = nsIDOMMouseEvent::MOZ_SOURCE_KEYBOARD; } event.pressure = pressure; event.mClickCount = clickCount; + event.pointerId = pointerId; event.inputSource = inputSource; event.mModifiers = aSourceEvent->mModifiers; if (aExtraEventFlags) { // Be careful not to overwrite existing flags! event.mFlags.Union(*aExtraEventFlags); } return DispatchEvent(aPresContext, &event, aTarget, aFullDispatch, aStatus);
--- a/dom/base/nsContentUtils.cpp +++ b/dom/base/nsContentUtils.cpp @@ -8235,16 +8235,17 @@ nsContentUtils::SendMouseEvent(const nsC float aY, int32_t aButton, int32_t aButtons, int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, + uint32_t aIdentifier, bool aToWindow, bool *aPreventDefault, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized) { nsPoint offset; nsCOMPtr<nsIWidget> widget = GetWidget(aPresShell, &offset); if (!widget) @@ -8282,16 +8283,17 @@ nsContentUtils::SendMouseEvent(const nsC } WidgetMouseEvent event(true, msg, widget, aIsWidgetEventSynthesized ? WidgetMouseEvent::eSynthesized : WidgetMouseEvent::eReal, contextMenuKey ? WidgetMouseEvent::eContextMenuKey : WidgetMouseEvent::eNormal); + event.pointerId = aIdentifier; event.mModifiers = GetWidgetModifiers(aModifiers); event.button = aButton; event.buttons = aButtons != nsIDOMWindowUtils::MOUSE_BUTTONS_NOT_SPECIFIED ? aButtons : msg == eMouseUp ? 0 : GetButtonsFlagForButton(aButton); event.pressure = aPressure; event.inputSource = aInputSourceArg; event.mClickCount = aClickCount;
--- a/dom/base/nsContentUtils.h +++ b/dom/base/nsContentUtils.h @@ -2581,16 +2581,17 @@ public: float aY, int32_t aButton, int32_t aButtons, int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, + uint32_t aIdentifier, bool aToWindow, bool *aPreventDefault, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized); static void FirePageShowEvent(nsIDocShellTreeItem* aItem, mozilla::dom::EventTarget* aChromeEventHandler, bool aFireIfShowing);
--- a/dom/base/nsDOMWindowUtils.cpp +++ b/dom/base/nsDOMWindowUtils.cpp @@ -637,22 +637,26 @@ nsDOMWindowUtils::SendMouseEvent(const n int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized, int32_t aButtons, + uint32_t aIdentifier, uint8_t aOptionalArgCount, bool *aPreventDefault) { return SendMouseEventCommon(aType, aX, aY, aButton, aClickCount, aModifiers, aIgnoreRootScrollFrame, aPressure, - aInputSourceArg, false, aPreventDefault, + aInputSourceArg, + aOptionalArgCount >= 7 ? + aIdentifier : DEFAULT_MOUSE_POINTER_ID, + false, aPreventDefault, aOptionalArgCount >= 4 ? aIsDOMEventSynthesized : true, aOptionalArgCount >= 5 ? aIsWidgetEventSynthesized : false, aOptionalArgCount >= 6 ? aButtons : MOUSE_BUTTONS_NOT_SPECIFIED); } @@ -664,24 +668,28 @@ nsDOMWindowUtils::SendMouseEventToWindow int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized, int32_t aButtons, + uint32_t aIdentifier, uint8_t aOptionalArgCount) { PROFILER_LABEL("nsDOMWindowUtils", "SendMouseEventToWindow", js::ProfileEntry::Category::EVENTS); return SendMouseEventCommon(aType, aX, aY, aButton, aClickCount, aModifiers, aIgnoreRootScrollFrame, aPressure, - aInputSourceArg, true, nullptr, + aInputSourceArg, + aOptionalArgCount >= 7 ? + aIdentifier : DEFAULT_MOUSE_POINTER_ID, + true, nullptr, aOptionalArgCount >= 4 ? aIsDOMEventSynthesized : true, aOptionalArgCount >= 5 ? aIsWidgetEventSynthesized : false, aOptionalArgCount >= 6 ? aButtons : MOUSE_BUTTONS_NOT_SPECIFIED); } @@ -690,27 +698,28 @@ nsDOMWindowUtils::SendMouseEventCommon(c float aX, float aY, int32_t aButton, int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, + uint32_t aPointerId, bool aToWindow, bool *aPreventDefault, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized, int32_t aButtons) { nsCOMPtr<nsIPresShell> presShell = GetPresShell(); return nsContentUtils::SendMouseEvent(presShell, aType, aX, aY, aButton, aButtons, aClickCount, aModifiers, aIgnoreRootScrollFrame, aPressure, - aInputSourceArg, aToWindow, aPreventDefault, aIsDOMEventSynthesized, - aIsWidgetEventSynthesized); + aInputSourceArg, aPointerId, aToWindow, aPreventDefault, + aIsDOMEventSynthesized, aIsWidgetEventSynthesized); } NS_IMETHODIMP nsDOMWindowUtils::SendPointerEventCommon(const nsAString& aType, float aX, float aY, int32_t aButton, int32_t aClickCount,
--- a/dom/base/nsDOMWindowUtils.h +++ b/dom/base/nsDOMWindowUtils.h @@ -86,16 +86,17 @@ protected: float aX, float aY, int32_t aButton, int32_t aClickCount, int32_t aModifiers, bool aIgnoreRootScrollFrame, float aPressure, unsigned short aInputSourceArg, + uint32_t aIdentifier, bool aToWindow, bool *aPreventDefault, bool aIsDOMEventSynthesized, bool aIsWidgetEventSynthesized, int32_t aButtons); NS_IMETHOD SendPointerEventCommon(const nsAString& aType, float aX,
--- a/dom/canvas/test/captureStream_common.js +++ b/dom/canvas/test/captureStream_common.js @@ -141,18 +141,26 @@ CaptureStreamTestHelper.prototype = { * Returns a promise that resolves when the top left pixel of |video| matches * on all channels. Use |threshold| for fuzzy matching the color on each * channel, in the range [0,255]. */ waitForPixelColor: function (video, refColor, threshold, infoString) { info("Waiting for video " + video.id + " to match [" + refColor.data.join(',') + "] - " + refColor.name + " (" + infoString + ")"); + var paintedFrames = video.mozPaintedFrames-1; return this.waitForPixel(video, 0, 0, - px => this.isPixel(px, refColor, threshold)) + px => { if (paintedFrames != video.mozPaintedFrames) { + info("Frame: " + video.mozPaintedFrames + + " IsPixel ref=" + refColor.data + + " threshold=" + threshold + + " value=" + px); + paintedFrames = video.mozPaintedFrames; + } + return this.isPixel(px, refColor, threshold); }) .then(() => ok(true, video.id + " " + infoString)); }, /* * Returns a promise that resolves after |timeout| ms of playback or when the * top left pixel of |video| becomes |refColor|. The test is failed if the * timeout is not reached. */
--- a/dom/events/EventStateManager.cpp +++ b/dom/events/EventStateManager.cpp @@ -4492,16 +4492,17 @@ EventStateManager::FireDragEnterOrExit(n nsWeakFrame& aTargetFrame) { nsEventStatus status = nsEventStatus_eIgnore; WidgetDragEvent event(aDragEvent->IsTrusted(), aMessage, aDragEvent->mWidget); event.mRefPoint = aDragEvent->mRefPoint; event.mModifiers = aDragEvent->mModifiers; event.buttons = aDragEvent->buttons; event.relatedTarget = aRelatedTarget; + event.pointerId = aDragEvent->pointerId; event.inputSource = aDragEvent->inputSource; mCurrentTargetContent = aTargetContent; if (aTargetContent != aRelatedTarget) { //XXX This event should still go somewhere!! if (aTargetContent) { EventDispatcher::Dispatch(aTargetContent, aPresContext, &event, @@ -4643,16 +4644,17 @@ EventStateManager::InitAndDispatchClickE event.mRefPoint = aEvent->mRefPoint; event.mClickCount = aEvent->mClickCount; event.mModifiers = aEvent->mModifiers; event.buttons = aEvent->buttons; event.mTime = aEvent->mTime; event.mTimeStamp = aEvent->mTimeStamp; event.mFlags.mNoContentDispatch = aNoContentDispatch; event.button = aEvent->button; + event.pointerId = aEvent->pointerId; event.inputSource = aEvent->inputSource; return aPresShell->HandleEventWithTarget(&event, aCurrentTarget, aMouseTarget, aStatus); } nsresult EventStateManager::CheckForAndDispatchClick(WidgetMouseEvent* aEvent,
--- a/dom/events/test/pointerevents/mochitest_support_external.js +++ b/dom/events/test/pointerevents/mochitest_support_external.js @@ -30,21 +30,23 @@ function turnOnPointerEvents(callback) { SpecialPowers.pushPrefEnv({ "set": [ ["dom.w3c_pointer_events.enabled", true], ["layout.css.touch_action.enabled", true] ] }, callback); } +var utils = SpecialPowers.Ci.nsIDOMWindowUtils; + // Mouse Event Helper Object var MouseEventHelper = (function() { - var utils = SpecialPowers.Ci.nsIDOMWindowUtils; - return { + MOUSE_ID: utils.DEFAULT_MOUSE_POINTER_ID, + PEN_ID: utils.DEFAULT_PEN_POINTER_ID, // State // TODO: Separate this to support mouse and pen simultaneously. BUTTONS_STATE: utils.MOUSE_BUTTONS_NO_BUTTON, // Button BUTTON_NONE: -1, // Used by test framework only. (replaced before sending) BUTTON_LEFT: utils.MOUSE_BUTTON_LEFT_BUTTON, BUTTON_MIDDLE: utils.MOUSE_BUTTON_MIDDLE_BUTTON, @@ -81,19 +83,24 @@ var MouseEventHelper = (function() { // Helper function to send MouseEvent with different parameters function sendMouseEvent(int_win, elemId, mouseEventType, params) { var elem = int_win.document.getElementById(elemId); if(!!elem) { var rect = elem.getBoundingClientRect(); var eventObj = {type: mouseEventType}; - if(params && "inputSource" in params) - eventObj.inputSource = params.inputSource; - + // Default to mouse. + eventObj.inputSource = + (params && "inputSource" in params) ? params.inputSource : + MouseEvent.MOZ_SOURCE_MOUSE; + // Compute pointerId + eventObj.id = + (eventObj.inputSource === MouseEvent.MOZ_SOURCE_MOUSE) ? MouseEventHelper.MOUSE_ID : + MouseEventHelper.PEN_ID; // Check or generate a |button| value. var isButtonEvent = mouseEventType === "mouseup" || mouseEventType === "mousedown"; // Set |button| to the default value first. eventObj.button = isButtonEvent ? MouseEventHelper.BUTTON_LEFT : MouseEventHelper.BUTTON_NONE; @@ -136,31 +143,36 @@ function sendMouseEvent(int_win, elemId, } else { is(!!elem, true, "Document should have element with id: " + elemId); } } // Touch Event Helper Object var TouchEventHelper = { // State - // TODO: Support multiple point scenarios. + TOUCH_ID: utils.DEFAULT_TOUCH_POINTER_ID, TOUCH_STATE: false, // Utils checkExitState: function() { ok(!this.TOUCH_STATE, "Mismatched touchstart/touchend caught."); } } // Helper function to send TouchEvent with different parameters +// TODO: Support multiple touch points to test more features such as +// PointerEvent.isPrimary and pinch-zoom. function sendTouchEvent(int_win, elemId, touchEventType, params) { var elem = int_win.document.getElementById(elemId); if(!!elem) { var rect = elem.getBoundingClientRect(); - var eventObj = {type: touchEventType}; + var eventObj = { + type: touchEventType, + id: TouchEventHelper.TOUCH_ID + }; // Update touch state switch(touchEventType) { case "touchstart": TouchEventHelper.TOUCH_STATE = true; // Set touch flag. break; case "touchend": TouchEventHelper.TOUCH_STATE = false; // Clear touch flag. @@ -178,40 +190,50 @@ function sendTouchEvent(int_win, elemId, is(!!elem, true, "Document should have element with id: " + elemId); } } // Helper function to run Point Event test in a new tab. function runTestInNewWindow(aFile) { var testURL = location.href.substring(0, location.href.lastIndexOf('/') + 1) + aFile; var testWindow = window.open(testURL, "_blank"); + var testDone = false; // We start testing when receiving load event. Inject the mochitest helper js // to the test case after DOM elements are constructed and before the load // event is fired. testWindow.addEventListener("DOMContentLoaded", function scriptInjector() { testWindow.removeEventListener("DOMContentLoaded", scriptInjector); const PARENT_ORIGIN = "http://mochi.test:8888/"; var e = testWindow.document.createElement('script'); e.type = 'text/javascript'; e.src = "mochitest_support_internal.js"; testWindow.document.getElementsByTagName('head')[0].appendChild(e); }); window.addEventListener("message", function(aEvent) { switch(aEvent.data.type) { case "START": + // Update constants + MouseEventHelper.MOUSE_ID = aEvent.data.message.mouseId; + MouseEventHelper.PEN_ID = aEvent.data.message.penId; + TouchEventHelper.TOUCH_ID = aEvent.data.message.touchId; + turnOnPointerEvents(() => { executeTest(testWindow); }); return; case "RESULT": - ok(aEvent.data.result, aEvent.data.message); + // Should not perform checking after SimpleTest.finish(). + if (!testDone) { + ok(aEvent.data.result, aEvent.data.message); + } return; case "FIN": + testDone = true; MouseEventHelper.checkExitState(); TouchEventHelper.checkExitState(); testWindow.close(); SimpleTest.finish(); return; } }); }
--- a/dom/events/test/pointerevents/mochitest_support_internal.js +++ b/dom/events/test/pointerevents/mochitest_support_internal.js @@ -1,43 +1,80 @@ // This file supports translating W3C tests // to tests on auto MochiTest system with minimum changes. // Author: Maksim Lebedev <alessarik@gmail.com> const PARENT_ORIGIN = "http://mochi.test:8888/"; +// Since web platform tests don't check pointerId, we have to use some heuristic +// to test them. and thus pointerIds are send to mochitest_support_external.js +// before we start sending synthesized widget events. Here, we avoid using +// default values used in Gecko to insure everything works as expected. +const POINTER_MOUSE_ID = 7; +const POINTER_PEN_ID = 8; +const POINTER_TOUCH_ID = 9; // Extend for multiple touch points if needed. + // Setup environment. addListeners(document.getElementById("target0")); addListeners(document.getElementById("target1")); // Setup communication between mochitest_support_external.js. // Function allows to initialize prerequisites before testing // and adds some callbacks to support mochitest system. -add_result_callback((aTestObj) => { +function resultCallback(aTestObj) { var message = aTestObj["name"] + " ("; message += "Get: " + JSON.stringify(aTestObj["status"]) + ", "; message += "Expect: " + JSON.stringify(aTestObj["PASS"]) + ")"; window.opener.postMessage({type: "RESULT", message: message, result: aTestObj["status"] === aTestObj["PASS"]}, PARENT_ORIGIN); -}); +} +add_result_callback(resultCallback); add_completion_callback(() => { window.opener.postMessage({type: "FIN"}, PARENT_ORIGIN); }); window.addEventListener("load", () => { - // Start testing when the document is loaded. - window.opener.postMessage({type: "START"}, PARENT_ORIGIN); + // Start testing. + var startMessage = { + type: "START", + message: { + mouseId: POINTER_MOUSE_ID, + penId: POINTER_PEN_ID, + touchId: POINTER_TOUCH_ID + } + } + window.opener.postMessage(startMessage, PARENT_ORIGIN); }); function addListeners(elem) { if(!elem) return; var All_Events = ["pointerdown","pointerup","pointercancel","pointermove","pointerover","pointerout", "pointerenter","pointerleave","gotpointercapture","lostpointercapture"]; All_Events.forEach(function(name) { elem.addEventListener(name, function(event) { console.log('('+event.type+')-('+event.pointerType+')'); + + // Perform checks only for trusted events. + if (!event.isTrusted) { + return; + } + + // Compute the desired event.pointerId from event.pointerType. + var pointerId = { + mouse: POINTER_MOUSE_ID, + pen: POINTER_PEN_ID, + touch: POINTER_TOUCH_ID + }[event.pointerType]; + + // Compare the pointerId. + resultCallback({ + name: "Mismatched event.pointerId recieved.", + status: event.pointerId, + PASS: pointerId + }); + }, false); }); }
--- a/dom/interfaces/base/nsIDOMWindowUtils.idl +++ b/dom/interfaces/base/nsIDOMWindowUtils.idl @@ -317,32 +317,35 @@ interface nsIDOMWindowUtils : nsISupport * @param aPressure touch input pressure: 0.0 -> 1.0 * @param aInputSourceArg input source, see nsIDOMMouseEvent for values, * defaults to mouse input. * @param aIsDOMEventSynthesized controls nsIDOMEvent.isSynthesized value * that helps identifying test related events, * defaults to true * @param aIsWidgetEventSynthesized controls WidgetMouseEvent.mReason value * defaults to false (WidgetMouseEvent::eReal) + * @param aIdentifier A unique identifier for the pointer causing the event, + * defaulting to nsIDOMWindowUtils::DEFAULT_MOUSE_POINTER_ID. * * returns true if the page called prevent default on this event */ [optional_argc] boolean sendMouseEvent(in AString aType, in float aX, in float aY, in long aButton, in long aClickCount, in long aModifiers, [optional] in boolean aIgnoreRootScrollFrame, [optional] in float aPressure, [optional] in unsigned short aInputSourceArg, [optional] in boolean aIsDOMEventSynthesized, [optional] in boolean aIsWidgetEventSynthesized, - [optional] in long aButtons); + [optional] in long aButtons, + [optional] in unsigned long aIdentifier); /** Synthesize a pointer event. The event types supported are: * pointerdown, pointerup, pointermove, pointerover, pointerout * * Events are sent in coordinates offset by aX and aY from the window. * * Note that additional events may be fired as a result of this call. For @@ -367,17 +370,18 @@ interface nsIDOMWindowUtils : nsISupport * @param aButton button to synthesize * @param aClickCount number of clicks that have been performed * @param aModifiers modifiers pressed, using constants defined as MODIFIER_* * @param aIgnoreRootScrollFrame whether the event should ignore viewport bounds * during dispatch * @param aPressure touch input pressure: 0.0 -> 1.0 * @param aInputSourceArg input source, see nsIDOMMouseEvent for values, * defaults to mouse input. - * @param aPointerId A unique identifier for the pointer causing the event. default is 0 + * @param aPointerId A unique identifier for the pointer causing the event, + * defaulting to nsIDOMWindowUtils::DEFAULT_MOUSE_POINTER_ID. * @param aWidth The width (magnitude on the X axis), default is 0 * @param aHeight The height (magnitude on the Y axis), default is 0 * @param aTilt The plane angle between the Y-Z plane * and the plane containing both the transducer (e.g. pen stylus) axis and the Y axis. default is 0 * @param aTiltX The plane angle between the X-Z plane * and the plane containing both the transducer (e.g. pen stylus) axis and the X axis. default is 0 * @param aIsPrimary Indicates if the pointer represents the primary pointer of this pointer type. * @param aIsSynthesized controls nsIDOMEvent.isSynthesized value @@ -453,17 +457,18 @@ interface nsIDOMWindowUtils : nsISupport in long aButton, in long aClickCount, in long aModifiers, [optional] in boolean aIgnoreRootScrollFrame, [optional] in float aPressure, [optional] in unsigned short aInputSourceArg, [optional] in boolean aIsDOMEventSynthesized, [optional] in boolean aIsWidgetEventSynthesized, - [optional] in long aButtons); + [optional] in long aButtons, + [optional] in unsigned long aIdentifier); /** The same as sendPointerEvent but ensures that the event * is dispatched to this DOM window or one of its children. */ [optional_argc] void sendPointerEventToWindow(in AString aType, in float aX, in float aY, @@ -1974,16 +1979,21 @@ interface nsIDOMWindowUtils : nsISupport readonly attribute int32_t gpuProcessPid; /** * Returns true if the given timeout ID is in the list of tracking * timeouts. */ boolean isTimeoutTracking(in unsigned long timeoutId); + // These consts are only for testing purposes. + const long DEFAULT_MOUSE_POINTER_ID = 0; + const long DEFAULT_PEN_POINTER_ID = 1; + const long DEFAULT_TOUCH_POINTER_ID = 2; + // Match WidgetMouseEventBase::buttonType. const long MOUSE_BUTTON_LEFT_BUTTON = 0; const long MOUSE_BUTTON_MIDDLE_BUTTON = 1; const long MOUSE_BUTTON_RIGHT_BUTTON = 2; // Match WidgetMouseEventBase::buttonsFlag. const long MOUSE_BUTTONS_NO_BUTTON = 0x00; const long MOUSE_BUTTONS_LEFT_BUTTON = 0x01;
--- a/dom/ipc/TabChild.cpp +++ b/dom/ipc/TabChild.cpp @@ -1553,17 +1553,18 @@ TabChild::RecvMouseEvent(const nsString& const float& aX, const float& aY, const int32_t& aButton, const int32_t& aClickCount, const int32_t& aModifiers, const bool& aIgnoreRootScrollFrame) { APZCCallbackHelper::DispatchMouseEvent(GetPresShell(), aType, CSSPoint(aX, aY), - aButton, aClickCount, aModifiers, aIgnoreRootScrollFrame, nsIDOMMouseEvent::MOZ_SOURCE_UNKNOWN); + aButton, aClickCount, aModifiers, aIgnoreRootScrollFrame, + nsIDOMMouseEvent::MOZ_SOURCE_UNKNOWN, 0 /* Use the default value here. */); return IPC_OK(); } mozilla::ipc::IPCResult TabChild::RecvRealMouseMoveEvent(const WidgetMouseEvent& aEvent, const ScrollableLayerGuid& aGuid, const uint64_t& aInputBlockId) {
--- a/dom/media/systemservices/CamerasChild.cpp +++ b/dom/media/systemservices/CamerasChild.cpp @@ -1,17 +1,16 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set sw=2 ts=8 et ft=cpp : */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "CamerasChild.h" -#include "webrtc/video_engine/include/vie_capture.h" #undef FF #include "mozilla/Assertions.h" #include "mozilla/ipc/BackgroundChild.h" #include "mozilla/ipc/PBackgroundChild.h" #include "mozilla/Logging.h" #include "mozilla/SyncRunnable.h" #include "mozilla/WeakPtr.h" @@ -340,17 +339,17 @@ CamerasChild::EnsureInitialized(CaptureE LOG(("Capture Devices: %d", dispatcher.ReturnValue())); return dispatcher.ReturnValue(); } int CamerasChild::GetCaptureCapability(CaptureEngine aCapEngine, const char* unique_idUTF8, const unsigned int capability_number, - webrtc::CaptureCapability& capability) + webrtc::VideoCaptureCapability& capability) { LOG(("GetCaptureCapability: %s %d", unique_idUTF8, capability_number)); nsCString unique_id(unique_idUTF8); nsCOMPtr<nsIRunnable> runnable = media::NewRunnableFrom([this, aCapEngine, unique_id, capability_number]() -> nsresult { if (this->SendGetCaptureCapability(aCapEngine, unique_id, capability_number)) { return NS_OK; } @@ -359,17 +358,17 @@ CamerasChild::GetCaptureCapability(Captu LockAndDispatch<> dispatcher(this, __func__, runnable); if (dispatcher.Success()) { capability = mReplyCapability; } return dispatcher.ReturnValue(); } mozilla::ipc::IPCResult -CamerasChild::RecvReplyGetCaptureCapability(const CaptureCapability& ipcCapability) +CamerasChild::RecvReplyGetCaptureCapability(const VideoCaptureCapability& ipcCapability) { LOG((__PRETTY_FUNCTION__)); MonitorAutoLock monitor(mReplyMonitor); mReceivedReply = true; mReplySuccess = true; mReplyCapability.width = ipcCapability.width(); mReplyCapability.height = ipcCapability.height(); mReplyCapability.maxFPS = ipcCapability.maxFPS(); @@ -424,33 +423,33 @@ CamerasChild::RecvReplyGetCaptureDevice( monitor.Notify(); return IPC_OK(); } int CamerasChild::AllocateCaptureDevice(CaptureEngine aCapEngine, const char* unique_idUTF8, const unsigned int unique_idUTF8Length, - int& capture_id, + int& aStreamId, const nsACString& aOrigin) { LOG((__PRETTY_FUNCTION__)); nsCString unique_id(unique_idUTF8); nsCString origin(aOrigin); nsCOMPtr<nsIRunnable> runnable = media::NewRunnableFrom([this, aCapEngine, unique_id, origin]() -> nsresult { if (this->SendAllocateCaptureDevice(aCapEngine, unique_id, origin)) { return NS_OK; } return NS_ERROR_FAILURE; }); LockAndDispatch<> dispatcher(this, __func__, runnable); if (dispatcher.Success()) { LOG(("Capture Device allocated: %d", mReplyInteger)); - capture_id = mReplyInteger; + aStreamId = mReplyInteger; } return dispatcher.ReturnValue(); } mozilla::ipc::IPCResult CamerasChild::RecvReplyAllocateCaptureDevice(const int& numdev) { @@ -476,17 +475,17 @@ CamerasChild::ReleaseCaptureDevice(Captu return NS_ERROR_FAILURE; }); LockAndDispatch<> dispatcher(this, __func__, runnable); return dispatcher.ReturnValue(); } void CamerasChild::AddCallback(const CaptureEngine aCapEngine, const int capture_id, - webrtc::ExternalRenderer* render) + FrameRelay* render) { MutexAutoLock lock(mCallbackMutex); CapturerElement ce; ce.engine = aCapEngine; ce.id = capture_id; ce.callback = render; mCallbacks.AppendElement(ce); } @@ -502,22 +501,22 @@ CamerasChild::RemoveCallback(const Captu break; } } } int CamerasChild::StartCapture(CaptureEngine aCapEngine, const int capture_id, - webrtc::CaptureCapability& webrtcCaps, - webrtc::ExternalRenderer* cb) + webrtc::VideoCaptureCapability& webrtcCaps, + FrameRelay* cb) { LOG((__PRETTY_FUNCTION__)); AddCallback(aCapEngine, capture_id, cb); - CaptureCapability capCap(webrtcCaps.width, + VideoCaptureCapability capCap(webrtcCaps.width, webrtcCaps.height, webrtcCaps.maxFPS, webrtcCaps.expectedCaptureDelay, webrtcCaps.rawType, webrtcCaps.codecType, webrtcCaps.interlaced); nsCOMPtr<nsIRunnable> runnable = media::NewRunnableFrom([this, aCapEngine, capture_id, capCap]() -> nsresult { @@ -641,28 +640,22 @@ CamerasChild::ShutdownChild() } CamerasSingleton::FakeDeviceChangeEventThread() = nullptr; } mozilla::ipc::IPCResult CamerasChild::RecvDeliverFrame(const CaptureEngine& capEngine, const int& capId, mozilla::ipc::Shmem&& shmem, - const size_t& size, - const uint32_t& time_stamp, - const int64_t& ntp_time, - const int64_t& render_time) + const VideoFrameProperties & prop) { MutexAutoLock lock(mCallbackMutex); if (Callback(capEngine, capId)) { unsigned char* image = shmem.get<unsigned char>(); - Callback(capEngine, capId)->DeliverFrame(image, size, - time_stamp, - ntp_time, render_time, - nullptr); + Callback(capEngine, capId)->DeliverFrame(image, prop); } else { LOG(("DeliverFrame called with dead callback")); } SendReleaseFrame(shmem); return IPC_OK(); } mozilla::ipc::IPCResult @@ -697,17 +690,17 @@ CamerasChild::SetFakeDeviceChangeEvents( mozilla::ipc::IPCResult CamerasChild::RecvFrameSizeChange(const CaptureEngine& capEngine, const int& capId, const int& w, const int& h) { LOG((__PRETTY_FUNCTION__)); MutexAutoLock lock(mCallbackMutex); if (Callback(capEngine, capId)) { - Callback(capEngine, capId)->FrameSizeChange(w, h, 0); + Callback(capEngine, capId)->FrameSizeChange(w, h); } else { LOG(("Frame size change with dead callback")); } return IPC_OK(); } void CamerasChild::ActorDestroy(ActorDestroyReason aWhy) @@ -741,17 +734,17 @@ CamerasChild::~CamerasChild() // get destructed immediately, and should not try to reach // the parent. ShutdownChild(); } MOZ_COUNT_DTOR(CamerasChild); } -webrtc::ExternalRenderer* CamerasChild::Callback(CaptureEngine aCapEngine, +FrameRelay* CamerasChild::Callback(CaptureEngine aCapEngine, int capture_id) { for (unsigned int i = 0; i < mCallbacks.Length(); i++) { CapturerElement ce = mCallbacks[i]; if (ce.engine == aCapEngine && ce.id == capture_id) { return ce.callback; } }
--- a/dom/media/systemservices/CamerasChild.h +++ b/dom/media/systemservices/CamerasChild.h @@ -15,33 +15,40 @@ #include "mozilla/media/DeviceChangeCallback.h" #include "mozilla/Mutex.h" #include "base/singleton.h" #include "nsCOMPtr.h" // conflicts with #include of scoped_ptr.h #undef FF #include "webrtc/common.h" -// Video Engine -#include "webrtc/video_engine/include/vie_base.h" -#include "webrtc/video_engine/include/vie_capture.h" -#include "webrtc/video_engine/include/vie_render.h" +#include "webrtc/video_renderer.h" +#include "webrtc/modules/video_capture/video_capture_defines.h" + + namespace mozilla { namespace ipc { class BackgroundChildImpl; } namespace camera { +class FrameRelay { +public: + virtual int DeliverFrame(uint8_t* buffer, + const mozilla::camera::VideoFrameProperties& props) = 0; + virtual void FrameSizeChange(unsigned int w, unsigned int h) = 0; +}; + struct CapturerElement { CaptureEngine engine; int id; - webrtc::ExternalRenderer* callback; + FrameRelay* callback; }; // Forward declaration so we can work with pointers to it. class CamerasChild; // Helper class in impl that we friend. template <class T> class LockAndDispatch; // We emulate the sync webrtc.org API with the help of singleton @@ -144,79 +151,79 @@ class CamerasChild final : public PCamer public: // We are owned by the PBackground thread only. CamerasSingleton // takes a non-owning reference. NS_INLINE_DECL_REFCOUNTING(CamerasChild) // IPC messages recevied, received on the PBackground thread // these are the actual callbacks with data - virtual mozilla::ipc::IPCResult RecvDeliverFrame(const CaptureEngine&, const int&, mozilla::ipc::Shmem&&, - const size_t&, const uint32_t&, const int64_t&, - const int64_t&) override; + virtual mozilla::ipc::IPCResult RecvDeliverFrame(const CaptureEngine&, const int&, + mozilla::ipc::Shmem&&, + const VideoFrameProperties & prop) override; virtual mozilla::ipc::IPCResult RecvFrameSizeChange(const CaptureEngine&, const int&, const int& w, const int& h) override; virtual mozilla::ipc::IPCResult RecvDeviceChange() override; virtual int AddDeviceChangeCallback(DeviceChangeCallback* aCallback) override; int SetFakeDeviceChangeEvents(); // these are response messages to our outgoing requests virtual mozilla::ipc::IPCResult RecvReplyNumberOfCaptureDevices(const int&) override; virtual mozilla::ipc::IPCResult RecvReplyNumberOfCapabilities(const int&) override; virtual mozilla::ipc::IPCResult RecvReplyAllocateCaptureDevice(const int&) override; - virtual mozilla::ipc::IPCResult RecvReplyGetCaptureCapability(const CaptureCapability& capability) override; + virtual mozilla::ipc::IPCResult RecvReplyGetCaptureCapability(const VideoCaptureCapability& capability) override; virtual mozilla::ipc::IPCResult RecvReplyGetCaptureDevice(const nsCString& device_name, const nsCString& device_id, const bool& scary) override; virtual mozilla::ipc::IPCResult RecvReplyFailure(void) override; virtual mozilla::ipc::IPCResult RecvReplySuccess(void) override; virtual void ActorDestroy(ActorDestroyReason aWhy) override; // the webrtc.org ViECapture calls are mirrored here, but with access // to a specific PCameras instance to communicate over. These also // run on the MediaManager thread int NumberOfCaptureDevices(CaptureEngine aCapEngine); int NumberOfCapabilities(CaptureEngine aCapEngine, const char* deviceUniqueIdUTF8); int ReleaseCaptureDevice(CaptureEngine aCapEngine, const int capture_id); int StartCapture(CaptureEngine aCapEngine, - const int capture_id, webrtc::CaptureCapability& capability, - webrtc::ExternalRenderer* func); + const int capture_id, webrtc::VideoCaptureCapability& capability, + FrameRelay* func); int StopCapture(CaptureEngine aCapEngine, const int capture_id); int AllocateCaptureDevice(CaptureEngine aCapEngine, const char* unique_idUTF8, const unsigned int unique_idUTF8Length, int& capture_id, const nsACString& aOrigin); int GetCaptureCapability(CaptureEngine aCapEngine, const char* unique_idUTF8, const unsigned int capability_number, - webrtc::CaptureCapability& capability); + webrtc::VideoCaptureCapability& capability); int GetCaptureDevice(CaptureEngine aCapEngine, unsigned int list_number, char* device_nameUTF8, const unsigned int device_nameUTF8Length, char* unique_idUTF8, const unsigned int unique_idUTF8Length, bool* scary = nullptr); void ShutdownAll(); int EnsureInitialized(CaptureEngine aCapEngine); - webrtc::ExternalRenderer* Callback(CaptureEngine aCapEngine, int capture_id); + FrameRelay* Callback(CaptureEngine aCapEngine, int capture_id); private: CamerasChild(); ~CamerasChild(); // Dispatch a Runnable to the PCamerasParent, by executing it on the // decidecated Cameras IPC/PBackground thread. bool DispatchToParent(nsIRunnable* aRunnable, MonitorAutoLock& aMonitor); void AddCallback(const CaptureEngine aCapEngine, const int capture_id, - webrtc::ExternalRenderer* render); + FrameRelay* render); void RemoveCallback(const CaptureEngine aCapEngine, const int capture_id); void ShutdownParent(); void ShutdownChild(); nsTArray<CapturerElement> mCallbacks; // Protects the callback arrays Mutex mCallbackMutex; @@ -233,17 +240,17 @@ private: Mutex mRequestMutex; // Hold to wait for an async response to our calls Monitor mReplyMonitor; // Async response valid? bool mReceivedReply; // Async responses data contents; bool mReplySuccess; int mReplyInteger; - webrtc::CaptureCapability mReplyCapability; + webrtc::VideoCaptureCapability mReplyCapability; nsCString mReplyDeviceName; nsCString mReplyDeviceID; bool mReplyScary; }; } // namespace camera } // namespace mozilla
--- a/dom/media/systemservices/CamerasParent.cpp +++ b/dom/media/systemservices/CamerasParent.cpp @@ -2,16 +2,17 @@ /* vim: set sw=2 ts=8 et ft=cpp : */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "CamerasParent.h" #include "MediaEngine.h" #include "MediaUtils.h" +#include "VideoFrameUtils.h" #include "mozilla/Assertions.h" #include "mozilla/Unused.h" #include "mozilla/Services.h" #include "mozilla/Logging.h" #include "mozilla/ipc/BackgroundParent.h" #include "mozilla/ipc/PBackgroundParent.h" #include "mozilla/Preferences.h" @@ -23,34 +24,36 @@ #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #if defined(_WIN32) #include <process.h> #define getpid() _getpid() #endif #undef LOG +#undef LOG_VERBOSE #undef LOG_ENABLED mozilla::LazyLogModule gCamerasParentLog("CamerasParent"); #define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args) +#define LOG_VERBOSE(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Verbose, args) #define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) namespace mozilla { namespace camera { // 3 threads are involved in this code: // - the main thread for some setups, and occassionally for video capture setup // calls that don't work correctly elsewhere. // - the IPC thread on which PBackground is running and which receives and // sends messages // - a thread which will execute the actual (possibly slow) camera access // called "VideoCapture". On Windows this is a thread with an event loop // suitable for UI access. -void InputObserver::DeviceChange() { +void InputObserver::OnDeviceChange() { LOG((__PRETTY_FUNCTION__)); MOZ_ASSERT(mParent); RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([this]() -> nsresult { if (mParent->IsShuttingDown()) { return NS_ERROR_FAILURE; } @@ -58,122 +61,67 @@ void InputObserver::DeviceChange() { return NS_OK; }); nsIThread* thread = mParent->GetBackgroundThread(); MOZ_ASSERT(thread != nullptr); thread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); }; -class FrameSizeChangeRunnable : public Runnable { +class DeliverFrameRunnable : public ::mozilla::Runnable { public: - FrameSizeChangeRunnable(CamerasParent *aParent, CaptureEngine capEngine, - int cap_id, unsigned int aWidth, unsigned int aHeight) - : mParent(aParent), mCapEngine(capEngine), mCapId(cap_id), - mWidth(aWidth), mHeight(aHeight) {} + DeliverFrameRunnable(CamerasParent *aParent, CaptureEngine aEngine, + uint32_t aStreamId, const webrtc::VideoFrame& aFrame, + const VideoFrameProperties& aProperties) + : mParent(aParent), mCapEngine(aEngine), mStreamId(aStreamId), + mProperties(aProperties) + { + // No ShmemBuffer (of the right size) was available, so make an + // extra buffer here. We have no idea when we are going to run and + // it will be potentially long after the webrtc frame callback has + // returned, so the copy needs to be no later than here. + // We will need to copy this back into a Shmem later on so we prefer + // using ShmemBuffers to avoid the extra copy. + mAlternateBuffer.reset(new unsigned char[aProperties.bufferSize()]); + VideoFrameUtils::CopyVideoFrameBuffers(mAlternateBuffer.get(), + aProperties.bufferSize(), aFrame); + } + + DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, + uint32_t aStreamId, ShmemBuffer aBuffer, VideoFrameProperties& aProperties) + : mParent(aParent), mCapEngine(aEngine), mStreamId(aStreamId), + mBuffer(Move(aBuffer)), mProperties(aProperties) + {}; NS_IMETHOD Run() override { if (mParent->IsShuttingDown()) { // Communication channel is being torn down - LOG(("FrameSizeChangeRunnable is active without active Child")); mResult = 0; return NS_OK; } - if (!mParent->SendFrameSizeChange(mCapEngine, mCapId, mWidth, mHeight)) { + if (!mParent->DeliverFrameOverIPC(mCapEngine, mStreamId, Move(mBuffer), + mAlternateBuffer.get(), mProperties)) { mResult = -1; } else { mResult = 0; } return NS_OK; } int GetResult() { return mResult; } private: RefPtr<CamerasParent> mParent; CaptureEngine mCapEngine; - int mCapId; - unsigned int mWidth; - unsigned int mHeight; - int mResult; -}; - -int -CallbackHelper::FrameSizeChange(unsigned int w, unsigned int h, - unsigned int streams) -{ - LOG(("CallbackHelper Video FrameSizeChange: %ux%u", w, h)); - RefPtr<FrameSizeChangeRunnable> runnable = - new FrameSizeChangeRunnable(mParent, mCapEngine, mCapturerId, w, h); - MOZ_ASSERT(mParent); - nsIThread * thread = mParent->GetBackgroundThread(); - MOZ_ASSERT(thread != nullptr); - thread->Dispatch(runnable, NS_DISPATCH_NORMAL); - return 0; -} - -class DeliverFrameRunnable : public Runnable { -public: - DeliverFrameRunnable(CamerasParent *aParent, - CaptureEngine engine, - int cap_id, - ShmemBuffer buffer, - unsigned char* altbuffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time) - : mParent(aParent), mCapEngine(engine), mCapId(cap_id), mBuffer(Move(buffer)), - mSize(size), mTimeStamp(time_stamp), mNtpTime(ntp_time), - mRenderTime(render_time) { - // No ShmemBuffer (of the right size) was available, so make an - // extra buffer here. We have no idea when we are going to run and - // it will be potentially long after the webrtc frame callback has - // returned, so the copy needs to be no later than here. - // We will need to copy this back into a Shmem later on so we prefer - // using ShmemBuffers to avoid the extra copy. - if (altbuffer != nullptr) { - mAlternateBuffer.reset(new unsigned char[size]); - memcpy(mAlternateBuffer.get(), altbuffer, size); - } - }; - - NS_IMETHOD Run() override { - if (mParent->IsShuttingDown()) { - // Communication channel is being torn down - mResult = 0; - return NS_OK; - } - if (!mParent->DeliverFrameOverIPC(mCapEngine, mCapId, - Move(mBuffer), mAlternateBuffer.get(), - mSize, mTimeStamp, - mNtpTime, mRenderTime)) { - mResult = -1; - } else { - mResult = 0; - } - return NS_OK; - } - - int GetResult() { - return mResult; - } - -private: - RefPtr<CamerasParent> mParent; - CaptureEngine mCapEngine; - int mCapId; + uint32_t mStreamId; ShmemBuffer mBuffer; mozilla::UniquePtr<unsigned char[]> mAlternateBuffer; - size_t mSize; - uint32_t mTimeStamp; - int64_t mNtpTime; - int64_t mRenderTime; + VideoFrameProperties mProperties; int mResult; }; NS_IMPL_ISUPPORTS(CamerasParent, nsIObserver) NS_IMETHODIMP CamerasParent::Observe(nsISupports *aSubject, const char *aTopic, @@ -253,128 +201,128 @@ CamerasParent::StopVideoCapture() }); if (NS_FAILED(NS_DispatchToMainThread(threadShutdown))) { LOG(("Could not dispatch VideoCaptureThread destruction")); } } } int -CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine, - int cap_id, - ShmemBuffer buffer, - unsigned char* altbuffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time) +CamerasParent::DeliverFrameOverIPC(CaptureEngine capEng, + uint32_t aStreamId, + ShmemBuffer buffer, + unsigned char* altbuffer, + VideoFrameProperties& aProps) { // No ShmemBuffers were available, so construct one now of the right size // and copy into it. That is an extra copy, but we expect this to be // the exceptional case, because we just assured the next call *will* have a // buffer of the right size. if (altbuffer != nullptr) { // Get a shared memory buffer from the pool, at least size big - ShmemBuffer shMemBuff = mShmemPool.Get(this, size); + ShmemBuffer shMemBuff = mShmemPool.Get(this, aProps.bufferSize()); if (!shMemBuff.Valid()) { LOG(("No usable Video shmem in DeliverFrame (out of buffers?)")); // We can skip this frame if we run out of buffers, it's not a real error. return 0; } // get() and Size() check for proper alignment of the segment - memcpy(shMemBuff.GetBytes(), altbuffer, size); + memcpy(shMemBuff.GetBytes(), altbuffer, aProps.bufferSize()); - if (!SendDeliverFrame(cap_engine, cap_id, - shMemBuff.Get(), size, - time_stamp, ntp_time, render_time)) { + if (!SendDeliverFrame(capEng, aStreamId, + shMemBuff.Get(), aProps)) { return -1; } } else { MOZ_ASSERT(buffer.Valid()); // ShmemBuffer was available, we're all good. A single copy happened // in the original webrtc callback. - if (!SendDeliverFrame(cap_engine, cap_id, - buffer.Get(), size, - time_stamp, ntp_time, render_time)) { + if (!SendDeliverFrame(capEng, aStreamId, + buffer.Get(), aProps)) { return -1; } } return 0; } ShmemBuffer CamerasParent::GetBuffer(size_t aSize) { return mShmemPool.GetIfAvailable(aSize); } -int -CallbackHelper::DeliverFrame(unsigned char* buffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time, - void *handle) +int32_t +CallbackHelper::RenderFrame(uint32_t aStreamId, const webrtc::VideoFrame& aVideoFrame) { + LOG_VERBOSE((__PRETTY_FUNCTION__)); + RefPtr<DeliverFrameRunnable> runnable = nullptr; + // Get frame properties + camera::VideoFrameProperties properties; + VideoFrameUtils::InitFrameBufferProperties(aVideoFrame, properties); // Get a shared memory buffer to copy the frame data into - ShmemBuffer shMemBuffer = mParent->GetBuffer(size); + ShmemBuffer shMemBuffer = mParent->GetBuffer(properties.bufferSize()); if (!shMemBuffer.Valid()) { // Either we ran out of buffers or they're not the right size yet LOG(("Correctly sized Video shmem not available in DeliverFrame")); // We will do the copy into a(n extra) temporary buffer inside // the DeliverFrameRunnable constructor. } else { // Shared memory buffers of the right size are available, do the copy here. - memcpy(shMemBuffer.GetBytes(), buffer, size); - // Mark the original buffer as cleared. - buffer = nullptr; + VideoFrameUtils::CopyVideoFrameBuffers(shMemBuffer.GetBytes(), + properties.bufferSize(), aVideoFrame); + runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, + Move(shMemBuffer), properties); } - RefPtr<DeliverFrameRunnable> runnable = - new DeliverFrameRunnable(mParent, mCapEngine, mCapturerId, - Move(shMemBuffer), buffer, size, time_stamp, - ntp_time, render_time); + if (!runnable.get()) { + runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, + aVideoFrame, properties); + } MOZ_ASSERT(mParent); nsIThread* thread = mParent->GetBackgroundThread(); MOZ_ASSERT(thread != nullptr); thread->Dispatch(runnable, NS_DISPATCH_NORMAL); return 0; } -// XXX!!! FIX THIS -- we should move to pure DeliverI420Frame -int -CallbackHelper::DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) + +void +CallbackHelper::OnIncomingCapturedFrame(const int32_t id, const webrtc::VideoFrame& aVideoFrame) { - return DeliverFrame(const_cast<uint8_t*>(webrtc_frame.buffer(webrtc::kYPlane)), - CalcBufferSize(webrtc::kI420, webrtc_frame.width(), webrtc_frame.height()), - webrtc_frame.timestamp(), - webrtc_frame.ntp_time_ms(), - webrtc_frame.render_time_ms(), - (void*) webrtc_frame.native_handle()); + LOG_VERBOSE((__PRETTY_FUNCTION__)); + RenderFrame(id,aVideoFrame); +} + +void +CallbackHelper::OnCaptureDelayChanged(const int32_t id, const int32_t delay) +{ + LOG((__PRETTY_FUNCTION__)); } mozilla::ipc::IPCResult CamerasParent::RecvReleaseFrame(mozilla::ipc::Shmem&& s) { mShmemPool.Put(ShmemBuffer(s)); return IPC_OK(); } bool CamerasParent::SetupEngine(CaptureEngine aCapEngine) { + LOG((__PRETTY_FUNCTION__)); MOZ_ASSERT(mVideoCaptureThread->thread_id() == PlatformThread::CurrentId()); - EngineHelper *helper = &mEngines[aCapEngine]; + RefPtr<mozilla::camera::VideoEngine>* engine = &mEngines[aCapEngine]; // Already initialized - if (helper->mEngine) { + if (engine->get()) { return true; } webrtc::CaptureDeviceInfo *captureDeviceInfo = nullptr; + UniquePtr<webrtc::Config> config(new webrtc::Config); switch (aCapEngine) { case ScreenEngine: captureDeviceInfo = new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen); break; case BrowserEngine: captureDeviceInfo = @@ -393,54 +341,29 @@ CamerasParent::SetupEngine(CaptureEngine new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Camera); break; default: LOG(("Invalid webrtc Video engine")); MOZ_CRASH(); break; } - helper->mConfig.Set<webrtc::CaptureDeviceInfo>(captureDeviceInfo); - helper->mEngine = webrtc::VideoEngine::Create(helper->mConfig); + config->Set<webrtc::CaptureDeviceInfo>(captureDeviceInfo); + *engine = mozilla::camera::VideoEngine::Create(UniquePtr<const webrtc::Config>(config.release())); - if (!helper->mEngine) { + if (!engine->get()) { LOG(("VideoEngine::Create failed")); return false; } - helper->mPtrViEBase = webrtc::ViEBase::GetInterface(helper->mEngine); - if (!helper->mPtrViEBase) { - LOG(("ViEBase::GetInterface failed")); - return false; - } - - if (helper->mPtrViEBase->Init() < 0) { - LOG(("ViEBase::Init failed")); - return false; - } - - helper->mPtrViECapture = webrtc::ViECapture::GetInterface(helper->mEngine); - if (!helper->mPtrViECapture) { - LOG(("ViECapture::GetInterface failed")); - return false; - } - - InputObserver** observer = mObservers.AppendElement( - new InputObserver(this)); - -#ifdef DEBUG - MOZ_ASSERT(0 == helper->mPtrViECapture->RegisterInputObserver(*observer)); -#else - helper->mPtrViECapture->RegisterInputObserver(*observer); -#endif - - helper->mPtrViERender = webrtc::ViERender::GetInterface(helper->mEngine); - if (!helper->mPtrViERender) { - LOG(("ViERender::GetInterface failed")); - return false; + InputObserver** observer = mObservers.AppendElement(new InputObserver(this)); + auto device_info = engine->get()->GetOrCreateVideoCaptureDeviceInfo(); + MOZ_ASSERT(device_info); + if (device_info) { + device_info->RegisterVideoInputFeedBack(**observer); } return true; } void CamerasParent::CloseEngines() { @@ -448,92 +371,81 @@ CamerasParent::CloseEngines() if (!mWebRTCAlive) { return; } MOZ_ASSERT(mVideoCaptureThread->thread_id() == PlatformThread::CurrentId()); // Stop the callers while (mCallbacks.Length()) { auto capEngine = mCallbacks[0]->mCapEngine; - auto capNum = mCallbacks[0]->mCapturerId; - LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, capNum)); - StopCapture(capEngine, capNum); - Unused << ReleaseCaptureDevice(capEngine, capNum); + auto streamNum = mCallbacks[0]->mStreamId; + LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, streamNum)); + StopCapture(capEngine, streamNum); + Unused << ReleaseCaptureDevice(capEngine, streamNum); } for (int i = 0; i < CaptureEngine::MaxEngine; i++) { - if (mEngines[i].mEngineIsRunning) { - LOG(("Being closed down while engine %d is running!", i)); - } - if (mEngines[i].mPtrViERender) { - mEngines[i].mPtrViERender->Release(); - mEngines[i].mPtrViERender = nullptr; - } - if (mEngines[i].mPtrViECapture) { -#ifdef DEBUG - MOZ_ASSERT(0 == mEngines[i].mPtrViECapture->DeregisterInputObserver()); -#else - mEngines[i].mPtrViECapture->DeregisterInputObserver(); -#endif + if (auto engine = mEngines[i].get() ){ + if (engine->IsRunning()) { + LOG(("Being closed down while engine %d is running!", i)); + } - mEngines[i].mPtrViECapture->Release(); - mEngines[i].mPtrViECapture = nullptr; - } - if(mEngines[i].mPtrViEBase) { - mEngines[i].mPtrViEBase->Release(); - mEngines[i].mPtrViEBase = nullptr; - } - if (mEngines[i].mEngine) { - mEngines[i].mEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mEngines[i].mEngine); - mEngines[i].mEngine = nullptr; + auto device_info = engine->GetOrCreateVideoCaptureDeviceInfo(); + MOZ_ASSERT(device_info); + if (device_info) { + device_info->DeRegisterVideoInputFeedBack(); + } + mozilla::camera::VideoEngine::Delete(engine); + mEngines[i] = nullptr; } } for (InputObserver* observer : mObservers) { delete observer; } mObservers.Clear(); mWebRTCAlive = false; } -bool +VideoEngine * CamerasParent::EnsureInitialized(int aEngine) { - LOG((__PRETTY_FUNCTION__)); + LOG_VERBOSE((__PRETTY_FUNCTION__)); // We're shutting down, don't try to do new WebRTC ops. if (!mWebRTCAlive) { - return false; + return nullptr; } CaptureEngine capEngine = static_cast<CaptureEngine>(aEngine); if (!SetupEngine(capEngine)) { LOG(("CamerasParent failed to initialize engine")); - return false; + return nullptr; } - return true; + return mEngines[aEngine]; } // Dispatch the runnable to do the camera operation on the // specific Cameras thread, preventing us from blocking, and // chain a runnable to send back the result on the IPC thread. // It would be nice to get rid of the code duplication here, // perhaps via Promises. mozilla::ipc::IPCResult CamerasParent::RecvNumberOfCaptureDevices(const CaptureEngine& aCapEngine) { LOG((__PRETTY_FUNCTION__)); - + LOG(("CaptureEngine=%d", aCapEngine)); RefPtr<CamerasParent> self(this); RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, aCapEngine]() -> nsresult { int num = -1; - if (self->EnsureInitialized(aCapEngine)) { - num = self->mEngines[aCapEngine].mPtrViECapture->NumberOfCaptureDevices(); + if (auto engine = self->EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + num = devInfo->NumberOfDevices(); + } } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, num]() -> nsresult { if (self->IsShuttingDown()) { return NS_ERROR_FAILURE; } if (num < 0) { LOG(("RecvNumberOfCaptureDevices couldn't find devices")); @@ -590,21 +502,20 @@ CamerasParent::RecvNumberOfCapabilities( { LOG((__PRETTY_FUNCTION__)); LOG(("Getting caps for %s", unique_id.get())); RefPtr<CamerasParent> self(this); RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, unique_id, aCapEngine]() -> nsresult { int num = -1; - if (self->EnsureInitialized(aCapEngine)) { - num = - self->mEngines[aCapEngine].mPtrViECapture->NumberOfCapabilities( - unique_id.get(), - MediaEngineSource::kMaxUniqueIdLength); + if (auto engine = self->EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + num = devInfo->NumberOfCapabilities(unique_id.get()); + } } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, num]() -> nsresult { if (self->IsShuttingDown()) { return NS_ERROR_FAILURE; } if (num < 0) { LOG(("RecvNumberOfCapabilities couldn't find capabilities")); @@ -629,28 +540,29 @@ CamerasParent::RecvGetCaptureCapability( const int& num) { LOG((__PRETTY_FUNCTION__)); LOG(("RecvGetCaptureCapability: %s %d", unique_id.get(), num)); RefPtr<CamerasParent> self(this); RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, unique_id, aCapEngine, num]() -> nsresult { - webrtc::CaptureCapability webrtcCaps; + webrtc::VideoCaptureCapability webrtcCaps; int error = -1; - if (self->EnsureInitialized(aCapEngine)) { - error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureCapability( - unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, num, webrtcCaps); + if (auto engine = self->EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()){ + error = devInfo->GetCapability(unique_id.get(), num, webrtcCaps); + } } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, webrtcCaps, error]() -> nsresult { if (self->IsShuttingDown()) { return NS_ERROR_FAILURE; } - CaptureCapability capCap(webrtcCaps.width, + VideoCaptureCapability capCap(webrtcCaps.width, webrtcCaps.height, webrtcCaps.maxFPS, webrtcCaps.expectedCaptureDelay, webrtcCaps.rawType, webrtcCaps.codecType, webrtcCaps.interlaced); LOG(("Capability: %u %u %u %u %d %d", webrtcCaps.width, @@ -681,25 +593,25 @@ CamerasParent::RecvGetCaptureDevice(cons RefPtr<CamerasParent> self(this); RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, aCapEngine, aListNumber]() -> nsresult { char deviceName[MediaEngineSource::kMaxDeviceNameLength]; char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength]; nsCString name; nsCString uniqueId; - int devicePid = 0; + pid_t devicePid = 0; int error = -1; - if (self->EnsureInitialized(aCapEngine)) { - error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureDevice(aListNumber, - deviceName, - sizeof(deviceName), - deviceUniqueId, - sizeof(deviceUniqueId), - &devicePid); + if (auto engine = self->EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + error = devInfo->GetDeviceName(aListNumber, deviceName, sizeof(deviceName), + deviceUniqueId, sizeof(deviceUniqueId), + nullptr, 0, + &devicePid); + } } if (!error) { name.Assign(deviceName); uniqueId.Assign(deviceUniqueId); } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, error, name, uniqueId, devicePid]() { if (self->IsShuttingDown()) { @@ -805,18 +717,27 @@ CamerasParent::RecvAllocateCaptureDevice // After retrieving the permission (or not) on the main thread, // bounce to the WebRTC thread to allocate the device (or not), // then bounce back to the IPC thread for the reply to content. RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, allowed, aCapEngine, unique_id]() -> nsresult { int numdev = -1; int error = -1; if (allowed && self->EnsureInitialized(aCapEngine)) { - error = self->mEngines[aCapEngine].mPtrViECapture->AllocateCaptureDevice( - unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, numdev); + auto engine = self->mEngines[aCapEngine].get(); + engine->CreateVideoCapture(numdev, unique_id.get()); + engine->WithEntry(numdev, [&error](VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + if (!cap.VideoRenderer()) { + LOG(("VideoEngine::VideoRenderer() failed")); + } else { + error = 0; + } + } + }); } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, numdev, error]() -> nsresult { if (self->IsShuttingDown()) { return NS_ERROR_FAILURE; } if (error) { Unused << self->SendReplyFailure(); @@ -837,18 +758,18 @@ CamerasParent::RecvAllocateCaptureDevice return IPC_OK(); } int CamerasParent::ReleaseCaptureDevice(const CaptureEngine& aCapEngine, const int& capnum) { int error = -1; - if (EnsureInitialized(aCapEngine)) { - error = mEngines[aCapEngine].mPtrViECapture->ReleaseCaptureDevice(capnum); + if (auto engine = EnsureInitialized(aCapEngine)) { + error = engine->ReleaseVideoCapture(capnum); } return error; } mozilla::ipc::IPCResult CamerasParent::RecvReleaseCaptureDevice(const CaptureEngine& aCapEngine, const int& numdev) { @@ -880,54 +801,58 @@ CamerasParent::RecvReleaseCaptureDevice( }); DispatchToVideoCaptureThread(webrtc_runnable); return IPC_OK(); } mozilla::ipc::IPCResult CamerasParent::RecvStartCapture(const CaptureEngine& aCapEngine, const int& capnum, - const CaptureCapability& ipcCaps) + const VideoCaptureCapability& ipcCaps) { LOG((__PRETTY_FUNCTION__)); RefPtr<CamerasParent> self(this); RefPtr<Runnable> webrtc_runnable = media::NewRunnableFrom([self, aCapEngine, capnum, ipcCaps]() -> nsresult { + LOG((__PRETTY_FUNCTION__)); CallbackHelper** cbh; - webrtc::ExternalRenderer* render; - EngineHelper* helper = nullptr; + webrtc::VideoRenderCallback* render; + VideoEngine* engine = nullptr; int error = -1; if (self->EnsureInitialized(aCapEngine)) { cbh = self->mCallbacks.AppendElement( new CallbackHelper(static_cast<CaptureEngine>(aCapEngine), capnum, self)); - render = static_cast<webrtc::ExternalRenderer*>(*cbh); + render = static_cast<webrtc::VideoRenderCallback*>(*cbh); - helper = &self->mEngines[aCapEngine]; - error = - helper->mPtrViERender->AddRenderer(capnum, webrtc::kVideoI420, render); - if (!error) { - error = helper->mPtrViERender->StartRender(capnum); - } + engine = self->mEngines[aCapEngine]; + engine->WithEntry(capnum, [capnum, &render, &engine, &error, &ipcCaps, &cbh](VideoEngine::CaptureEntry& cap) { + cap.VideoRenderer()->AddIncomingRenderStream(capnum,0, 0., 0., 1., 1.); + error = cap.VideoRenderer()->AddExternalRenderCallback(capnum, render); + if (!error) { + error = cap.VideoRenderer()->StartRender(capnum); + } - webrtc::CaptureCapability capability; - capability.width = ipcCaps.width(); - capability.height = ipcCaps.height(); - capability.maxFPS = ipcCaps.maxFPS(); - capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay(); - capability.rawType = static_cast<webrtc::RawVideoType>(ipcCaps.rawType()); - capability.codecType = static_cast<webrtc::VideoCodecType>(ipcCaps.codecType()); - capability.interlaced = ipcCaps.interlaced(); + webrtc::VideoCaptureCapability capability; + capability.width = ipcCaps.width(); + capability.height = ipcCaps.height(); + capability.maxFPS = ipcCaps.maxFPS(); + capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay(); + capability.rawType = static_cast<webrtc::RawVideoType>(ipcCaps.rawType()); + capability.codecType = static_cast<webrtc::VideoCodecType>(ipcCaps.codecType()); + capability.interlaced = ipcCaps.interlaced(); - if (!error) { - error = helper->mPtrViECapture->StartCapture(capnum, capability); - } - if (!error) { - helper->mEngineIsRunning = true; - } + if (!error) { + error = cap.VideoCapture()->StartCapture(capability); + } + if (!error) { + engine->Startup(); + cap.VideoCapture()->RegisterCaptureDataCallback(*static_cast<webrtc::VideoCaptureDataCallback*>(*cbh)); + } + }); } RefPtr<nsIRunnable> ipc_runnable = media::NewRunnableFrom([self, error]() -> nsresult { if (self->IsShuttingDown()) { return NS_ERROR_FAILURE; } if (!error) { Unused << self->SendReplySuccess(); @@ -943,30 +868,37 @@ CamerasParent::RecvStartCapture(const Ca DispatchToVideoCaptureThread(webrtc_runnable); return IPC_OK(); } void CamerasParent::StopCapture(const CaptureEngine& aCapEngine, const int& capnum) { - if (EnsureInitialized(aCapEngine)) { - mEngines[aCapEngine].mPtrViECapture->StopCapture(capnum); - mEngines[aCapEngine].mPtrViERender->StopRender(capnum); - mEngines[aCapEngine].mPtrViERender->RemoveRenderer(capnum); - mEngines[aCapEngine].mEngineIsRunning = false; - - for (size_t i = 0; i < mCallbacks.Length(); i++) { - if (mCallbacks[i]->mCapEngine == aCapEngine - && mCallbacks[i]->mCapturerId == capnum) { - delete mCallbacks[i]; - mCallbacks.RemoveElementAt(i); + if (auto engine = EnsureInitialized(aCapEngine)) { + engine->WithEntry(capnum,[capnum](VideoEngine::CaptureEntry& cap){ + if (cap.VideoCapture()) { + cap.VideoCapture()->StopCapture(); + cap.VideoCapture()->DeRegisterCaptureDataCallback(); + } + if (cap.VideoRenderer()) { + cap.VideoRenderer()->StopRender(capnum); + } + }); + // we're removing elements, iterate backwards + for (size_t i = mCallbacks.Length(); i > 0; i--) { + if (mCallbacks[i-1]->mCapEngine == aCapEngine + && mCallbacks[i-1]->mStreamId == (uint32_t) capnum) { + delete mCallbacks[i-1]; + mCallbacks.RemoveElementAt(i-1); break; } } + engine->RemoveRenderer(capnum); + engine->Shutdown(); } } mozilla::ipc::IPCResult CamerasParent::RecvStopCapture(const CaptureEngine& aCapEngine, const int& capnum) { LOG((__PRETTY_FUNCTION__)); @@ -1084,17 +1016,17 @@ CamerasParent::~CamerasParent() LOG(("~CamerasParent: %p", this)); #ifdef DEBUG // Verify we have shut down the webrtc engines, this is // supposed to happen in ActorDestroy. // That runnable takes a ref to us, so it must have finished // by the time we get here. for (int i = 0; i < CaptureEngine::MaxEngine; i++) { - MOZ_ASSERT(!mEngines[i].mEngine); + MOZ_ASSERT(!mEngines[i]); } #endif } already_AddRefed<CamerasParent> CamerasParent::Create() { mozilla::ipc::AssertIsOnBackgroundThread(); RefPtr<CamerasParent> camerasParent = new CamerasParent();
--- a/dom/media/systemservices/CamerasParent.h +++ b/dom/media/systemservices/CamerasParent.h @@ -3,90 +3,75 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef mozilla_CamerasParent_h #define mozilla_CamerasParent_h #include "nsIObserver.h" +#include "VideoEngine.h" #include "mozilla/dom/ContentParent.h" #include "mozilla/camera/PCamerasParent.h" #include "mozilla/ipc/Shmem.h" #include "mozilla/ShmemPool.h" #include "mozilla/Atomics.h" +#include "webrtc/modules/video_capture/video_capture.h" +#include "webrtc/modules/video_render/video_render_impl.h" +#include "webrtc/modules/video_capture/video_capture_defines.h" +#include "webrtc/common_video/include/incoming_video_stream.h" // conflicts with #include of scoped_ptr.h #undef FF #include "webrtc/common.h" -// Video Engine -#include "webrtc/video_engine/include/vie_base.h" -#include "webrtc/video_engine/include/vie_capture.h" -#include "webrtc/video_engine/include/vie_render.h" + #include "CamerasChild.h" #include "base/thread.h" namespace mozilla { namespace camera { class CamerasParent; -class CallbackHelper : public webrtc::ExternalRenderer +class CallbackHelper : + public webrtc::VideoRenderCallback, + public webrtc::VideoCaptureDataCallback { public: - CallbackHelper(CaptureEngine aCapEng, int aCapId, CamerasParent *aParent) - : mCapEngine(aCapEng), mCapturerId(aCapId), mParent(aParent) {}; + CallbackHelper(CaptureEngine aCapEng, uint32_t aStreamId, CamerasParent *aParent) + : mCapEngine(aCapEng), mStreamId(aStreamId), mParent(aParent) {}; // ViEExternalRenderer implementation. These callbacks end up // running on the VideoCapture thread. - virtual int FrameSizeChange(unsigned int w, unsigned int h, - unsigned int streams) override; - virtual int DeliverFrame(unsigned char* buffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time, - void *handle) override; - virtual int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override; - virtual bool IsTextureSupported() override { return false; }; + virtual int32_t RenderFrame(const uint32_t aStreamId, const webrtc::VideoFrame& video_frame) override; + + // From VideoCaptureCallback + virtual void OnIncomingCapturedFrame(const int32_t id, const webrtc::VideoFrame& videoFrame) override; + virtual void OnCaptureDelayChanged(const int32_t id, const int32_t delay) override; + + // TODO(@@NG) This is now part of webrtc::VideoRenderer, not in the webrtc::VideoRenderCallback + // virtual bool IsTextureSupported() const override { return false; }; + // + // virtual bool SmoothsRenderedFrames() const override { return false; } friend CamerasParent; private: CaptureEngine mCapEngine; - int mCapturerId; + uint32_t mStreamId; CamerasParent *mParent; }; -class EngineHelper -{ -public: - EngineHelper() : - mEngine(nullptr), mPtrViEBase(nullptr), mPtrViECapture(nullptr), - mPtrViERender(nullptr), mEngineIsRunning(false) {}; - - webrtc::VideoEngine *mEngine; - webrtc::ViEBase *mPtrViEBase; - webrtc::ViECapture *mPtrViECapture; - webrtc::ViERender *mPtrViERender; - - // The webrtc code keeps a reference to this one. - webrtc::Config mConfig; - - // Engine alive - bool mEngineIsRunning; -}; - -class InputObserver : public webrtc::ViEInputObserver +class InputObserver : public webrtc::VideoInputFeedBack { public: explicit InputObserver(CamerasParent* aParent) : mParent(aParent) {}; - virtual void DeviceChange(); + virtual void OnDeviceChange(); friend CamerasParent; private: RefPtr<CamerasParent> mParent; }; class CamerasParent : public PCamerasParent, @@ -105,58 +90,55 @@ public: const int&) override; virtual mozilla::ipc::IPCResult RecvNumberOfCaptureDevices(const CaptureEngine&) override; virtual mozilla::ipc::IPCResult RecvNumberOfCapabilities(const CaptureEngine&, const nsCString&) override; virtual mozilla::ipc::IPCResult RecvGetCaptureCapability(const CaptureEngine&, const nsCString&, const int&) override; virtual mozilla::ipc::IPCResult RecvGetCaptureDevice(const CaptureEngine&, const int&) override; virtual mozilla::ipc::IPCResult RecvStartCapture(const CaptureEngine&, const int&, - const CaptureCapability&) override; + const VideoCaptureCapability&) override; virtual mozilla::ipc::IPCResult RecvStopCapture(const CaptureEngine&, const int&) override; virtual mozilla::ipc::IPCResult RecvReleaseFrame(mozilla::ipc::Shmem&&) override; virtual mozilla::ipc::IPCResult RecvAllDone() override; virtual void ActorDestroy(ActorDestroyReason aWhy) override; virtual mozilla::ipc::IPCResult RecvEnsureInitialized(const CaptureEngine&) override; nsIThread* GetBackgroundThread() { return mPBackgroundThread; }; bool IsShuttingDown() { return !mChildIsAlive || mDestroyed || !mWebRTCAlive; }; ShmemBuffer GetBuffer(size_t aSize); // helper to forward to the PBackground thread int DeliverFrameOverIPC(CaptureEngine capEng, - int cap_id, + uint32_t aStreamId, ShmemBuffer buffer, unsigned char* altbuffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time); + VideoFrameProperties& aProps); CamerasParent(); protected: virtual ~CamerasParent(); // We use these helpers for shutdown and for the respective IPC commands. void StopCapture(const CaptureEngine& aCapEngine, const int& capnum); int ReleaseCaptureDevice(const CaptureEngine& aCapEngine, const int& capnum); bool SetupEngine(CaptureEngine aCapEngine); - bool EnsureInitialized(int aEngine); + VideoEngine* EnsureInitialized(int aEngine); void CloseEngines(); void StopIPC(); void StopVideoCapture(); // Can't take already_AddRefed because it can fail in stupid ways. nsresult DispatchToVideoCaptureThread(Runnable* event); - EngineHelper mEngines[CaptureEngine::MaxEngine]; + RefPtr<VideoEngine> mEngines[CaptureEngine::MaxEngine]; nsTArray<CallbackHelper*> mCallbacks; // image buffers mozilla::ShmemPool mShmemPool; // PBackground parent thread nsCOMPtr<nsIThread> mPBackgroundThread;
--- a/dom/media/systemservices/LoadManager.h +++ b/dom/media/systemservices/LoadManager.h @@ -9,18 +9,18 @@ #include "LoadMonitor.h" #include "mozilla/StaticPtr.h" #include "mozilla/TimeStamp.h" #include "mozilla/Services.h" #include "nsTArray.h" #include "nsIObserver.h" #include "webrtc/common_types.h" -#include "webrtc/video_engine/include/vie_base.h" - +#include "webrtc/call.h" +#include "webrtc/video/overuse_frame_detector.h" extern mozilla::LazyLogModule gLoadManagerLog; namespace mozilla { class LoadManagerSingleton : public LoadNotificationCallback, public webrtc::CPULoadStateCallbackInvoker, public webrtc::CpuOveruseObserver, public nsIObserver @@ -72,39 +72,40 @@ private: int mAveragingMeasurements; float mHighLoadThreshold; float mLowLoadThreshold; static StaticRefPtr<LoadManagerSingleton> sSingleton; }; class LoadManager final : public webrtc::CPULoadStateCallbackInvoker, - public webrtc::CpuOveruseObserver + public webrtc::LoadObserver { public: explicit LoadManager(LoadManagerSingleton* aManager) : mManager(aManager) {} ~LoadManager() {} void AddObserver(webrtc::CPULoadStateObserver * aObserver) override { mManager->AddObserver(aObserver); } void RemoveObserver(webrtc::CPULoadStateObserver * aObserver) override { mManager->RemoveObserver(aObserver); } - void OveruseDetected() override + + void OnLoadUpdate(webrtc::LoadObserver::Load load_state) override { - mManager->OveruseDetected(); - } - void NormalUsage() override - { - mManager->NormalUsage(); + if (load_state == webrtc::LoadObserver::kOveruse) { + mManager->OveruseDetected(); + } else if (load_state == webrtc::LoadObserver::kUnderuse) { + mManager->NormalUsage(); + } } private: RefPtr<LoadManagerSingleton> mManager; }; } //namespace
--- a/dom/media/systemservices/PCameras.ipdl +++ b/dom/media/systemservices/PCameras.ipdl @@ -5,57 +5,84 @@ include protocol PContent; include protocol PBackground; using mozilla::camera::CaptureEngine from "mozilla/media/CamerasTypes.h"; namespace mozilla { namespace camera { -struct CaptureCapability +// IPC analog for webrtc::VideoCaptureCapability +struct VideoCaptureCapability { int width; int height; int maxFPS; int expectedCaptureDelay; int rawType; int codecType; bool interlaced; }; + +// IPC analog for webrtc::VideoFrame +// the described buffer is transported seperately in a Shmem +// See VideoFrameUtils.h +struct VideoFrameProperties +{ + // Size of image data within the ShMem, + // the ShMem is at least this large + size_t bufferSize; + // From webrtc::VideoFrame + uint32_t timeStamp; + int64_t ntpTimeMs; + int64_t renderTimeMs; + // See webrtc/**/rotation.h + int rotation; + int yAllocatedSize; + int uAllocatedSize; + int vAllocatedSize; + // From webrtc::VideoFrameBuffer + int width; + int height; + int yStride; + int uStride; + int vStride; +}; + async protocol PCameras { manager PBackground; child: async FrameSizeChange(CaptureEngine capEngine, int cap_id, int w, int h); // transfers ownership of |buffer| from parent to child - async DeliverFrame(CaptureEngine capEngine, int cap_id, - Shmem buffer, size_t size, uint32_t time_stamp, - int64_t ntp_time, int64_t render_time); + async DeliverFrame(CaptureEngine capEngine, int streamId, + Shmem buffer, VideoFrameProperties props); async DeviceChange(); async ReplyNumberOfCaptureDevices(int numdev); async ReplyNumberOfCapabilities(int numdev); async ReplyAllocateCaptureDevice(int numdev); - async ReplyGetCaptureCapability(CaptureCapability cap); + async ReplyGetCaptureCapability(VideoCaptureCapability cap); async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id, bool scary); async ReplyFailure(); async ReplySuccess(); async __delete__(); parent: async NumberOfCaptureDevices(CaptureEngine engine); async NumberOfCapabilities(CaptureEngine engine, nsCString deviceUniqueIdUTF8); - async GetCaptureCapability(CaptureEngine engine, nsCString unique_idUTF8, int capability_number); + async GetCaptureCapability(CaptureEngine engine, nsCString unique_idUTF8, + int capability_number); async GetCaptureDevice(CaptureEngine engine, int num); async AllocateCaptureDevice(CaptureEngine engine, nsCString unique_idUTF8, nsCString origin); async ReleaseCaptureDevice(CaptureEngine engine, int numdev); - async StartCapture(CaptureEngine engine, int numdev, CaptureCapability capability); + async StartCapture(CaptureEngine engine, int numdev, VideoCaptureCapability capability); async StopCapture(CaptureEngine engine, int numdev); // transfers frame back async ReleaseFrame(Shmem s); // Ask parent to delete us async AllDone(); // setup camera engine async EnsureInitialized(CaptureEngine engine);
--- a/dom/media/systemservices/ShmemPool.cpp +++ b/dom/media/systemservices/ShmemPool.cpp @@ -35,17 +35,17 @@ mozilla::ShmemBuffer ShmemPool::GetIfAva if (!res.mInitialized) { LOG(("No free preallocated Shmem")); return ShmemBuffer(); } MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?"); - if (res.mShmem.Size<char>() < aSize) { + if (res.mShmem.Size<uint8_t>() < aSize) { LOG(("Free Shmem but not of the right size")); return ShmemBuffer(); } mPoolFree--; #ifdef DEBUG size_t poolUse = mShmemPool.Length() - mPoolFree; if (poolUse > mMaxPoolUse) { @@ -60,17 +60,17 @@ void ShmemPool::Put(ShmemBuffer&& aShmem { MutexAutoLock lock(mMutex); MOZ_ASSERT(mPoolFree < mShmemPool.Length()); mShmemPool[mPoolFree] = Move(aShmem); mPoolFree++; #ifdef DEBUG size_t poolUse = mShmemPool.Length() - mPoolFree; if (poolUse > 0) { - LOG(("ShmemPool usage reduced to %d buffers", poolUse)); + LOG_VERBOSE(("ShmemPool usage reduced to %d buffers", poolUse)); } #endif } ShmemPool::~ShmemPool() { #ifdef DEBUG for (size_t i = 0; i < mShmemPool.Length(); i++) {
--- a/dom/media/systemservices/ShmemPool.h +++ b/dom/media/systemservices/ShmemPool.h @@ -43,18 +43,18 @@ public: // No copies allowed ShmemBuffer(const ShmemBuffer&) = delete; ShmemBuffer& operator=(const ShmemBuffer&) = delete; bool Valid() { return mInitialized; } - char* GetBytes() { - return mShmem.get<char>(); + uint8_t * GetBytes() { + return mShmem.get<uint8_t>(); } mozilla::ipc::Shmem& Get() { return mShmem; } private: friend class ShmemPool;
new file mode 100644 --- /dev/null +++ b/dom/media/systemservices/VideoEngine.cpp @@ -0,0 +1,185 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VideoEngine.h" +#include "webrtc/video_engine/browser_capture_impl.h" +#ifdef WEBRTC_ANDROID +#include "webrtc/modules/video_capture/video_capture.h" +#ifdef WEBRTC_INCLUDE_INTERNAL_VIDEO_RENDER +#include "webrtc/modules/video_render/video_render.h" +#endif +#endif + + +namespace mozilla { +namespace camera { + +#undef LOG +#undef LOG_ENABLED +mozilla::LazyLogModule gVideoEngineLog("VideoEngine"); +#define LOG(args) MOZ_LOG(gVideoEngineLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gVideoEngineLog, mozilla::LogLevel::Debug) + +int VideoEngine::sId = 0; + +#if defined(ANDROID) +int VideoEngine::SetAndroidObjects(JavaVM* javaVM) { + LOG((__PRETTY_FUNCTION__)); + + if (webrtc::SetCaptureAndroidVM(javaVM) != 0) { + LOG(("Could not set capture Android VM")); + return -1; + } +#ifdef WEBRTC_INCLUDE_INTERNAL_VIDEO_RENDER + if (webrtc::SetRenderAndroidVM(javaVM) != 0) { + LOG(("Could not set render Android VM")); + return -1; + } +#endif + return 0; +} +#endif + +void +VideoEngine::CreateVideoCapture(int32_t& id, const char* deviceUniqueIdUTF8) { + LOG((__PRETTY_FUNCTION__)); + id = GenerateId(); + LOG(("CaptureDeviceInfo.type=%s id=%d",mCaptureDevInfo.TypeName(),id)); + CaptureEntry entry = {-1,nullptr,nullptr}; + + if (mCaptureDevInfo.type == webrtc::CaptureDeviceType::Camera) { + entry = CaptureEntry(id, + webrtc::VideoCaptureFactory::Create(id, deviceUniqueIdUTF8), + nullptr); + } else { +#ifndef WEBRTC_ANDROID + entry = CaptureEntry( + id, + webrtc::DesktopCaptureImpl::Create(id, deviceUniqueIdUTF8, mCaptureDevInfo.type), + nullptr); +#else + MOZ_ASSERT("CreateVideoCapture NO DESKTOP CAPTURE IMPL ON ANDROID" == nullptr); +#endif + } + mCaps.emplace(id,std::move(entry)); +} + +int +VideoEngine::ReleaseVideoCapture(const int32_t id) { + bool found = false; + WithEntry(id, [&found](CaptureEntry& cap) { + cap.mVideoCaptureModule = nullptr; + found = true; + }); + return found ? 0 : (-1); +} + +std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> +VideoEngine::GetOrCreateVideoCaptureDeviceInfo() { + if (mDeviceInfo) { + return mDeviceInfo; + } + switch (mCaptureDevInfo.type) { + case webrtc::CaptureDeviceType::Camera: { + mDeviceInfo.reset(webrtc::VideoCaptureFactory::CreateDeviceInfo(0)); + break; + } + case webrtc::CaptureDeviceType::Browser: { + mDeviceInfo.reset(webrtc::BrowserDeviceInfoImpl::CreateDeviceInfo()); + break; + } + // Window, Application, and Screen types are handled by DesktopCapture + case webrtc::CaptureDeviceType::Window: + case webrtc::CaptureDeviceType::Application: + case webrtc::CaptureDeviceType::Screen: { +#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) + mDeviceInfo.reset(webrtc::DesktopCaptureImpl::CreateDeviceInfo(mId,mCaptureDevInfo.type)); +#else + MOZ_ASSERT("GetVideoCaptureDeviceInfo NO DESKTOP CAPTURE IMPL ON ANDROID" == nullptr); + mDeviceInfo.reset(); +#endif + break; + } + } + return mDeviceInfo; +} + +void +VideoEngine::RemoveRenderer(int capnum) { + WithEntry(capnum, [](CaptureEntry& cap) { + cap.mVideoRender = nullptr; + }); +} + +const UniquePtr<const webrtc::Config>& +VideoEngine::GetConfiguration() { + return mConfig; +} + +RefPtr<VideoEngine> VideoEngine::Create(UniquePtr<const webrtc::Config>&& aConfig) { + LOG((__PRETTY_FUNCTION__)); + LOG(("Creating new VideoEngine with CaptureDeviceType %s", + aConfig->Get<webrtc::CaptureDeviceInfo>().TypeName())); + RefPtr<VideoEngine> engine(new VideoEngine(std::move(aConfig))); + return engine; +} + +VideoEngine::CaptureEntry::CaptureEntry(int32_t aCapnum, + rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture, + webrtc::VideoRender * aRenderer): + mCapnum(aCapnum), + mVideoCaptureModule(aCapture), + mVideoRender(aRenderer) +{} + +rtc::scoped_refptr<webrtc::VideoCaptureModule> +VideoEngine::CaptureEntry::VideoCapture() { + return mVideoCaptureModule; +} + +const UniquePtr<webrtc::VideoRender>& +VideoEngine::CaptureEntry::VideoRenderer() { + if (!mVideoRender) { + MOZ_ASSERT(mCapnum != -1); + // Create a VideoRender on demand + mVideoRender = UniquePtr<webrtc::VideoRender>( + webrtc::VideoRender::CreateVideoRender(mCapnum,nullptr,false,webrtc::kRenderExternal)); + } + return mVideoRender; +} + +int32_t +VideoEngine::CaptureEntry::Capnum() const { + return mCapnum; +} + +bool VideoEngine::WithEntry(const int32_t entryCapnum, + const std::function<void(CaptureEntry &entry)>&& fn) { + auto it = mCaps.find(entryCapnum); + if (it == mCaps.end()) { + return false; + } + fn(it->second); + return true; +} + +int32_t +VideoEngine::GenerateId() { + // XXX Something better than this (a map perhaps, or a simple boolean TArray, given + // the number in-use is O(1) normally!) + return mId = sId++; +} + +VideoEngine::VideoEngine(UniquePtr<const webrtc::Config>&& aConfig): + mCaptureDevInfo(aConfig->Get<webrtc::CaptureDeviceInfo>()), + mDeviceInfo(nullptr), + mConfig(std::move(aConfig)) +{ + LOG((__PRETTY_FUNCTION__)); +} + +} +}
new file mode 100644 --- /dev/null +++ b/dom/media/systemservices/VideoEngine.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_VideoEngine_h +#define mozilla_VideoEngine_h + +#include "MediaEngine.h" +#include "VideoFrameUtils.h" +#include "mozilla/media/MediaUtils.h" +#include "webrtc/common.h" +#include "webrtc/modules/video_capture/video_capture_impl.h" +#include "webrtc/modules/video_render/video_render.h" +#include "webrtc/modules/video_capture/video_capture_defines.h" +#include "webrtc/modules/video_capture/video_capture_factory.h" +#include "webrtc/video_engine/desktop_capture_impl.h" +#include <memory> +#include <functional> + +namespace mozilla { +namespace camera { + +// Historically the video engine was part of webrtc +// it was removed (and reimplemented in Talk) +class VideoEngine +{ +private: + virtual ~VideoEngine (){}; + +public: + VideoEngine (){}; + NS_INLINE_DECL_REFCOUNTING(VideoEngine) + + static RefPtr<VideoEngine> Create(UniquePtr<const webrtc::Config>&& aConfig); +#if defined(ANDROID) + static int SetAndroidObjects(JavaVM* javaVM); +#endif + void CreateVideoCapture(int32_t& id, const char* deviceUniqueIdUTF8); + + int ReleaseVideoCapture(const int32_t id); + + // VideoEngine is responsible for any cleanup in its modules + static void Delete(VideoEngine * engine) { } + + /** Returns or creates a new new DeviceInfo. + * It is cached to prevent repeated lengthy polling for "realness" + * of the hardware devices. This could be handled in a more elegant + * way in the future. + * @return on failure the shared_ptr will be null, otherwise it will contain a DeviceInfo. + * @see bug 1305212 https://bugzilla.mozilla.org/show_bug.cgi?id=1305212 + */ + std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> GetOrCreateVideoCaptureDeviceInfo(); + + void RemoveRenderer(int capnum); + + const UniquePtr<const webrtc::Config>& GetConfiguration(); + + void Startup() { + mIsRunning = true; + } + + void Shutdown() { + mIsRunning = false; + } + + bool IsRunning() const { + return mIsRunning; + } + + class CaptureEntry { + public: + CaptureEntry(int32_t aCapnum, + rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture, + webrtc::VideoRender* aRenderer); + int32_t Capnum() const; + rtc::scoped_refptr<webrtc::VideoCaptureModule> VideoCapture(); + const UniquePtr<webrtc::VideoRender> & VideoRenderer(); + private: + int32_t mCapnum; + rtc::scoped_refptr<webrtc::VideoCaptureModule> mVideoCaptureModule; + UniquePtr<webrtc::VideoRender> mVideoRender; + friend class VideoEngine; + }; + + // Returns true iff an entry for capnum exists + bool WithEntry(const int32_t entryCapnum, const std::function<void(CaptureEntry &entry)>&& fn); + +private: + explicit VideoEngine(UniquePtr<const webrtc::Config>&& aConfig); + bool mIsRunning; + int32_t mId; + webrtc::CaptureDeviceInfo mCaptureDevInfo; + std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> mDeviceInfo; + UniquePtr<const webrtc::Config> mConfig; + std::map<int32_t, CaptureEntry> mCaps; + + int32_t GenerateId(); + static int32_t sId; +}; +} +} +#endif
new file mode 100644 --- /dev/null +++ b/dom/media/systemservices/VideoFrameUtils.cpp @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VideoFrameUtils.h" +#include "webrtc/video_frame.h" +#include "mozilla/ShmemPool.h" + +namespace mozilla { + +size_t +VideoFrameUtils::TotalRequiredBufferSize( + const webrtc::VideoFrame& aVideoFrame) +{ + static const webrtc::PlaneType kPlanes[] = + {webrtc::kYPlane, webrtc::kUPlane, webrtc::kVPlane}; + if (aVideoFrame.IsZeroSize()) { + return 0; + } + + size_t sum = 0; + for (auto plane : kPlanes) { + sum += aVideoFrame.allocated_size(plane); + } + return sum; +} + +void VideoFrameUtils::InitFrameBufferProperties( + const webrtc::VideoFrame& aVideoFrame, + camera::VideoFrameProperties& aDestProps) +{ + // The VideoFrameBuffer image data stored in the accompanying buffer + // the buffer is at least this size of larger. + aDestProps.bufferSize() = TotalRequiredBufferSize(aVideoFrame); + + aDestProps.timeStamp() = aVideoFrame.timestamp(); + aDestProps.ntpTimeMs() = aVideoFrame.ntp_time_ms(); + aDestProps.renderTimeMs() = aVideoFrame.render_time_ms(); + + aDestProps.rotation() = aVideoFrame.rotation(); + + aDestProps.yAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane); + aDestProps.uAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane); + aDestProps.vAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane); + + aDestProps.width() = aVideoFrame.width(); + aDestProps.height() = aVideoFrame.height(); + + aDestProps.yStride() = aVideoFrame.stride(webrtc::kYPlane); + aDestProps.uStride() = aVideoFrame.stride(webrtc::kUPlane); + aDestProps.vStride() = aVideoFrame.stride(webrtc::kVPlane); +} + +void VideoFrameUtils::CopyVideoFrameBuffers(uint8_t* aDestBuffer, + const size_t aDestBufferSize, + const webrtc::VideoFrame& aFrame) +{ + static const webrtc::PlaneType planes[] = {webrtc::kYPlane, webrtc::kUPlane, webrtc::kVPlane}; + + size_t aggregateSize = TotalRequiredBufferSize(aFrame); + + MOZ_ASSERT(aDestBufferSize >= aggregateSize); + + // If planes are ordered YUV and contiguous then do a single copy + if ((aFrame.buffer(webrtc::kYPlane) != nullptr) + // Check that the three planes are ordered + && (aFrame.buffer(webrtc::kYPlane) < aFrame.buffer(webrtc::kUPlane)) + && (aFrame.buffer(webrtc::kUPlane) < aFrame.buffer(webrtc::kVPlane)) + // Check that the last plane ends at firstPlane[totalsize] + && (&aFrame.buffer(webrtc::kYPlane)[aggregateSize] == &aFrame.buffer(webrtc::kVPlane)[aFrame.allocated_size(webrtc::kVPlane)])) + { + memcpy(aDestBuffer,aFrame.buffer(webrtc::kYPlane),aggregateSize); + return; + } + + // Copy each plane + size_t offset = 0; + for (auto plane: planes) { + memcpy(&aDestBuffer[offset], aFrame.buffer(plane), aFrame.allocated_size(plane)); + offset += aFrame.allocated_size(plane); + } +} + +void VideoFrameUtils::CopyVideoFrameBuffers(ShmemBuffer& aDestShmem, + const webrtc::VideoFrame& aVideoFrame) +{ + CopyVideoFrameBuffers(aDestShmem.Get().get<uint8_t>(), aDestShmem.Get().Size<uint8_t>(), aVideoFrame); +} + +}
new file mode 100644 --- /dev/null +++ b/dom/media/systemservices/VideoFrameUtils.h @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_VideoFrameUtil_h +#define mozilla_VideoFrameUtil_h + +#include "mozilla/camera/PCameras.h" + +namespace webrtc { + class VideoFrame; +} + +namespace mozilla +{ + class ShmemBuffer; + +// Util methods for working with webrtc::VideoFrame(s) and +// the IPC classes that are used to deliver their contents to the +// MediaEnginge + +class VideoFrameUtils { +public: + + // Returns the total number of bytes necessary to copy a VideoFrame's buffer + // across all planes. + static size_t TotalRequiredBufferSize(const webrtc::VideoFrame & frame); + + // Initializes a camera::VideoFrameProperties from a VideoFrameBuffer + static void InitFrameBufferProperties(const webrtc::VideoFrame& aVideoFrame, + camera::VideoFrameProperties & aDestProperties); + + // Copies the buffers out of a VideoFrameBuffer into a buffer. + // Attempts to make as few memcopies as possible. + static void CopyVideoFrameBuffers(uint8_t * aDestBuffer, + const size_t aDestBufferSize, + const webrtc::VideoFrame & aVideoFrame); + + // Copies the buffers in a VideoFrameBuffer into a Shmem + // returns the eno from the underlying memcpy. + static void CopyVideoFrameBuffers(ShmemBuffer & aDestShmem, + const webrtc::VideoFrame & aVideoFrame); + + +}; + +} /* namespace mozilla */ + +#endif
--- a/dom/media/systemservices/moz.build +++ b/dom/media/systemservices/moz.build @@ -1,39 +1,46 @@ -# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. if CONFIG['MOZ_WEBRTC']: EXPORTS += [ 'CamerasChild.h', 'CamerasParent.h', 'LoadManager.h', 'LoadManagerFactory.h', 'LoadMonitor.h', + 'VideoEngine.h', + 'VideoFrameUtils.h' ] UNIFIED_SOURCES += [ 'CamerasChild.cpp', 'CamerasParent.cpp', 'LoadManager.cpp', 'LoadManagerFactory.cpp', 'LoadMonitor.cpp', 'ShmemPool.cpp', + 'VideoEngine.cpp', + 'VideoFrameUtils.cpp' ] LOCAL_INCLUDES += [ '/media/webrtc/signaling', '/media/webrtc/trunk', ] if CONFIG['OS_TARGET'] == 'WINNT': DEFINES['WEBRTC_WIN'] = True else: DEFINES['WEBRTC_POSIX'] = True +if CONFIG['OS_TARGET'] == 'Android': + DEFINES['WEBRTC_ANDROID'] = True + if CONFIG['OS_TARGET'] == 'Android': EXPORTS += [ 'OpenSLESProvider.h' ] UNIFIED_SOURCES += [ 'OpenSLESProvider.cpp', ]
--- a/dom/media/tests/mochitest/mochitest.ini +++ b/dom/media/tests/mochitest/mochitest.ini @@ -35,17 +35,16 @@ skip-if = android_version == '18' # andr [test_dataChannel_basicVideo.html] skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator) [test_dataChannel_bug1013809.html] skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator) [test_dataChannel_noOffer.html] [test_enumerateDevices.html] [test_ondevicechange.html] skip-if = os == 'android' -[test_getUserMedia_active_autoplay.html] [test_getUserMedia_audioCapture.html] skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator) [test_getUserMedia_addTrackRemoveTrack.html] skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator) [test_getUserMedia_addtrack_removetrack_events.html] [test_getUserMedia_basicAudio.html] [test_getUserMedia_basicVideo.html] [test_getUserMedia_basicVideo_playAfterLoadedmetadata.html] @@ -93,17 +92,17 @@ skip-if = android_version == '18' # andr skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_basicAudioNATSrflx.html] skip-if = toolkit == 'android' # websockets don't work on android (bug 1266217) [test_peerConnection_basicAudioNATRelay.html] skip-if = toolkit == 'android' # websockets don't work on android (bug 1266217) [test_peerConnection_basicAudioNATRelayTCP.html] skip-if = toolkit == 'android' # websockets don't work on android (bug 1266217) [test_peerConnection_basicAudioRequireEOC.html] -skip-if = (android_version == '18' && debug) # android(Bug 1189784, timeouts on 4.3 emulator) +skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_basicAudioPcmaPcmuOnly.html] skip-if = android_version == '18' [test_peerConnection_basicAudioDynamicPtMissingRtpmap.html] skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_basicAudioVideo.html] skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_basicAudioVideoCombined.html] skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) @@ -121,17 +120,17 @@ skip-if = toolkit == 'android' # no scre [test_peerConnection_basicWindowshare.html] # frequent timeouts/crashes on e10s (bug 1048455) skip-if = toolkit == 'android' # no screenshare on android [test_peerConnection_basicH264Video.html] skip-if = os == 'android' # bug 1043403 [test_peerConnection_bug822674.html] [test_peerConnection_bug825703.html] [test_peerConnection_bug827843.html] -skip-if = (android_version == '18' && debug) # android(Bug 1189784, timeouts on 4.3 emulator) +skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_bug834153.html] [test_peerConnection_bug1013809.html] skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator) [test_peerConnection_bug1042791.html] skip-if = os == 'android' # bug 1043403 [test_peerConnection_bug1064223.html] [test_peerConnection_capturedVideo.html] tags=capturestream
--- a/dom/media/tests/mochitest/pc.js +++ b/dom/media/tests/mochitest/pc.js @@ -1414,36 +1414,72 @@ PeerConnectionWrapper.prototype = { * Wait for RTP packet flow for the given MediaStreamTrack. * * @param {object} track * A MediaStreamTrack to wait for data flow on. * @returns {Promise} * A promise that resolves when media is flowing. */ waitForRtpFlow(track) { - var hasFlow = stats => { - var rtp = stats.get([...stats.keys()].find(key => + var hasFlow = (stats, retries) => { + info("Checking for stats in " + JSON.stringify(stats) + " for " + track.kind + + " track " + track.id + ", retry number " + retries); + var rtp = stats.get([...Object.keys(stats)].find(key => !stats.get(key).isRemote && stats.get(key).type.endsWith("boundrtp"))); - ok(rtp, "Should have RTP stats for track " + track.id); if (!rtp) { + return false; } + info("Should have RTP stats for track " + track.id); + info("RTP stats: "+JSON.stringify(rtp)); var nrPackets = rtp[rtp.type == "outboundrtp" ? "packetsSent" : "packetsReceived"]; info("Track " + track.id + " has " + nrPackets + " " + rtp.type + " RTP packets."); return nrPackets > 0; }; - info("Checking RTP packet flow for track " + track.id); + // Time between stats checks + var retryInterval = 500; + // Timeout in ms + var timeoutInterval = 30000; + // Check hasFlow at a reasonable interval + var checkStats = new Promise((resolve, reject)=>{ + var retries = 0; + var timer = setInterval(()=>{ + this._pc.getStats(track).then(stats=>{ + if (hasFlow(stats, retries)) { + clearInterval(timer); + ok(true, "RTP flowing for " + track.kind + " track " + track.id); + resolve(); + } + retries = retries + 1; + // This is not accurate but it will tear down + // the timer eventually and probably not + // before timeoutInterval has elapsed. + if ((retries * retryInterval) > timeoutInterval) { + clearInterval(timer); + } + }); + }, retryInterval); + }); - var retry = (delay) => this._pc.getStats(track) - .then(stats => hasFlow(stats)? ok(true, "RTP flowing for track " + track.id) : - wait(delay).then(retry(1000))); - return retry(200); + info("Checking RTP packet flow for track " + track.id); + var retry = Promise.race([checkStats.then(new Promise((resolve, reject)=>{ + info("checkStats completed for " + track.kind + " track " + track.id); + resolve(); + })), + new Promise((accept,reject)=>wait(timeoutInterval).then(()=>{ + info("Timeout checking for stats for track " + track.id + " after " + timeoutInterval + "ms"); + reject("Timeout checking for stats for " + track.kind + + " track " + track.id + " after " + timeoutInterval + "ms"); + }) + )]); + + return retry; }, /** * Wait for presence of video flow on all media elements and rtp flow on * all sending and receiving track involved in this test. * * @returns {Promise} * A promise that resolves when media flows for all elements and tracks @@ -1535,17 +1571,19 @@ PeerConnectionWrapper.prototype = { var counters = {}; for (let [key, res] of stats) { // validate stats ok(res.id == key, "Coherent stats id"); var nowish = Date.now() + 1000; // TODO: clock drift observed var minimum = this.whenCreated - 1000; // on Windows XP (Bug 979649) if (isWinXP) { todo(false, "Can't reliably test rtcp timestamps on WinXP (Bug 979649)"); - } else if (!twoMachines) { + + } else if (false) { // Bug 1325430 - timestamps aren't working properly in update 49 + // else if (!twoMachines) { // Bug 1225729: On android, sometimes the first RTCP of the first // test run gets this value, likely because no RTP has been sent yet. if (res.timestamp != 2085978496000) { ok(res.timestamp >= minimum, "Valid " + (res.isRemote? "rtcp" : "rtp") + " timestamp " + res.timestamp + " >= " + minimum + " (" + (res.timestamp - minimum) + " ms)"); ok(res.timestamp <= nowish, @@ -1583,18 +1621,27 @@ PeerConnectionWrapper.prototype = { var rem = stats[res.remoteId]; ok(rem.isRemote, "Remote is rtcp"); ok(rem.remoteId == res.id, "Remote backlink match"); if(res.type == "outboundrtp") { ok(rem.type == "inboundrtp", "Rtcp is inbound"); ok(rem.packetsReceived !== undefined, "Rtcp packetsReceived"); ok(rem.packetsLost !== undefined, "Rtcp packetsLost"); ok(rem.bytesReceived >= rem.packetsReceived, "Rtcp bytesReceived"); - if (!this.disableRtpCountChecking) { - ok(rem.packetsReceived <= res.packetsSent, "No more than sent packets"); + if (false) { // Bug 1325430 if (!this.disableRtpCountChecking) { + // no guarantee which one is newer! + // Note: this must change when we add a timestamp field to remote RTCP reports + // and make rem.timestamp be the reception time + if (res.timestamp >= rem.timestamp) { + ok(rem.packetsReceived <= res.packetsSent, "No more than sent packets"); + } else { + info("REVERSED timestamps: rec:" + + rem.packetsReceived + " time:" + rem.timestamp + " sent:" + res.packetsSent + " time:" + res.timestamp); + } + // Else we may have received more than outdated Rtcp packetsSent ok(rem.bytesReceived <= res.bytesSent, "No more than sent bytes"); } ok(rem.jitter !== undefined, "Rtcp jitter"); ok(rem.mozRtt !== undefined, "Rtcp rtt"); ok(rem.mozRtt >= 0, "Rtcp rtt " + rem.mozRtt + " >= 0"); ok(rem.mozRtt < 60000, "Rtcp rtt " + rem.mozRtt + " < 1 min"); } else { ok(rem.type == "outboundrtp", "Rtcp is outbound");
--- a/dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_2d.html +++ b/dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_2d.html @@ -41,16 +41,28 @@ runNetworkTest(() => { "pcRemote's remote should become green"); }, function PC_LOCAL_DRAW_LOCAL_RED() { // After requesting a frame it will be captured at the time of next render. // Next render will happen at next stable state, at the earliest, // i.e., this order of `requestFrame(); draw();` should work. stream.requestFrame(); h.drawColor(canvas, h.red); + var i = 0; + return setInterval(function() { + try { + info("draw " + i ? "green" : "red"); + h.drawColor(canvas, i ? h.green : h.red); + i = 1 - i; + stream.requestFrame(); + } catch (e) { + // ignore; stream might have shut down, and we don't bother clearing + // the setInterval. + } + }, 500); }, function PC_REMOTE_WAIT_FOR_REMOTE_RED() { return h.waitForPixelColor(mediaElement, h.red, 128, "pcRemote's remote should become red"); } ]); test.run(); });
--- a/dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_webgl.html +++ b/dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_webgl.html @@ -47,17 +47,17 @@ runNetworkTest(() => { test.setMediaConstraints([{video: true}], []); test.chain.replace("PC_LOCAL_GUM", [ function WEBGL_SETUP(test) { var program = WebGLUtil.createProgramByIds(gl, 'v-shader', 'f-shader'); if (!program) { ok(false, "Program should link"); - return Promise.reject(); + return Promise.reject("Program should link"); } gl.useProgram(program); var uColorLocation = gl.getUniformLocation(program, "uColor"); h.setFragmentColorLocation(uColorLocation); var squareBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, squareBuffer); @@ -95,16 +95,26 @@ runNetworkTest(() => { function REQUEST_FRAME(test) { // After requesting a frame it will be captured at the time of next render. // Next render will happen at next stable state, at the earliest, // i.e., this order of `requestFrame(); draw();` should work. test.pcLocal.canvasStream.requestFrame(); }, function DRAW_LOCAL_RED() { h.drawColor(canvas, h.red); + return setInterval(function() { + try { + info("draw"); + h.drawColor(canvas, h.red); + test.pcLocal.canvasStream.requestFrame(); + } catch (e) { + // ignore; stream might have shut down, and we don't bother clearing + // the setInterval. + } + }, 500); }, function WAIT_FOR_REMOTE_RED() { return h.waitForPixelColor(vremote, h.red, 128, "pcRemote's remote should become red"); } ]); test.run(); });
--- a/dom/media/tests/mochitest/test_peerConnection_multiple_captureStream_canvas_2d.html +++ b/dom/media/tests/mochitest/test_peerConnection_multiple_captureStream_canvas_2d.html @@ -67,25 +67,48 @@ runNetworkTest(() => { "pcRemote's remote1 should become blue") ]) ]); }, function DRAW_LOCAL1_RED() { // After requesting a frame it will be captured at the time of next render. // Next render will happen at next stable state, at the earliest, // i.e., this order of `requestFrame(); draw();` should work. + h.drawColor(canvas1, h.red); stream1.requestFrame(); - h.drawColor(canvas1, h.red); + var i = 0; + return setInterval(function() { + try { + info("draw " + i ? "green" : "red"); + h.drawColor(canvas1, i ? h.green : h.red); + i = 1 - i; + stream1.requestFrame(); + } catch (e) { + // ignore; stream might have shut down, and we don't bother clearing + // the setInterval. + } + }, 500); }, function DRAW_LOCAL2_RED() { // After requesting a frame it will be captured at the time of next render. // Next render will happen at next stable state, at the earliest, // i.e., this order of `requestFrame(); draw();` should work. + h.drawColor(canvas2, h.red); stream2.requestFrame(); - h.drawColor(canvas2, h.red); + return setInterval(function() { + try { + info("draw"); + h.drawColor(canvas2, i ? h.green : h.red); + i = 1 - i; + stream2.requestFrame(); + } catch (e) { + // ignore; stream might have shut down, and we don't bother clearing + // the setInterval. + } + }, 500); }, function WAIT_FOR_REMOTE1_RED() { return h.waitForPixelColor(vremote1, h.red, 128, "pcRemote's remote1 should become red"); }, function WAIT_FOR_REMOTE2_RED() { return h.waitForPixelColor(vremote2, h.red, 128, "pcRemote's remote2 should become red");
--- a/dom/media/tests/mochitest/test_peerConnection_simulcastOffer.html +++ b/dom/media/tests/mochitest/test_peerConnection_simulcastOffer.html @@ -55,17 +55,18 @@ ]); test.chain.insertAfter('PC_LOCAL_GET_ANSWER', [ function PC_LOCAL_ADD_RIDS_TO_ANSWER(test) { test._remote_answer.sdp = sdputils.transferSimulcastProperties( test.originalOffer.sdp, test._remote_answer.sdp); info("Answer with RIDs: " + JSON.stringify(test._remote_answer)); ok(test._remote_answer.sdp.match(/a=simulcast:/), "Modified answer has simulcast"); - ok(test._remote_answer.sdp.match(/a=rid:/), "Modified answer has rid"); + ok(test._remote_answer.sdp.match(/a=rid:foo/), "Modified answer has rid foo"); + ok(test._remote_answer.sdp.match(/a=rid:bar/), "Modified answer has rid bar"); } ]); test.chain.insertAfter('PC_REMOTE_WAIT_FOR_MEDIA_FLOW',[ function PC_REMOTE_SET_RTP_FIRST_RID(test) { // Cause pcRemote to filter out everything but the first SSRC. This // lets only one of the simulcast streams through. selectRecvSsrc(test.pcRemote, 0); @@ -80,39 +81,27 @@ }, function PC_REMOTE_CHECK_SIZE_1() { var vlocal = test.pcLocal.localMediaElements[0]; var vremote = test.pcRemote.remoteMediaElements[0]; ok(vlocal, "Should have local video element for pcLocal"); ok(vremote, "Should have remote video element for pcRemote"); ok(vlocal.videoWidth > 0, "source width is positive"); ok(vlocal.videoHeight > 0, "source height is positive"); - is(vremote.videoWidth, vlocal.videoWidth, "sink is same width as source"); - is(vremote.videoHeight, vlocal.videoHeight, "sink is same height as source"); + is(vremote.videoWidth, vlocal.videoWidth / 2, "sink is same width as source"); + is(vremote.videoHeight, vlocal.videoHeight / 2, "sink is same height as source"); }, function PC_REMOTE_SET_RTP_SECOND_RID(test) { // Now, cause pcRemote to filter out everything but the second SSRC. // This lets only the other simulcast stream through. selectRecvSsrc(test.pcRemote, 1); }, function PC_REMOTE_WAIT_FOR_SECOND_MEDIA_FLOW(test) { return test.pcRemote.waitForMediaFlow(); }, - function PC_REMOTE_WAIT_FOR_FRAMES_2() { - var vremote = test.pcRemote.remoteMediaElements[0]; - ok(vremote, "Should have remote video element for pcRemote"); - return helper.waitForFrames(vremote); - }, - // For some reason, even though we're getting a 25x25 stream, sometimes - // the resolution isn't updated on the video element on the first frame. - function PC_REMOTE_WAIT_FOR_FRAMES_3() { - var vremote = test.pcRemote.remoteMediaElements[0]; - ok(vremote, "Should have remote video element for pcRemote"); - return helper.waitForFrames(vremote); - }, function PC_REMOTE_CHECK_SIZE_2() { var vlocal = test.pcLocal.localMediaElements[0]; var vremote = test.pcRemote.remoteMediaElements[0]; ok(vlocal, "Should have local video element for pcLocal"); ok(vremote, "Should have remote video element for pcRemote"); ok(vlocal.videoWidth > 0, "source width is positive"); ok(vlocal.videoHeight > 0, "source height is positive"); is(vremote.videoWidth, vlocal.videoWidth / 2, "sink is 1/2 width of source");
--- a/dom/media/tests/mochitest/test_peerConnection_verifyVideoAfterRenegotiation.html +++ b/dom/media/tests/mochitest/test_peerConnection_verifyVideoAfterRenegotiation.html @@ -46,16 +46,28 @@ runNetworkTest(() => { "pcRemote's remote should become green"); }, function DRAW_LOCAL_RED() { // After requesting a frame it will be captured at the time of next render. // Next render will happen at next stable state, at the earliest, // i.e., this order of `requestFrame(); draw();` should work. stream1.requestFrame(); h1.drawColor(canvas1, h1.red); + var i = 0; + return setInterval(function() { + try { + info("draw " + i ? "green" : "red"); + h1.drawColor(canvas1, i ? h1.green : h1.red); + i = 1 - i; + stream1.requestFrame(); + } catch (e) { + // ignore; stream might have shut down, and we don't bother clearing + // the setInterval. + } + }, 500); }, function WAIT_FOR_REMOTE_RED() { return h1.waitForPixelColor(vremote1, h1.red, 128, "pcRemote's remote should become red"); } ]); addRenegotiation(test.chain,
--- a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp @@ -392,27 +392,9 @@ MediaEngineCameraVideoSource::GetUUID() void MediaEngineCameraVideoSource::SetDirectListeners(bool aHasDirectListeners) { LOG((__FUNCTION__)); mHasDirectListeners = aHasDirectListeners; } -bool operator == (const webrtc::CaptureCapability& a, - const webrtc::CaptureCapability& b) -{ - return a.width == b.width && - a.height == b.height && - a.maxFPS == b.maxFPS && - a.rawType == b.rawType && - a.codecType == b.codecType && - a.expectedCaptureDelay == b.expectedCaptureDelay && - a.interlaced == b.interlaced; -}; - -bool operator != (const webrtc::CaptureCapability& a, - const webrtc::CaptureCapability& b) -{ - return !(a == b); -} - } // namespace mozilla
--- a/dom/media/webrtc/MediaEngineCameraVideoSource.h +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.h @@ -6,25 +6,28 @@ #define MediaEngineCameraVideoSource_h #include "MediaEngine.h" #include "nsDirectoryServiceDefs.h" // conflicts with #include of scoped_ptr.h #undef FF -#include "webrtc/video_engine/include/vie_capture.h" +// Avoid warnings about redefinition of WARN_UNUSED_RESULT +#include "ipc/IPCMessageUtils.h" + +// WebRTC includes +#include "webrtc/modules/video_capture/video_capture_defines.h" + +namespace webrtc { + using CaptureCapability = VideoCaptureCapability; +} namespace mozilla { -bool operator == (const webrtc::CaptureCapability& a, - const webrtc::CaptureCapability& b); -bool operator != (const webrtc::CaptureCapability& a, - const webrtc::CaptureCapability& b); - class MediaEngineCameraVideoSource : public MediaEngineVideoSource { public: // Some subclasses use an index to track multiple instances. explicit MediaEngineCameraVideoSource(int aIndex, const char* aMonitorName = "Camera.Monitor") : MediaEngineVideoSource(kReleased) , mMonitor(aMonitorName)
--- a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -342,49 +342,47 @@ MediaEngineRemoteVideoSource::NotifyPull StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID); if (delta > 0) { // nullptr images are allowed AppendToTrack(aSource, mImage, aID, delta, aPrincipalHandle); } } -int -MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h, - unsigned int streams) +void +MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h) { - mWidth = w; - mHeight = h; - LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u", w, h)); - return 0; +#if defined(MOZ_WIDGET_GONK) + mMonitor.AssertCurrentThreadOwns(); // mWidth and mHeight are protected... +#endif + if ((mWidth < 0) || (mHeight < 0) || + (w != (unsigned int) mWidth) || (h != (unsigned int) mHeight)) { + LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u was %ux%u", w, h, mWidth, mHeight)); + mWidth = w; + mHeight = h; + } } int -MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time, - void *handle) +MediaEngineRemoteVideoSource::DeliverFrame(uint8_t* aBuffer , + const camera::VideoFrameProperties& aProps) { // Check for proper state. if (mState != kStarted) { LOG(("DeliverFrame: video not started")); return 0; } - if ((size_t) (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2))) != size) { - MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!"); - return 0; - } + // Update the dimensions + FrameSizeChange(aProps.width(), aProps.height()); // Create a video frame and append it to the track. RefPtr<layers::PlanarYCbCrImage> image = mImageContainer->CreatePlanarYCbCrImage(); - uint8_t* frame = static_cast<uint8_t*> (buffer); + uint8_t* frame = static_cast<uint8_t*> (aBuffer); const uint8_t lumaBpp = 8; const uint8_t chromaBpp = 4; // Take lots of care to round up! layers::PlanarYCbCrData data; data.mYChannel = frame; data.mYSize = IntSize(mWidth, mHeight); data.mYStride = (mWidth * lumaBpp + 7)/ 8; @@ -399,18 +397,19 @@ MediaEngineRemoteVideoSource::DeliverFra if (!image->CopyData(data)) { MOZ_ASSERT(false); return 0; } #ifdef DEBUG static uint32_t frame_num = 0; - LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %" PRIu64 ", render_time %" PRIu64, - frame_num++, mWidth, mHeight, time_stamp, ntp_time, render_time)); + LOGFRAME(("frame %d (%dx%d); timeStamp %u, ntpTimeMs %" PRIu64 ", renderTimeMs %" PRIu64, + frame_num++, mWidth, mHeight, + aProps.timeStamp(), aProps.ntpTimeMs(), aProps.renderTimeMs())); #endif // we don't touch anything in 'this' until here (except for snapshot, // which has it's own lock) MonitorAutoLock lock(mMonitor); // implicitly releases last image mImage = image.forget();
--- a/dom/media/webrtc/MediaEngineRemoteVideoSource.h +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.h @@ -15,62 +15,53 @@ #include "mozilla/Mutex.h" #include "mozilla/Monitor.h" #include "nsCOMPtr.h" #include "nsThreadUtils.h" #include "DOMMediaStream.h" #include "nsDirectoryServiceDefs.h" #include "nsComponentManagerUtils.h" +// Avoid warnings about redefinition of WARN_UNUSED_RESULT +#include "ipc/IPCMessageUtils.h" #include "VideoUtils.h" #include "MediaEngineCameraVideoSource.h" #include "VideoSegment.h" #include "AudioSegment.h" #include "StreamTracks.h" #include "MediaStreamGraph.h" #include "MediaEngineWrapper.h" #include "mozilla/dom/MediaStreamTrackBinding.h" // WebRTC library includes follow #include "webrtc/common.h" -#include "webrtc/video_engine/include/vie_capture.h" -#include "webrtc/video_engine/include/vie_render.h" + +// Camera Access via IPC #include "CamerasChild.h" #include "NullTransport.h" -namespace webrtc { -class I420VideoFrame; -} - namespace mozilla { /** * The WebRTC implementation of the MediaEngine interface. */ class MediaEngineRemoteVideoSource : public MediaEngineCameraVideoSource, - public webrtc::ExternalRenderer + public camera::FrameRelay { typedef MediaEngineCameraVideoSource Super; public: NS_DECL_THREADSAFE_ISUPPORTS + // Old ExternalRenderer + void FrameSizeChange(unsigned int w, unsigned int h) override; // ExternalRenderer - int FrameSizeChange(unsigned int w, unsigned int h, - unsigned int streams) override; - int DeliverFrame(unsigned char* buffer, - size_t size, - uint32_t time_stamp, - int64_t ntp_time, - int64_t render_time, - void *handle) override; - // XXX!!!! FIX THIS - int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override { return 0; }; - bool IsTextureSupported() override { return false; }; + int DeliverFrame(uint8_t* buffer, + const camera::VideoFrameProperties& properties) override; // MediaEngineCameraVideoSource MediaEngineRemoteVideoSource(int aIndex, mozilla::camera::CaptureEngine aCapEngine, dom::MediaSourceEnum aMediaSource, bool aScary = false, const char* aMonitorName = "RemoteVideo.Monitor"); nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
--- a/dom/media/webrtc/MediaEngineWebRTC.cpp +++ b/dom/media/webrtc/MediaEngineWebRTC.cpp @@ -22,16 +22,17 @@ static mozilla::LazyLogModule sGetUserMe #include "nsIComponentRegistrar.h" #include "MediaEngineTabVideoSource.h" #include "MediaEngineRemoteVideoSource.h" #include "CamerasChild.h" #include "nsITabSource.h" #include "MediaTrackConstraints.h" #ifdef MOZ_WIDGET_ANDROID +#include "VideoEngine.h" #include "AndroidJNIWrapper.h" #include "AndroidBridge.h" #endif #undef LOG #define LOG(args) MOZ_LOG(sGetUserMediaLog, mozilla::LogLevel::Debug, args) namespace mozilla { @@ -148,18 +149,18 @@ MediaEngineWebRTC::EnumerateVideoDevices mozilla::camera::CaptureEngine capEngine = mozilla::camera::InvalidEngine; #ifdef MOZ_WIDGET_ANDROID // get the JVM JavaVM* jvm; JNIEnv* const env = jni::GetEnvForThread(); MOZ_ALWAYS_TRUE(!env->GetJavaVM(&jvm)); - if (webrtc::VideoEngine::SetAndroidObjects(jvm) != 0) { - LOG(("VieCapture:SetAndroidObjects Failed")); + if (!jvm || mozilla::camera::VideoEngine::SetAndroidObjects(jvm)) { + LOG(("VideoEngine::SetAndroidObjects Failed")); return; } #endif bool scaryKind = false; // flag sources with cross-origin exploit potential switch (aMediaSource) { case dom::MediaSourceEnum::Window: capEngine = mozilla::camera::WinEngine;
--- a/dom/media/webrtc/MediaEngineWebRTC.h +++ b/dom/media/webrtc/MediaEngineWebRTC.h @@ -19,49 +19,50 @@ #include "nsAutoPtr.h" #include "nsCOMPtr.h" #include "nsThreadUtils.h" #include "DOMMediaStream.h" #include "nsDirectoryServiceDefs.h" #include "nsComponentManagerUtils.h" #include "nsRefPtrHashtable.h" +#include "ipc/IPCMessageUtils.h" #include "VideoUtils.h" #include "MediaEngineCameraVideoSource.h" #include "VideoSegment.h" #include "AudioSegment.h" #include "StreamTracks.h" #include "MediaStreamGraph.h" #include "cubeb/cubeb.h" #include "CubebUtils.h" #include "AudioPacketizer.h" #include "MediaEngineWrapper.h" #include "mozilla/dom/MediaStreamTrackBinding.h" +#include "CamerasChild.h" + // WebRTC library includes follow #include "webrtc/common.h" // Audio Engine #include "webrtc/voice_engine/include/voe_base.h" #include "webrtc/voice_engine/include/voe_codec.h" #include "webrtc/voice_engine/include/voe_hardware.h" #include "webrtc/voice_engine/include/voe_network.h" #include "webrtc/voice_engine/include/voe_audio_processing.h" #include "webrtc/voice_engine/include/voe_volume_control.h" #include "webrtc/voice_engine/include/voe_external_media.h" #include "webrtc/voice_engine/include/voe_audio_processing.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" // Video Engine // conflicts with #include of scoped_ptr.h #undef FF -#include "webrtc/video_engine/include/vie_base.h" -#include "webrtc/video_engine/include/vie_codec.h" -#include "webrtc/video_engine/include/vie_render.h" -#include "webrtc/video_engine/include/vie_capture.h" -#include "CamerasChild.h" + +// WebRTC imports +#include "webrtc/modules/video_capture/video_capture_defines.h" #include "NullTransport.h" #include "AudioOutputObserver.h" namespace mozilla { class MediaEngineWebRTCAudioCaptureSource : public MediaEngineAudioSource { @@ -271,20 +272,20 @@ public: #ifdef MOZ_WIDGET_ANDROID // OpenSL ES does not support enumerating devices. MOZ_ASSERT(!mDevices); #else MOZ_ASSERT(mDevices); #endif if (mInUseCount == 0) { - ScopedCustomReleasePtr<webrtc::VoEExternalMedia> ptrVoERender; - ptrVoERender = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine); - if (ptrVoERender) { - ptrVoERender->SetExternalRecordingStatus(true); + ScopedCustomReleasePtr<webrtc::VoEExternalMedia> ptrVoEXMedia; + ptrVoEXMedia = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine); + if (ptrVoEXMedia) { + ptrVoEXMedia->SetExternalRecordingStatus(true); } mAnyInUse = true; } mInUseCount++; // Always tell the stream we're using it for input aStream->OpenAudioInput(mSelectedDevice, aListener); } @@ -490,19 +491,19 @@ public: return NS_ERROR_NOT_IMPLEMENTED; } uint32_t GetBestFitnessDistance( const nsTArray<const NormalizedConstraintSet*>& aConstraintSets, const nsString& aDeviceId) const override; // VoEMediaProcess. - void Process(int channel, webrtc::ProcessingTypes type, - int16_t audio10ms[], int length, - int samplingFreq, bool isStereo) override; + virtual void Process(int channel, webrtc::ProcessingTypes type, + int16_t audio10ms[], size_t length, + int samplingFreq, bool isStereo) override; void Shutdown() override; NS_DECL_THREADSAFE_ISUPPORTS protected: ~MediaEngineWebRTCMicrophoneSource() {}
--- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -822,17 +822,17 @@ MediaEngineWebRTCMicrophoneSource::Shutd mAudioInput = nullptr; } typedef int16_t sample; void MediaEngineWebRTCMicrophoneSource::Process(int channel, webrtc::ProcessingTypes type, - sample *audio10ms, int length, + sample *audio10ms, size_t length, int samplingFreq, bool isStereo) { MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode."); // On initial capture, throw away all far-end data except the most recent sample // since it's already irrelevant and we want to keep avoid confusing the AEC far-end // input code with "old" audio. if (!mStarted) { mStarted = true;
--- a/dom/media/webrtc/MediaTrackConstraints.cpp +++ b/dom/media/webrtc/MediaTrackConstraints.cpp @@ -7,16 +7,19 @@ #include "mozilla/dom/MediaStreamTrackBinding.h" #include <limits> #include <algorithm> #include <iterator> namespace mozilla { +using dom::ConstrainBooleanParameters; +using dom::OwningLongOrConstrainLongRange; + template<class ValueType> template<class ConstrainRange> void NormalizedConstraintSet::Range<ValueType>::SetFrom(const ConstrainRange& aOther) { if (aOther.mIdeal.WasPassed()) { mIdeal.emplace(aOther.mIdeal.Value()); }
copy from gfx/layers/client/ClientCanvasLayer.cpp copy to gfx/layers/ShareableCanvasLayer.cpp --- a/gfx/layers/client/ClientCanvasLayer.cpp +++ b/gfx/layers/ShareableCanvasLayer.cpp @@ -1,158 +1,92 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "ClientCanvasLayer.h" +#include "ShareableCanvasLayer.h" + #include "GLContext.h" // for GLContext #include "GLScreenBuffer.h" // for GLScreenBuffer -#include "GeckoProfiler.h" // for PROFILER_LABEL -#include "SharedSurfaceEGL.h" // for SurfaceFactory_EGLImage #include "SharedSurfaceGL.h" // for SurfaceFactory_GLTexture, etc -#include "ClientLayerManager.h" // for ClientLayerManager, etc -#include "mozilla/gfx/Point.h" // for IntSize #include "mozilla/layers/AsyncCanvasRenderer.h" -#include "mozilla/layers/CompositorTypes.h" -#include "mozilla/layers/LayersTypes.h" -#include "nsCOMPtr.h" // for already_AddRefed -#include "nsISupportsImpl.h" // for Layer::AddRef, etc -#include "nsRect.h" // for mozilla::gfx::IntRect -#include "nsXULAppAPI.h" // for XRE_GetProcessType, etc -#include "gfxPrefs.h" // for WebGLForceLayersReadback -#include "gfxUtils.h" #include "mozilla/layers/TextureClientSharedSurface.h" -using namespace mozilla::gfx; -using namespace mozilla::gl; - namespace mozilla { namespace layers { -ClientCanvasLayer::~ClientCanvasLayer() +ShareableCanvasLayer::ShareableCanvasLayer(LayerManager* aLayerManager, void *aImplData) + : CopyableCanvasLayer(aLayerManager, aImplData) + , mFlags(TextureFlags::NO_FLAGS) { - MOZ_COUNT_DTOR(ClientCanvasLayer); + MOZ_COUNT_CTOR(ShareableCanvasLayer); +} + +ShareableCanvasLayer::~ShareableCanvasLayer() +{ + MOZ_COUNT_DTOR(ShareableCanvasLayer); if (mBufferProvider) { mBufferProvider->ClearCachedResources(); } if (mCanvasClient) { mCanvasClient->OnDetach(); mCanvasClient = nullptr; } } void -ClientCanvasLayer::Initialize(const Data& aData) +ShareableCanvasLayer::Initialize(const Data& aData) { CopyableCanvasLayer::Initialize(aData); mCanvasClient = nullptr; if (!mGLContext) return; - GLScreenBuffer* screen = mGLContext->Screen(); + gl::GLScreenBuffer* screen = mGLContext->Screen(); - SurfaceCaps caps; + gl::SurfaceCaps caps; if (mGLFrontbuffer) { // The screen caps are irrelevant if we're using a separate frontbuffer. - caps = mGLFrontbuffer->mHasAlpha ? SurfaceCaps::ForRGBA() - : SurfaceCaps::ForRGB(); + caps = mGLFrontbuffer->mHasAlpha ? gl::SurfaceCaps::ForRGBA() + : gl::SurfaceCaps::ForRGB(); } else { MOZ_ASSERT(screen); caps = screen->mCaps; } MOZ_ASSERT(caps.alpha == aData.mHasAlpha); - auto forwarder = ClientManager()->AsShadowForwarder(); + auto forwarder = GetForwarder(); mFlags = TextureFlags::ORIGIN_BOTTOM_LEFT; if (!aData.mIsGLAlphaPremult) { mFlags |= TextureFlags::NON_PREMULTIPLIED; } - UniquePtr<SurfaceFactory> factory = GLScreenBuffer::CreateFactory(mGLContext, caps, forwarder, mFlags); + UniquePtr<gl::SurfaceFactory> factory = + gl::GLScreenBuffer::CreateFactory(mGLContext, caps, forwarder, mFlags); if (mGLFrontbuffer || aData.mIsMirror) { // We're using a source other than the one in the default screen. // (SkiaGL) mFactory = Move(factory); if (!mFactory) { // Absolutely must have a factory here, so create a basic one - mFactory = MakeUnique<SurfaceFactory_Basic>(mGLContext, caps, mFlags); + mFactory = MakeUnique<gl::SurfaceFactory_Basic>(mGLContext, caps, mFlags); } } else { if (factory) screen->Morph(Move(factory)); } } -void -ClientCanvasLayer::RenderLayer() -{ - PROFILER_LABEL("ClientCanvasLayer", "RenderLayer", - js::ProfileEntry::Category::GRAPHICS); - - RenderMaskLayers(this); - - if (!mCanvasClient) { - TextureFlags flags = TextureFlags::DEFAULT; - if (mOriginPos == gl::OriginPos::BottomLeft) { - flags |= TextureFlags::ORIGIN_BOTTOM_LEFT; - } - - if (!mIsAlphaPremultiplied) { - flags |= TextureFlags::NON_PREMULTIPLIED; - } - - mCanvasClient = CanvasClient::CreateCanvasClient(GetCanvasClientType(), - ClientManager()->AsShadowForwarder(), - flags); - if (!mCanvasClient) { - return; - } - if (HasShadow()) { - if (mAsyncRenderer) { - static_cast<CanvasClientBridge*>(mCanvasClient.get())->SetLayer(this); - } else { - mCanvasClient->Connect(); - ClientManager()->AsShadowForwarder()->Attach(mCanvasClient, this); - } - } - } - - if (mCanvasClient && mAsyncRenderer) { - mCanvasClient->UpdateAsync(mAsyncRenderer); - } - - if (!IsDirty()) { - return; - } - Painted(); - - FirePreTransactionCallback(); - if (mBufferProvider && mBufferProvider->GetTextureClient()) { - if (!mBufferProvider->SetForwarder(ClientManager()->AsShadowForwarder())) { - gfxCriticalNote << "BufferProvider::SetForwarder failed"; - return; - } - mCanvasClient->UpdateFromTexture(mBufferProvider->GetTextureClient()); - } else { - mCanvasClient->Update(gfx::IntSize(mBounds.width, mBounds.height), this); - } - - FireDidTransactionCallback(); - - ClientManager()->Hold(this); - mCanvasClient->Updated(); -} - bool -ClientCanvasLayer::UpdateTarget(DrawTarget* aDestTarget) +ShareableCanvasLayer::UpdateTarget(DrawTarget* aDestTarget) { MOZ_ASSERT(aDestTarget); if (!aDestTarget) { return false; } RefPtr<SourceSurface> surface; @@ -173,21 +107,21 @@ ClientCanvasLayer::UpdateTarget(DrawTarg } aDestTarget->CopySurface(surface, IntRect(0, 0, mBounds.width, mBounds.height), IntPoint(0, 0)); return true; } - SharedSurface* frontbuffer = nullptr; + gl::SharedSurface* frontbuffer = nullptr; if (mGLFrontbuffer) { frontbuffer = mGLFrontbuffer.get(); } else { - GLScreenBuffer* screen = mGLContext->Screen(); + gl::GLScreenBuffer* screen = mGLContext->Screen(); const auto& front = screen->Front(); if (front) { frontbuffer = front->Surf(); } } if (!frontbuffer) { NS_WARNING("Null frame received."); @@ -235,32 +169,70 @@ ClientCanvasLayer::UpdateTarget(DrawTarg aDestTarget->CopySurface(resultSurf, IntRect(0, 0, readSize.width, readSize.height), IntPoint(0, 0)); return true; } CanvasClient::CanvasClientType -ClientCanvasLayer::GetCanvasClientType() +ShareableCanvasLayer::GetCanvasClientType() { if (mAsyncRenderer) { return CanvasClient::CanvasClientAsync; } if (mGLContext) { return CanvasClient::CanvasClientTypeShSurf; } return CanvasClient::CanvasClientSurface; } -already_AddRefed<CanvasLayer> -ClientLayerManager::CreateCanvasLayer() +void +ShareableCanvasLayer::UpdateCompositableClient() { - NS_ASSERTION(InConstruction(), "Only allowed in construction phase"); - RefPtr<ClientCanvasLayer> layer = - new ClientCanvasLayer(this); - CREATE_SHADOW(Canvas); - return layer.forget(); + if (!mCanvasClient) { + TextureFlags flags = TextureFlags::DEFAULT; + if (mOriginPos == gl::OriginPos::BottomLeft) { + flags |= TextureFlags::ORIGIN_BOTTOM_LEFT; + } + + if (!mIsAlphaPremultiplied) { + flags |= TextureFlags::NON_PREMULTIPLIED; + } + + mCanvasClient = CanvasClient::CreateCanvasClient(GetCanvasClientType(), + GetForwarder(), + flags); + if (!mCanvasClient) { + return; + } + + AttachCompositable(); + } + + if (mCanvasClient && mAsyncRenderer) { + mCanvasClient->UpdateAsync(mAsyncRenderer); + } + + if (!IsDirty()) { + return; + } + Painted(); + + FirePreTransactionCallback(); + if (mBufferProvider && mBufferProvider->GetTextureClient()) { + if (!mBufferProvider->SetForwarder(mManager->AsShadowForwarder())) { + gfxCriticalNote << "BufferProvider::SetForwarder failed"; + return; + } + mCanvasClient->UpdateFromTexture(mBufferProvider->GetTextureClient()); + } else { + mCanvasClient->Update(gfx::IntSize(mBounds.width, mBounds.height), this); + } + + FireDidTransactionCallback(); + + mCanvasClient->Updated(); } } // namespace layers } // namespace mozilla
copy from gfx/layers/client/ClientCanvasLayer.h copy to gfx/layers/ShareableCanvasLayer.h --- a/gfx/layers/client/ClientCanvasLayer.h +++ b/gfx/layers/ShareableCanvasLayer.h @@ -1,116 +1,51 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#ifndef GFX_CLIENTCANVASLAYER_H -#define GFX_CLIENTCANVASLAYER_H +#ifndef GFX_SHAREABLECANVASLAYER_H +#define GFX_SHAREABLECANVASLAYER_H -#include "CanvasClient.h" // for CanvasClient, etc -#include "ClientLayerManager.h" // for ClientLayerManager, etc -#include "CopyableCanvasLayer.h" // for CopyableCanvasLayer -#include "Layers.h" // for CanvasLayer, etc -#include "mozilla/Attributes.h" // for override -#include "mozilla/RefPtr.h" // for RefPtr -#include "mozilla/layers/LayersMessages.h" // for CanvasLayerAttributes, etc -#include "mozilla/mozalloc.h" // for operator delete -#include "nsDebug.h" // for NS_ASSERTION -#include "nsISupportsImpl.h" // for MOZ_COUNT_CTOR, etc -#include "nsRegion.h" // for nsIntRegion +#include "CompositorTypes.h" +#include "CopyableCanvasLayer.h" +#include "mozilla/layers/CanvasClient.h" namespace mozilla { namespace gl { class SurfaceFactory; } // namespace gl namespace layers { -class CompositableClient; -class ShadowableLayer; - -class ClientCanvasLayer : public CopyableCanvasLayer, - public ClientLayer +class ShareableCanvasLayer : public CopyableCanvasLayer { typedef CanvasClient::CanvasClientType CanvasClientType; public: - explicit ClientCanvasLayer(ClientLayerManager* aLayerManager) : - CopyableCanvasLayer(aLayerManager, static_cast<ClientLayer*>(this)) - { - MOZ_COUNT_CTOR(ClientCanvasLayer); - } + ShareableCanvasLayer(LayerManager* aLayerManager, void *aImplData); protected: - virtual ~ClientCanvasLayer(); + virtual ~ShareableCanvasLayer(); public: - virtual void SetVisibleRegion(const LayerIntRegion& aRegion) override - { - NS_ASSERTION(ClientManager()->InConstruction(), - "Can only set properties in construction phase"); - CanvasLayer::SetVisibleRegion(aRegion); - } - virtual void Initialize(const Data& aData) override; - virtual void RenderLayer() override; - - virtual void ClearCachedResources() override - { - if (mBufferProvider) { - mBufferProvider->ClearCachedResources(); - } - if (mCanvasClient) { - mCanvasClient->Clear(); - } - } + virtual CompositableForwarder* GetForwarder() = 0; - virtual void HandleMemoryPressure() override - { - if (mBufferProvider) { - mBufferProvider->ClearCachedResources(); - } - if (mCanvasClient) { - mCanvasClient->HandleMemoryPressure(); - } - } - - virtual void FillSpecificAttributes(SpecificLayerAttributes& aAttrs) override - { - aAttrs = CanvasLayerAttributes(mSamplingFilter, mBounds); - } + virtual void AttachCompositable() = 0; - virtual Layer* AsLayer() override { return this; } - virtual ShadowableLayer* AsShadowableLayer() override { return this; } - - virtual void Disconnect() override - { - if (mBufferProvider) { - mBufferProvider->ClearCachedResources(); - } - mCanvasClient = nullptr; - } - - virtual CompositableClient* GetCompositableClient() override - { - return mCanvasClient; - } + void UpdateCompositableClient(); const TextureFlags& Flags() const { return mFlags; } protected: bool UpdateTarget(gfx::DrawTarget* aDestTarget = nullptr); - ClientLayerManager* ClientManager() - { - return static_cast<ClientLayerManager*>(mManager); - } - CanvasClientType GetCanvasClientType(); RefPtr<CanvasClient> mCanvasClient; UniquePtr<gl::SurfaceFactory> mFactory; TextureFlags mFlags;