Bug 1525851: Add semi-automated test. r=jdescottes
authorDaisuke Akatsuka <dakatsuka@mozilla.com>
Tue, 26 Feb 2019 05:25:56 +0000
changeset 461046 110ea2a7c3d4f34b5079c195f7ea57966748e6da
parent 461045 95fa937ab43270890c88437fb7bd80ef8ce4480c
child 461047 9018a6cb05789db2a2f4a09eea3df881a289a37b
child 461195 7ad5d74c3240164caf11b5784958988294af05cc
child 461199 4a87b6ac14c030bea86c2ccd5a9a48c0470a29dc
push id78980
push userdakatsuka@mozilla.com
push dateTue, 26 Feb 2019 05:36:43 +0000
treeherderautoland@110ea2a7c3d4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjdescottes
bugs1525851
milestone67.0a1
first release with
nightly linux32
110ea2a7c3d4 / 67.0a1 / 20190226093755 / files
nightly linux64
110ea2a7c3d4 / 67.0a1 / 20190226093755 / files
nightly win32
110ea2a7c3d4 / 67.0a1 / 20190226093755 / files
nightly win64
110ea2a7c3d4 / 67.0a1 / 20190226093755 / files
nightly mac
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly win32
nightly win64
Bug 1525851: Add semi-automated test. r=jdescottes Differential Revision: https://phabricator.services.mozilla.com/D20117
devtools/client/aboutdebugging-new/documentation/TESTS_REAL_DEVICES.md
devtools/client/aboutdebugging-new/test/browser/browser.ini
devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_runtime_page_runtime_info.js
devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_sidebar.js
devtools/client/aboutdebugging-new/test/browser/helper-real-usb.js
devtools/client/aboutdebugging-new/test/browser/resources/real/usb-runtimes-sample.json
new file mode 100644
--- /dev/null
+++ b/devtools/client/aboutdebugging-new/documentation/TESTS_REAL_DEVICES.md
@@ -0,0 +1,67 @@
+# Running Tests with real USB connected devices for the new about:debugging
+
+This document explains how to test with real USB connected devices.
+
+## Tests overview
+
+The tests that use a real device are located in `devtools/client/aboutdebugging-new/test/browser/`and the name of tests starts with `browser_aboutdebugging_real`. These are normal mochitest, but we need to setup the environment before starting tests.
+
+## Setup environment
+### Real device side
+1. Enable USB debugging on your device
+2. Launch Firefox
+3. Enable USB debugging on your Firefox
+4. Connect to your PC via USB
+
+You can refer to https://developer.mozilla.org/docs/Tools/Remote_Debugging/Debugging_Firefox_for_Android_with_WebIDE#Setting_up_the_Android_device.
+
+### PC side
+Setup the real device information to evaluate the validity in tests.
+
+1. Copy a sample file which is located at `devtools/client/aboutdebugging-new/test/browser/real/usb-runtimes-sample.json` and rename it for example to `devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json`.
+2. Edit the file.
+
+   This is a JSON file like below, write your real device information in here. This example indicates that there should be one USB device and should be displayed `Pixel 2` as device name and `Firefox Nightly` as short name on the sidebar of about:debugging. Regarding the other information, please see `Detail of config file` section of this document.
+
+```
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2",
+      "shortName": "Firefox Nightly"
+    },
+    ...
+  },
+  ...
+]
+```
+
+## Test
+Normally, although we do test `./mach mochitest <path>`, to load the real device information created, specify the path as `USB_RUNTIME` environment variable, then do test.
+For example, if the name of the saved file is `devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json`, run all real device test with a command like the one below:
+
+```
+USB_RUNTIMES=devtools/client/aboutdebugging-new/test/browser/real/local-usb-runtimes.json ./mach mochitest devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real
+```
+
+If there is no `USB_RUNTIMES` environment variable, the tests will not run. This is also to avoid to run on try-server and so on.
+
+## Detail of config file
+
+```
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2", // This should display as device name on the sidebar.
+      "shortName": "Firefox Nightly" // This should display as short name on the sidebar.
+    },
+    "runtimeDetails": {
+      "info": {
+        "name": "Mozilla Nightly", // This should display on the runtime info of runtime page.
+        "version": "64.0a1" // This should display on the runtime info of runtime page.
+      }
+    }
+  }
+  // Of course, you can append additional USB devices. Some tests can do with multiple devices.
+]
+```
\ No newline at end of file
--- a/devtools/client/aboutdebugging-new/test/browser/browser.ini
+++ b/devtools/client/aboutdebugging-new/test/browser/browser.ini
@@ -6,16 +6,17 @@ prefs =
   # ensure consistent test behavior by always setting this to false.
   devtools.aboutdebugging.showSystemAddons=false
 support-files =
   head.js
   helper-adb.js
   helper-addons.js
   helper-collapsibilities.js
   helper-mocks.js
+  helper-real-usb.js
   helper-serviceworker.js
   helper-telemetry.js
   mocks/*
   resources/bad-extension/*
   resources/packaged-extension/*
   resources/service-workers/*
   resources/test-adb-extension/*
   resources/test-temporary-extension/*
@@ -57,16 +58,18 @@ skip-if = (os == 'linux' && bits == 32) 
 [browser_aboutdebugging_devtoolstoolbox_menubar.js]
 [browser_aboutdebugging_devtoolstoolbox_reload.js]
 [browser_aboutdebugging_devtoolstoolbox_shortcuts.js]
 skip-if = (os == "win" && ccov) # Bug 1521349
 [browser_aboutdebugging_devtoolstoolbox_tooltip_markupview.js]
 [browser_aboutdebugging_navigate.js]
 [browser_aboutdebugging_persist_connection.js]
 [browser_aboutdebugging_profiler_dialog.js]
+[browser_aboutdebugging_real_usb_runtime_page_runtime_info.js]
+[browser_aboutdebugging_real_usb_sidebar.js]
 [browser_aboutdebugging_routes.js]
 [browser_aboutdebugging_runtime_compatibility_warning.js]
 [browser_aboutdebugging_runtime_remote_runtime_buttons.js]
 [browser_aboutdebugging_runtime_usbclient_closed.js]
 [browser_aboutdebugging_select_network_runtime.js]
 [browser_aboutdebugging_select_page_with_serviceworker.js]
 [browser_aboutdebugging_serviceworker_fetch_flag.js]
 [browser_aboutdebugging_serviceworker_multie10s.js]
new file mode 100644
--- /dev/null
+++ b/devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_runtime_page_runtime_info.js
@@ -0,0 +1,47 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-real-usb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-real-usb.js", this);
+
+// Test that runtime info of USB runtime appears on main pane.
+// Documentation for real usb tests in /documentation/TESTS_REAL_DEVICES.md
+add_task(async function() {
+  if (!isAvailable()) {
+    ok(true, "Real usb runtime test is not available");
+    return;
+  }
+
+  const { document, tab } = await openAboutDebuggingWithADB();
+
+  const expectedRuntime = await getExpectedRuntime();
+  const { runtimeDetails, sidebarInfo } = expectedRuntime;
+
+  info("Connect a USB runtime");
+  await Promise.race([
+    connectToRuntime(sidebarInfo.deviceName, document),
+    /* eslint-disable mozilla/no-arbitrary-setTimeout */
+    new Promise(resolve => setTimeout(() => {
+      ok(false,
+         "Failed to connect, did you disable the connection prompt for this runtime?");
+      resolve();
+    }, 5000)),
+    /* eslint-enable mozilla/no-arbitrary-setTimeout */
+  ]);
+
+  info("Select a USB runtime");
+  await selectRuntime(sidebarInfo.deviceName, runtimeDetails.info.name, document);
+
+  info("Check that runtime info is properly displayed");
+  const runtimeInfo = document.querySelector(".js-runtime-info");
+  ok(runtimeInfo, "Runtime info is displayed");
+  const runtimeInfoText = runtimeInfo.textContent;
+  ok(runtimeInfoText.includes(runtimeDetails.info.name),
+     "Runtime info shows the correct runtime name: " + runtimeInfoText);
+  ok(runtimeInfoText.includes(runtimeDetails.info.version),
+     "Runtime info shows the correct version number: " + runtimeInfoText);
+
+  await removeTab(tab);
+});
new file mode 100644
--- /dev/null
+++ b/devtools/client/aboutdebugging-new/test/browser/browser_aboutdebugging_real_usb_sidebar.js
@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-real-usb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-real-usb.js", this);
+
+// Test that USB runtimes appear from the sidebar.
+// Documentation for real usb tests in /documentation/TESTS_REAL_DEVICES.md
+add_task(async function() {
+  if (!isAvailable()) {
+    ok(true, "Real usb runtime test is not available");
+    return;
+  }
+
+  const { document, tab } = await openAboutDebuggingWithADB();
+
+  for (const { sidebarInfo } of await getExpectedRuntimeAll()) {
+    const { deviceName, shortName } = sidebarInfo;
+    await waitUntil(() => findSidebarItemByText(deviceName, document));
+    const usbRuntimeSidebarItem = findSidebarItemByText(deviceName, document);
+    ok(usbRuntimeSidebarItem.textContent.includes(shortName),
+       "The device name and short name of the usb runtime are visible in sidebar item " +
+       `[${usbRuntimeSidebarItem.textContent}]`);
+  }
+
+  await removeTab(tab);
+});
new file mode 100644
--- /dev/null
+++ b/devtools/client/aboutdebugging-new/test/browser/helper-real-usb.js
@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/* import-globals-from helper-adb.js */
+Services.scriptloader.loadSubScript(CHROME_URL_ROOT + "helper-adb.js", this);
+
+async function getExpectedRuntime() {
+  const runtimes = await getExpectedRuntimeAll();
+  return runtimes[0];
+}
+/* exported getExpectedRuntime */
+
+async function getExpectedRuntimeAll() {
+  const runtimesPath = _getExpectedRuntimesPath();
+  const currentPath = env.get("PWD");
+  const path = `${currentPath}/${runtimesPath}`;
+  info(`Load ${ path }`);
+  const buffer = await OS.File.read(path);
+  const data = new TextDecoder().decode(buffer);
+  return JSON.parse(data);
+}
+/* exported getExpectedRuntimeAll */
+
+function isAvailable() {
+  return !!_getExpectedRuntimesPath();
+}
+/* exported isAvailable */
+
+async function openAboutDebuggingWithADB() {
+  const { document, tab, window } = await openAboutDebugging();
+
+  await pushPref("devtools.remote.adb.extensionURL",
+                 CHROME_URL_ROOT + "resources/test-adb-extension/adb-extension-#OS#.xpi");
+  await checkAdbNotRunning();
+
+  const { adbAddon } = require("devtools/shared/adb/adb-addon");
+  adbAddon.install("internal");
+  const usbStatusElement = document.querySelector(".js-sidebar-usb-status");
+  await waitUntil(() => usbStatusElement.textContent.includes("USB devices enabled"));
+  await waitForAdbStart();
+
+  return { document, tab, window };
+}
+/* exported openAboutDebuggingWithADB */
+
+function _getExpectedRuntimesPath() {
+  const env = Cc["@mozilla.org/process/environment;1"].getService(Ci.nsIEnvironment);
+  return env.get("USB_RUNTIMES");
+}
new file mode 100644
--- /dev/null
+++ b/devtools/client/aboutdebugging-new/test/browser/resources/real/usb-runtimes-sample.json
@@ -0,0 +1,14 @@
+[
+  {
+    "sidebarInfo": {
+      "deviceName": "Pixel 2",
+      "shortName": "Firefox Nightly"
+    },
+    "runtimeDetails": {
+      "info": {
+        "name": "Mozilla Nightly",
+        "version": "64.0a1"
+      }
+    }
+  }
+]