Bug 1169476 -- Implement |mach robocop --serve|. r=gbrown
authorNick Alexander <nalexander@mozilla.com>
Fri, 29 May 2015 17:18:07 -0700
changeset 247125 2feb31e63a451392b33f0d7f29ff052f846f5810
parent 247124 2923c5cc5dcc87f4fd6e186bba163d3f0094820f
child 247126 9b110d0cbbf605497bca4c72989c5da6e349269c
push id28854
push userryanvm@gmail.com
push dateThu, 04 Jun 2015 13:24:20 +0000
treeherdermozilla-central@5b4c240e1a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1169476
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1169476 -- Implement |mach robocop --serve|. r=gbrown This adds a flag to |mach robocop| that does everything to run a Robocop test except launch the actual test. Instead of launching the test, it starts the mochi.test server and launches Fennec with a test profile; then it sits and waits forever. This allows regular Java IDEs (IntelliJ, but previously Eclipse) to run Robocop tests like regular instrumentation tests, "injecting" them into the prepared testing environment. It's quite nice!
build/mobile/remoteautomation.py
build/mobile/robocop/AndroidManifest.xml.in
build/mobile/robocop/LaunchFennecWithConfigurationActivity.java
build/mobile/robocop/Makefile.in
mobile/android/tests/browser/robocop/BaseRobocopTest.java
mobile/android/tests/browser/robocop/BaseTest.java
mobile/android/tests/browser/robocop/UITest.java
testing/mochitest/mach_commands.py
testing/mochitest/runtestsremote.py
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -19,16 +19,20 @@ import mozcrash
 # signatures for logcat messages that we don't care about much
 fennecLogcatFilters = [ "The character encoding of the HTML document was not declared",
                         "Use of Mutation Events is deprecated. Use MutationObserver instead.",
                         "Unexpected value from nativeGetEnabledTags: 0" ]
 
 class RemoteAutomation(Automation):
     _devicemanager = None
 
+    # Part of a hack for Robocop: "am COMMAND" is handled specially if COMMAND
+    # is in this set. See usages below.
+    _specialAmCommands = ('instrument', 'start')
+
     def __init__(self, deviceManager, appName = '', remoteLog = None,
                  processArgs=None):
         self._devicemanager = deviceManager
         self._appName = appName
         self._remoteProfile = None
         self._remoteLog = remoteLog
         self._processArgs = processArgs or {};
 
@@ -232,17 +236,17 @@ class RemoteAutomation(Automation):
 
     def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
         # If remote profile is specified, use that instead
         if (self._remoteProfile):
             profileDir = self._remoteProfile
 
         # Hack for robocop, if app & testURL == None and extraArgs contains the rest of the stuff, lets
         # assume extraArgs is all we need
-        if app == "am" and extraArgs[0] == "instrument":
+        if app == "am" and extraArgs[0] in RemoteAutomation._specialAmCommands:
             return app, extraArgs
 
         cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
         # Remove -foreground if it exists, if it doesn't this just returns
         try:
             args.remove('-foreground')
         except:
             pass
@@ -270,17 +274,17 @@ class RemoteAutomation(Automation):
             self.messageLogger = messageLogger
 
             if (self.proc is None):
                 if cmd[0] == 'am':
                     self.proc = stdout
                 else:
                     raise Exception("unable to launch process")
             self.procName = cmd[0].split('/')[-1]
-            if cmd[0] == 'am' and cmd[1] == "instrument":
+            if cmd[0] == 'am' and cmd[1] in RemoteAutomation._specialAmCommands:
                 self.procName = app
                 print "Robocop process name: "+self.procName
 
             # Setting timeout at 1 hour since on a remote device this takes much longer.
             # Temporarily increased to 75 minutes because no more chunks can be created.
             self.timeout = 4500
             # The benefit of the following sleep is unclear; it was formerly 15 seconds
             time.sleep(1)
--- a/build/mobile/robocop/AndroidManifest.xml.in
+++ b/build/mobile/robocop/AndroidManifest.xml.in
@@ -3,17 +3,21 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="org.mozilla.roboexample.test"
 #ifdef MOZ_ANDROID_SHARED_ID
     android:sharedUserId="@MOZ_ANDROID_SHARED_ID@"
 #endif
     android:versionCode="1"
     android:versionName="1.0" >
 
-    <uses-sdk android:minSdkVersion="8" />
+    <uses-sdk android:minSdkVersion="@MOZ_ANDROID_MIN_SDK_VERSION@"
+#ifdef MOZ_ANDROID_MAX_SDK_VERSION
+              android:maxSdkVersion="@MOZ_ANDROID_MAX_SDK_VERSION@"
+#endif
+              android:targetSdkVersion="@ANDROID_TARGET_SDK@"/>
 
     <instrumentation
         android:name="org.mozilla.gecko.FennecInstrumentationTestRunner"
         android:targetPackage="@ANDROID_PACKAGE_NAME@" />
 
     <application
         android:label="@string/app_name" >
         <uses-library android:name="android.test.runner" />
@@ -38,11 +42,19 @@
                 <action android:name="android.intent.action.SEND" />
                 <category android:name="android.intent.category.DEFAULT" />
                 <data android:mimeType="text/*" />
                 <data android:mimeType="image/*" />
             </intent-filter>
 
         </activity>
 
+        <activity android:name="org.mozilla.gecko.LaunchFennecWithConfigurationActivity"
+                  android:label="Robocop Fennec">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.DEFAULT" />
+            </intent-filter>
+        </activity>
+
     </application>
 
 </manifest>
new file mode 100644
--- /dev/null
+++ b/build/mobile/robocop/LaunchFennecWithConfigurationActivity.java
@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko;
+
+import java.util.Map;
+
+import org.mozilla.gecko.tests.BaseRobocopTest;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.os.Bundle;
+
+/**
+ * An Activity that extracts Robocop settings from robotium.config, launches
+ * Fennec with the Robocop testing parameters, and finishes itself.
+ * <p>
+ * This is intended to be used by local testers using |mach robocop --serve|.
+ */
+public class LaunchFennecWithConfigurationActivity extends Activity {
+    @Override
+    public void onCreate(Bundle arguments) {
+        super.onCreate(arguments);
+    }
+
+    @Override
+    public void onResume() {
+        super.onResume();
+
+        final String configFile = FennecNativeDriver.getFile(BaseRobocopTest.DEFAULT_ROOT_PATH + "/robotium.config");
+        final Map<String, String> config = FennecNativeDriver.convertTextToTable(configFile);
+        final Intent intent = BaseRobocopTest.createActivityIntent(config);
+
+        intent.setClassName(AppConstants.ANDROID_PACKAGE_NAME, AppConstants.MOZ_ANDROID_BROWSER_INTENT_CLASS);
+
+        this.finish();
+        this.startActivity(intent);
+    }
+}
--- a/build/mobile/robocop/Makefile.in
+++ b/build/mobile/robocop/Makefile.in
@@ -19,16 +19,17 @@ ANDROID_ASSETS_DIR := $(TESTPATH)/assets
   Driver.java \
   Element.java \
   FennecInstrumentationTestRunner.java \
   FennecNativeActions.java \
   FennecMochitestAssert.java \
   FennecTalosAssert.java \
   FennecNativeDriver.java \
   FennecNativeElement.java \
+  LaunchFennecWithConfigurationActivity.java \
   RoboCopException.java \
   RobocopShare1.java \
   RobocopShare2.java \
   RobocopUtils.java \
   PaintedSurface.java \
   StructuredLogger.java \
   $(NULL)
 
--- a/mobile/android/tests/browser/robocop/BaseRobocopTest.java
+++ b/mobile/android/tests/browser/robocop/BaseRobocopTest.java
@@ -1,49 +1,57 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.tests;
 
-import java.util.Map;
+import android.app.Activity;
+import android.content.Context;
+import android.content.Intent;
+import android.os.PowerManager;
+import android.test.ActivityInstrumentationTestCase2;
+import android.text.TextUtils;
+import android.util.Log;
+
+import com.jayway.android.robotium.solo.Solo;
 
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.mozilla.gecko.Actions;
 import org.mozilla.gecko.AppConstants;
 import org.mozilla.gecko.Assert;
+import org.mozilla.gecko.BrowserApp;
 import org.mozilla.gecko.Driver;
 import org.mozilla.gecko.FennecInstrumentationTestRunner;
 import org.mozilla.gecko.FennecMochitestAssert;
 import org.mozilla.gecko.FennecNativeActions;
 import org.mozilla.gecko.FennecNativeDriver;
 import org.mozilla.gecko.FennecTalosAssert;
+import org.mozilla.gecko.GeckoAppShell;
+import org.mozilla.gecko.GeckoEvent;
 import org.mozilla.gecko.updater.UpdateServiceHelper;
 
-import android.app.Activity;
-import android.content.Context;
-import android.content.Intent;
-import android.content.res.Resources;
-import android.os.PowerManager;
-import android.test.ActivityInstrumentationTestCase2;
-import android.util.Log;
-
-import com.jayway.android.robotium.solo.Solo;
+import java.util.Map;
 
 @SuppressWarnings("unchecked")
 public abstract class BaseRobocopTest extends ActivityInstrumentationTestCase2<Activity> {
+    public static final String LOGTAG = "BaseTest";
+
     public enum Type {
         MOCHITEST,
         TALOS
     }
 
-    private static final String DEFAULT_ROOT_PATH = "/mnt/sdcard/tests";
+    public static final String DEFAULT_ROOT_PATH = "/mnt/sdcard/tests";
+
+    // How long to wait for a Robocop:Quit message to actually kill Fennec.
+    private static final int ROBOCOP_QUIT_WAIT_MS = 180000;
 
     /**
      * The Java Class instance that launches the browser.
      * <p>
      * This should always agree with {@link AppConstants#MOZ_ANDROID_BROWSER_INTENT_CLASS}.
      */
     public static final Class<? extends Activity> BROWSER_INTENT_CLASS;
 
@@ -71,18 +79,16 @@ public abstract class BaseRobocopTest ex
     protected Solo mSolo;
     protected Driver mDriver;
     protected Actions mActions;
 
     protected String mProfile;
 
     protected StringHelper mStringHelper;
 
-    protected abstract Intent createActivityIntent();
-
     /**
      * The browser is started at the beginning of this test. A single test is a
      * class inheriting from <code>BaseRobocopTest</code> that contains test
      * methods.
      * <p>
      * If a test should not start the browser at the beginning of a test,
      * specify a different activity class to the one-argument constructor. To do
      * as little as possible, specify <code>Activity.class</code>.
@@ -107,16 +113,40 @@ public abstract class BaseRobocopTest ex
      * <p>
      * By default tests are mochitests, but a test can override this method in
      * order to change its type. Most Robocop tests are mochitests.
      */
     protected Type getTestType() {
         return Type.MOCHITEST;
     }
 
+    // Member function to allow specialization.
+    protected Intent createActivityIntent() {
+        return BaseRobocopTest.createActivityIntent(mConfig);
+    }
+
+    // Static function to allow re-use.
+    public static Intent createActivityIntent(Map<String, String> config) {
+        final Intent intent = new Intent(Intent.ACTION_MAIN);
+        intent.putExtra("args", "-no-remote -profile " + config.get("profile"));
+        // Don't show the first run experience.
+        intent.putExtra(BrowserApp.EXTRA_SKIP_STARTPANE, true);
+
+        final String envString = config.get("envvars");
+        if (!TextUtils.isEmpty(envString)) {
+            final String[] envStrings = envString.split(",");
+
+            for (int iter = 0; iter < envStrings.length; iter++) {
+                intent.putExtra("env" + iter, envStrings[iter]);
+            }
+        }
+
+        return intent;
+    }
+
     @Override
     protected void setUp() throws Exception {
         // Disable the updater.
         UpdateServiceHelper.setEnabled(false);
 
         // Load config file from root path (set up by Python script).
         mRootPath = FennecInstrumentationTestRunner.getFennecArguments().getString("deviceroot");
         if (mRootPath == null) {
@@ -147,17 +177,53 @@ public abstract class BaseRobocopTest ex
         Activity tempActivity = getActivity();
 
         StringHelper.initialize(tempActivity.getResources());
         mStringHelper = StringHelper.get();
 
         mSolo = new Solo(getInstrumentation(), tempActivity);
         mDriver = new FennecNativeDriver(tempActivity, mSolo, mRootPath);
         mActions = new FennecNativeActions(tempActivity, mSolo, getInstrumentation(), mAsserter);
+    }
 
+    @Override
+    public void tearDown() throws Exception {
+        try {
+            mAsserter.endTest();
+
+            // By default, we don't quit Fennec on finish, and we don't finish
+            // all opened activities. Not quiting Fennec entirely is intended to
+            // make life better for local testers, who might want to alter a
+            // test that is under development rather than Fennec itself. Not
+            // finishing activities is intended to allow local testers to
+            // manually inspect an activity's state after a test
+            // run. runtestsremote.py sets this to "1".  Testers running via an
+            // IDE will not have this set at all.
+            final String quitAndFinish = FennecInstrumentationTestRunner.getFennecArguments()
+                    .getString("quit_and_finish"); // null means not specified.
+            if ("1".equals(quitAndFinish)) {
+                // Request the browser force quit and wait for it to take effect.
+                Log.i(LOGTAG, "Requesting force quit.");
+                GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Robocop:Quit", null));
+                mSolo.sleep(ROBOCOP_QUIT_WAIT_MS);
+
+                // If still running, finish activities as recommended by Robotium.
+                Log.i(LOGTAG, "Finishing all opened activities.");
+                mSolo.finishOpenedActivities();
+            } else {
+                // This has the effect of keeping the activity-under-test
+                // around; if we don't set it to null, it is killed, either by
+                // finishOpenedActivities above or super.tearDown below.
+                Log.i(LOGTAG, "Not requesting force quit and trying to keep started activity alive.");
+                setActivity(null);
+            }
+        } catch (Throwable e) {
+            e.printStackTrace();
+        }
+        super.tearDown();
     }
 
     /**
      * Function to early abort if we can't reach the given HTTP server. Provides local testers
      * with diagnostic information. Not currently available for TALOS tests, which are rarely run
      * locally in any case.
      */
     public void throwIfHttpGetFails() {
--- a/mobile/android/tests/browser/robocop/BaseTest.java
+++ b/mobile/android/tests/browser/robocop/BaseTest.java
@@ -139,48 +139,16 @@ abstract class BaseTest extends BaseRobo
                 mAsserter.dumpLog("Exception caught during test!", t);
                 mAsserter.ok(false, "Exception caught", t.toString());
             }
             // re-throw to continue bail-out
             throw t;
         }
     }
 
-    @Override
-    public void tearDown() throws Exception {
-        try {
-            mAsserter.endTest();
-            // request a force quit of the browser and wait for it to take effect
-            GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Robocop:Quit", null));
-            mSolo.sleep(120000);
-            // if still running, finish activities as recommended by Robotium
-            mSolo.finishOpenedActivities();
-        } catch (Throwable e) {
-            e.printStackTrace();
-        }
-        super.tearDown();
-    }
-
-    @Override
-    protected Intent createActivityIntent() {
-        final Intent intent = new Intent(Intent.ACTION_MAIN);
-        intent.putExtra("args", "-no-remote -profile " + mProfile);
-
-        final String envString = mConfig.get("envvars");
-        if (!TextUtils.isEmpty(envString)) {
-            final String[] envStrings = envString.split(",");
-
-            for (int iter = 0; iter < envStrings.length; iter++) {
-                intent.putExtra("env" + iter, envStrings[iter]);
-            }
-        }
-
-        return intent;
-    }
-
     public void assertMatches(String value, String regex, String name) {
         if (value == null) {
             mAsserter.ok(false, name, "Expected /" + regex + "/, got null");
             return;
         }
         mAsserter.ok(value.matches(regex), name, "Expected /" + regex +"/, got \"" + value + "\"");
     }
 
--- a/mobile/android/tests/browser/robocop/UITest.java
+++ b/mobile/android/tests/browser/robocop/UITest.java
@@ -54,32 +54,16 @@ abstract class UITest extends BaseRoboco
         initHelpers();
 
         // Ensure Robocop tests have access to network, and are run with Display powered on.
         throwIfHttpGetFails();
         throwIfScreenNotOn();
     }
 
     @Override
-    public void tearDown() throws Exception {
-        try {
-            mAsserter.endTest();
-            // request a force quit of the browser and wait for it to take effect
-            GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Robocop:Quit", null));
-            mSolo.sleep(120000);
-            // if still running, finish activities as recommended by Robotium
-            mSolo.finishOpenedActivities();
-        } catch (Throwable e) {
-            e.printStackTrace();
-        }
-
-        super.tearDown();
-    }
-
-    @Override
     protected void runTest() throws Throwable {
         try {
             super.runTest();
         } catch (Throwable t) {
             // save screenshot -- written to /mnt/sdcard/Robotium-Screenshots
             // as <filename>.jpg
             mSolo.takeScreenshot("robocop-screenshot");
             if (mAsserter != null) {
@@ -177,36 +161,16 @@ abstract class UITest extends BaseRoboco
     public String getAbsoluteIpUrl(final String url) {
         return getAbsoluteUrl(mBaseIpUrl, url);
     }
 
     private String getAbsoluteUrl(final String baseUrl, final String url) {
         return baseUrl + "/" + url.replaceAll("(^/)", "");
     }
 
-    @Override
-    protected Intent createActivityIntent() {
-        final Intent intent = new Intent(Intent.ACTION_MAIN);
-
-        // Don't show the first run experience.
-        intent.putExtra(BrowserApp.EXTRA_SKIP_STARTPANE, true);
-        intent.putExtra("args", "-no-remote -profile " + mProfile);
-
-        final String envString = mConfig.get("envvars");
-        if (!TextUtils.isEmpty(envString)) {
-            final String[] envStrings = envString.split(",");
-
-            for (int iter = 0; iter < envStrings.length; iter++) {
-                intent.putExtra("env" + iter, envStrings[iter]);
-            }
-        }
-
-        return intent;
-    }
-
     /**
      * Throws an Exception. Called from overridden JUnit methods to ensure JUnit assertions
      * are not accidentally used over AssertionHelper assertions (the latter of which contains
      * additional logging facilities for use in our test harnesses).
      */
     private static void junit() {
         throw new UnsupportedOperationException(JUNIT_FAILURE_MSG);
     }
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -575,17 +575,23 @@ class RobocopCommands(MachCommandBase):
     @Command('robocop', category='testing',
              conditions=[conditions.is_android],
              description='Run a Robocop test.',
              parser=setup_argument_parser)
     @CommandArgument('test_paths', nargs='*', metavar='TEST', default=None,
                      help='Test to run. Can be a single Robocop test file (like "testLoad.java") '
                           ' or a directory of tests '
                           '(to run recursively). If omitted, the entire Robocop suite is run.')
-    def run_robocop(self, test_paths, **kwargs):
+    @CommandArgument('--serve', default=False, action='store_true',
+        help='Run no tests but start the mochi.test web server and launch '
+             'Fennec with a test profile.')
+    def run_robocop(self, test_paths, serve=False, **kwargs):
+        if serve:
+            kwargs['autorun'] = False
+
         if not kwargs.get('robocopIni'):
             kwargs['robocopIni'] = os.path.join(self.topobjdir, '_tests', 'testing',
                                                 'mochitest', 'robocop.ini')
 
         if not kwargs.get('robocopApk'):
             kwargs['robocopApk'] = os.path.join(self.topobjdir, 'build', 'mobile',
                                                 'robocop', 'robocop-debug.apk')
 
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -538,16 +538,21 @@ def run_test_harness(options):
         options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
         options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
         options.extraPrefs.append('browser.snippets.enabled=false')
         options.extraPrefs.append('browser.casting.enabled=true')
 
         if (options.dm_trans == 'adb' and options.robocopApk):
             dm._checkCmd(["install", "-r", options.robocopApk])
 
+        if not options.autorun:
+            # Force a single loop iteration. The iteration will start Fennec and
+            # the httpd server, but not actually run a test.
+            options.testPath = robocop_tests[0]['name']
+
         retVal = None
         # Filtering tests
         active_tests = []
         for test in robocop_tests:
             if options.testPath and options.testPath != test['name']:
                 continue
 
             if 'disabled' in test:
@@ -565,30 +570,46 @@ def run_test_harness(options):
             # each cycle
             if mochitest.localProfile:
                 options.profilePath = mochitest.localProfile
                 os.system("rm -Rf %s" % options.profilePath)
                 options.profilePath = None
                 mochitest.localProfile = options.profilePath
 
             options.app = "am"
-            options.browserArgs = [
-                "instrument",
-                "-w",
-                "-e",
-                "deviceroot",
-                deviceRoot,
-                "-e",
-                "class"]
-            options.browserArgs.append(
-                "org.mozilla.gecko.tests.%s" %
-                test['name'].split('.java')[0])
-            options.browserArgs.append(
-                "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
             mochitest.nsprLogName = "nspr-%s.log" % test['name']
+            if options.autorun:
+                # This launches a test (using "am instrument") and instructs
+                # Fennec to /quit/ the browser (using Robocop:Quit) and to
+                # /finish/ all opened activities.
+                options.browserArgs = [
+                    "instrument",
+                    "-w",
+                    "-e", "quit_and_finish", "1",
+                    "-e", "deviceroot", deviceRoot,
+                    "-e",
+                    "class"]
+                options.browserArgs.append(
+                    "org.mozilla.gecko.tests.%s" %
+                    test['name'].split('.java')[0])
+                options.browserArgs.append(
+                    "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
+            else:
+                # This does not launch a test at all. It launches an activity
+                # that starts Fennec and then waits indefinitely, since cat
+                # never returns.
+                options.browserArgs = ["start",
+                                       "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
+                                       "&&", "cat"]
+                dm.default_timeout = sys.maxint # Forever.
+
+                mochitest.log.info("")
+                mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
+                    (options.remoteWebServer, options.httpPort))
+                mochitest.log.info("")
 
             # If the test is for checking the import from bookmarks then make
             # sure there is data to import
             if test['name'] == "testImportFromAndroid":
 
                 # Get the OS so we can run the insert in the apropriate
                 # database and following the correct table schema
                 osInfo = dm.getInfo("os")