Bug 1456209 - 3. Scale timeouts based on device; r=gbrown
authorJim Chen <nchen@mozilla.com>
Tue, 24 Apr 2018 10:13:35 -0400
changeset 468948 bb4f5395f6dcb81d7073b71e07faa30a594ea064
parent 468947 f6dd07476c05f9eca4b049218cec5d84d88455ba
child 468949 bffecfd92808acbf5f3ab2d16482ac8f020c5892
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1456209
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1456209 - 3. Scale timeouts based on device; r=gbrown Scale the default test timeout based on the architecture, and whether we're running inside an emulator or on a real device. This is done to account for performance differences in the device under test. We have some tests that are expected to timeout. In order to maintain accuracy of these tests across devices, their custom timeout values are scaled as well. MozReview-Commit-ID: 7a65eAAPHOB
mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
--- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
+++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
@@ -1,16 +1,17 @@
 /* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.geckoview.test.rule;
 
 import org.mozilla.gecko.gfx.GeckoDisplay;
+import org.mozilla.geckoview.BuildConfig;
 import org.mozilla.geckoview.GeckoRuntime;
 import org.mozilla.geckoview.GeckoRuntimeSettings;
 import org.mozilla.geckoview.GeckoSession;
 import org.mozilla.geckoview.GeckoSessionSettings;
 import org.mozilla.geckoview.test.util.Callbacks;
 
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.assertThat;
@@ -19,16 +20,17 @@ import org.hamcrest.Matcher;
 
 import org.junit.rules.ErrorCollector;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 
 import android.app.Instrumentation;
 import android.graphics.Point;
 import android.graphics.SurfaceTexture;
+import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
 import android.os.Handler;
 import android.os.Looper;
 import android.os.Message;
 import android.os.MessageQueue;
 import android.os.SystemClock;
 import android.support.annotation.NonNull;
@@ -69,17 +71,22 @@ import kotlin.reflect.KClass;
  * and tears down the GeckoSession at the end of the test. The rule also provides methods
  * for waiting on particular callbacks to be called, and methods for asserting that
  * callbacks are called in the proper order.
  */
 public class GeckoSessionTestRule extends UiThreadTestRule {
     private static final String LOGTAG = "GeckoSessionTestRule";
 
     private static final long DEFAULT_TIMEOUT_MILLIS = 10000;
-    private static final long DEFAULT_DEBUG_TIMEOUT_MILLIS = 86400000;
+    private static final long DEFAULT_ARM_DEVICE_TIMEOUT_MILLIS = 30000;
+    private static final long DEFAULT_ARM_EMULATOR_TIMEOUT_MILLIS = 120000;
+    private static final long DEFAULT_X86_DEVICE_TIMEOUT_MILLIS = 30000;
+    private static final long DEFAULT_X86_EMULATOR_TIMEOUT_MILLIS = 5000;
+    private static final long DEFAULT_IDE_DEBUG_TIMEOUT_MILLIS = 86400000;
+
     public static final String APK_URI_PREFIX = "resource://android/";
 
     private static final Method sOnLocationChange;
     private static final Method sOnPageStop;
 
     static {
         try {
             sOnLocationChange = GeckoSession.NavigationDelegate.class.getMethod(
@@ -87,18 +94,20 @@ public class GeckoSessionTestRule extend
             sOnPageStop = GeckoSession.ProgressDelegate.class.getMethod(
                     "onPageStop", GeckoSession.class, boolean.class);
         } catch (final NoSuchMethodException e) {
             throw new RuntimeException(e);
         }
     }
 
     /**
-     * Specify the timeout for any of the wait methods, in milliseconds. Can be used
-     * on classes or methods.
+     * Specify the timeout for any of the wait methods, in milliseconds, relative to
+     * {@link #DEFAULT_TIMEOUT_MILLIS}. When the default timeout scales to account
+     * for differences in the device under test, the timeout value here will be
+     * scaled as well. Can be used on classes or methods.
      */
     @Target({ElementType.METHOD, ElementType.TYPE})
     @Retention(RetentionPolicy.RUNTIME)
     public @interface TimeoutMillis {
         long value();
     }
 
     /**
@@ -415,16 +424,28 @@ public class GeckoSessionTestRule extend
             return Boolean.valueOf(InstrumentationRegistry.getArguments()
                                                           .getString("use_multiprocess",
                                                                      "true"));
         }
 
         public boolean isDebugging() {
             return Debug.isDebuggerConnected();
         }
+
+        public boolean isEmulator() {
+            return "generic".equals(Build.DEVICE) || Build.DEVICE.startsWith("generic_");
+        }
+
+        public boolean isDebugBuild() {
+            return BuildConfig.DEBUG_BUILD;
+        }
+
+        public String getCPUArch() {
+            return BuildConfig.ANDROID_CPU_ARCH;
+        }
     }
 
     protected class CallbackDelegates {
         private final Map<Pair<GeckoSession, Method>, MethodCall> mDelegates = new HashMap<>();
         private int mOrder;
 
         public void delegate(final @Nullable GeckoSession session,
                              final @NonNull Object callback) {
@@ -642,17 +663,20 @@ public class GeckoSessionTestRule extend
             throws NoSuchMethodException {
         return GeckoSession.class.getMethod("get" + cls.getSimpleName());
     }
 
     protected void applyAnnotations(final Collection<Annotation> annotations,
                                     final GeckoSessionSettings settings) {
         for (final Annotation annotation : annotations) {
             if (TimeoutMillis.class.equals(annotation.annotationType())) {
-                mTimeoutMillis = Math.max(((TimeoutMillis) annotation).value(), 100);
+                // Scale timeout based on the default timeout to account for the device under test.
+                final long value = ((TimeoutMillis) annotation).value();
+                final long timeout = value * getDefaultTimeoutMillis() / DEFAULT_TIMEOUT_MILLIS;
+                mTimeoutMillis = Math.max(timeout, 1000);
             } else if (Setting.class.equals(annotation.annotationType())) {
                 ((Setting) annotation).key().set(settings, ((Setting) annotation).value());
             } else if (Setting.List.class.equals(annotation.annotationType())) {
                 for (final Setting setting : ((Setting.List) annotation).value()) {
                     setting.key().set(settings, setting.value());
                 }
             } else if (WithDisplay.class.equals(annotation.annotationType())) {
                 final WithDisplay displaySize = (WithDisplay)annotation;
@@ -669,20 +693,29 @@ public class GeckoSessionTestRule extend
             return (RuntimeException) cause;
         } else if (e instanceof RuntimeException) {
             return (RuntimeException) e;
         }
 
         return new RuntimeException(cause != null ? cause : e);
     }
 
+    private long getDefaultTimeoutMillis() {
+        if ("x86".equals(env.getCPUArch())) {
+            return env.isEmulator() ? DEFAULT_X86_EMULATOR_TIMEOUT_MILLIS
+                                    : DEFAULT_X86_DEVICE_TIMEOUT_MILLIS;
+        }
+        return env.isEmulator() ? DEFAULT_ARM_EMULATOR_TIMEOUT_MILLIS
+                                : DEFAULT_ARM_DEVICE_TIMEOUT_MILLIS;
+    }
+
     protected void prepareStatement(final Description description) throws Throwable {
         final GeckoSessionSettings settings = new GeckoSessionSettings(mDefaultSettings);
-        mTimeoutMillis = env.isDebugging() ? DEFAULT_DEBUG_TIMEOUT_MILLIS
-                                           : DEFAULT_TIMEOUT_MILLIS;
+        mTimeoutMillis = env.isDebugging() ? DEFAULT_IDE_DEBUG_TIMEOUT_MILLIS
+                                           : getDefaultTimeoutMillis();
         mClosedSession = false;
 
         applyAnnotations(Arrays.asList(description.getTestClass().getAnnotations()), settings);
         applyAnnotations(description.getAnnotations(), settings);
 
         final List<CallRecord> records = new ArrayList<>();
         final CallbackDelegates waitDelegates = new CallbackDelegates();
         final CallbackDelegates testDelegates = new CallbackDelegates();
@@ -804,17 +837,17 @@ public class GeckoSessionTestRule extend
                         }
                         return true;
                     }
                     return false;
                 }
             };
 
             do {
-                loopUntilIdle(DEFAULT_TIMEOUT_MILLIS);
+                loopUntilIdle(getDefaultTimeoutMillis());
             } while (mCallRecordHandler != null);
 
         } finally {
             mCallRecordHandler = null;
         }
     }
 
     /**