Backed out changeset c1b764e0ddb3 (bug 1601009) for massive raptor failures.
authorCosmin Sabou <csabou@mozilla.com>
Tue, 04 Feb 2020 12:17:59 +0200
changeset 512449 3bcf437014eeb859253e928e36b4384475ecb259
parent 512448 1091ce1d58ece67823629aadfebde0f432cb5f96
child 512450 024d269299e62b7527bc4faaec6a7ce5fd61464c
push id37089
push usernerli@mozilla.com
push dateTue, 04 Feb 2020 16:20:28 +0000
treeherdermozilla-central@a117660624ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1601009
milestone74.0a1
backs outc1b764e0ddb311109436603c252cc0f556831843
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset c1b764e0ddb3 (bug 1601009) for massive raptor failures. CLOSED TREE
taskcluster/ci/test/browsertime-mobile.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/browsertime/android.py
testing/raptor/raptor/browsertime/base.py
testing/raptor/raptor/perftest.py
testing/raptor/raptor/webextension/base.py
testing/raptor/test/test_raptor.py
--- a/taskcluster/ci/test/browsertime-mobile.yml
+++ b/taskcluster/ci/test/browsertime-mobile.yml
@@ -91,17 +91,8 @@ browsertime-tp6m-profiling:
     treeherder-symbol: Btime-Prof(tp6m)
     mozharness:
         script: raptor_script.py
         config:
             - raptor/android_hw_config.py
         extra-options:
             - --browsertime
             - --gecko-profile
-
-browsertime-benchmark-speedometer:
-    description: "Browsertime speedometer on android"
-    apps: ["geckoview", "fenix", "fennec"]
-    treeherder-symbol: Btime(sp)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --test=raptor-speedometer
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -989,14 +989,12 @@ android-hw-aarch64-raptor-cpu-memory-pow
     - raptor-scn-cpu-memory-power-idle-refbrow
     - raptor-scn-cpu-memory-power-idle-bg-fenix
     - raptor-scn-cpu-memory-power-idle-bg-geckoview
     - raptor-scn-cpu-memory-power-idle-bg-refbrow
 
 android-hw-aarch64-browsertime:
     - browsertime-tp6m
     - browsertime-tp6m-profiling
-    - browsertime-benchmark-speedometer
 
 android-hw-arm7-browsertime:
     - browsertime-tp6m
     - browsertime-tp6m-profiling
-    - browsertime-benchmark-speedometer
--- a/testing/raptor/raptor/browsertime/android.py
+++ b/testing/raptor/raptor/browsertime/android.py
@@ -95,19 +95,17 @@ class BrowsertimeAndroid(PerftestAndroid
 
         self.clear_app_data()
         self.set_debug_app_flag()
 
     def run_test_setup(self, test):
         super(BrowsertimeAndroid, self).run_test_setup(test)
 
         self.set_reverse_ports()
-
-        if self.playback:
-            self.turn_on_android_app_proxy()
+        self.turn_on_android_app_proxy()
         self.remove_mozprofile_delimiters_from_profile()
 
     def run_tests(self, tests, test_names):
         self.setup_adb_device()
 
         return super(BrowsertimeAndroid, self).run_tests(tests, test_names)
 
     def run_test_teardown(self, test):
--- a/testing/raptor/raptor/browsertime/base.py
+++ b/testing/raptor/raptor/browsertime/base.py
@@ -14,20 +14,17 @@ import six
 
 import mozprocess
 from benchmark import Benchmark
 from logger.logger import RaptorLogger
 from perftest import Perftest
 from results import BrowsertimeResultsHandler
 
 LOG = RaptorLogger(component="raptor-browsertime")
-
 DEFAULT_CHROMEVERSION = "77"
-BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT = 120  # 2 minutes
-BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = 900  # 15 minutes
 
 
 class Browsertime(Perftest):
     """Abstract base class for Browsertime"""
 
     __metaclass__ = ABCMeta
 
     @property
@@ -289,17 +286,15 @@ class Browsertime(Perftest):
             env["PATH"] = new_path
 
         LOG.info("PATH: {}".format(env["PATH"]))
 
         try:
             proc = self.process_handler(cmd, env=env)
             proc.run(
                 timeout=self._compute_process_timeout(test, timeout),
-                outputTimeout=BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT
-                if self.benchmark
-                else BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT,
+                outputTimeout=2 * 60,
             )
             proc.wait()
 
         except Exception as e:
             LOG.critical("Error while attempting to run browsertime: %s" % str(e))
             raise
--- a/testing/raptor/raptor/perftest.py
+++ b/testing/raptor/raptor/perftest.py
@@ -139,16 +139,17 @@ either Raptor or browsertime."""
 
         self.browser_name = None
         self.browser_version = None
 
         self.raptor_venv = os.path.join(os.getcwd(), "raptor-venv")
         self.installerpath = installerpath
         self.playback = None
         self.benchmark = None
+        self.benchmark_port = 0
         self.gecko_profiler = None
         self.post_startup_delay = post_startup_delay
         self.device = None
         self.profile_class = profile_class or app
         self.conditioned_profile_dir = None
         self.interrupt_handler = interrupt_handler
         self.results_handler = results_handler_class(**self.config)
 
@@ -506,19 +507,19 @@ class PerftestAndroid(Perftest):
                 self.set_reverse_port(self.control_server.port)
 
             if self.playback:
                 LOG.info("making the raptor playback server port available to device")
                 self.set_reverse_port(self.playback.port)
 
             if self.benchmark:
                 LOG.info("making the raptor benchmarks server port available to device")
-                self.set_reverse_port(int(self.benchmark.port))
+                self.set_reverse_port(self.benchmark_port)
         else:
-            LOG.info("Reverse port forwarding is used only on local devices")
+            LOG.info("Reverse port forwarding is uded only on local devices")
 
     def build_browser_profile(self):
         super(PerftestAndroid, self).build_browser_profile()
 
         # Merge in the Android profile.
         path = os.path.join(self.profile_data_dir, "raptor-android")
         LOG.info("Merging profile: {}".format(path))
         self.profile.merge(path)
--- a/testing/raptor/raptor/webextension/base.py
+++ b/testing/raptor/raptor/webextension/base.py
@@ -61,17 +61,17 @@ class WebExtension(Perftest):
             self.serve_benchmark_source(test)
 
         gen_test_config(
             self.config["app"],
             test["name"],
             self.control_server.port,
             self.post_startup_delay,
             host=self.config["host"],
-            b_port=int(self.benchmark.port),
+            b_port=self.benchmark_port,
             debug_mode=1 if self.debug_mode else 0,
             browser_cycle=test.get("browser_cycle", 1),
         )
 
         self.install_raptor_webext()
 
     def wait_for_test_finish(self, test, timeout):
         # this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some
@@ -149,16 +149,17 @@ class WebExtension(Perftest):
         self.control_server.start()
 
         if self.config["enable_control_server_wait"]:
             self.control_server_wait_set("webext_shutdownBrowser")
 
     def serve_benchmark_source(self, test):
         # benchmark-type tests require the benchmark test to be served out
         self.benchmark = Benchmark(self.config, test)
+        self.benchmark_port = int(self.benchmark.port)
 
     def install_raptor_webext(self):
         # must intall raptor addon each time because we dynamically update some content
         # the webext is installed into the browser profile
         # note: for chrome the addon is just a list of paths that ultimately are added
         # to the chromium command line '--load-extension' argument
         self.raptor_webext = os.path.join(webext_dir, "raptor")
         LOG.info("installing webext %s" % self.raptor_webext)
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -321,17 +321,17 @@ def test_compute_process_timeout(
 )
 def test_android_reverse_ports(host, playback, benchmark):
     raptor = WebExtensionAndroid(
         "geckoview", "org.mozilla.geckoview_example", host=host
     )
     if benchmark:
         benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark")
         raptor.benchmark = benchmark_mock
-        raptor.benchmark.port = 1234
+        raptor.benchmark_port = 1234
 
     if playback:
         playback_mock = mock.patch(
             "mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid"
         )
         playback_mock.port = 4321
         raptor.playback = playback_mock