Bug 1500298 - Refactor TelemetryTestCase and add logging. r=Dexter,whimboo, a=test-only
authorRaphael Pierzina <rpierzina@mozilla.com>
Tue, 06 Nov 2018 11:20:29 +0000
changeset 501068 9ca3d0b3e93a8561ad1f18bcdace54e73f025ecc
parent 501067 2f1f32b76f7ee4fa06d9ad1e7a7929554293f48a
child 501069 7943fd2406300abe14a6aedbae8eb5c1d988443e
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersDexter, whimboo, test-only
bugs1500298
milestone64.0
Bug 1500298 - Refactor TelemetryTestCase and add logging. r=Dexter,whimboo, a=test-only Differential Revision: https://phabricator.services.mozilla.com/D10581
toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py
toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py
toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py
--- a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py
+++ b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py
@@ -1,14 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import re
 import simplejson as json
 import time
 import zlib
 
 from multiprocessing import Process
 
 from firefox_puppeteer import PuppeteerMixin
 from marionette_driver.addons import Addons
@@ -17,24 +16,58 @@ from marionette_driver.wait import Wait
 from marionette_harness import MarionetteTestCase
 from marionette_harness.runner import httpd
 
 
 class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
 
     def __init__(self, *args, **kwargs):
         super(TelemetryTestCase, self).__init__(*args, **kwargs)
-        self.ping_list = []
+        self.pings = []
 
     def setUp(self, *args, **kwargs):
         super(TelemetryTestCase, self).setUp(*args, **kwargs)
+
+        @httpd.handlers.handler
+        def pings_handler(request, response):
+            """Handler for HTTP requests to the ping server."""
+
+            request_data = request.body
+
+            if request.headers.get("Content-Encoding") == "gzip":
+                request_data = zlib.decompress(request_data, zlib.MAX_WBITS | 16)
+
+            ping_data = json.loads(request_data)
+
+            # Store JSON data to self.pings to be used by wait_for_pings()
+            self.pings.append(ping_data)
+
+            ping_type = ping_data["type"]
+
+            log_message = "pings_handler received '{}' ping".format(ping_type)
+
+            if ping_type == "main":
+                ping_reason = ping_data["payload"]["info"]["reason"]
+                log_message = "{} with reason '{}'".format(log_message, ping_reason)
+
+            self.logger.info(log_message)
+
+            status_code = 200
+            content = "OK"
+            headers = [
+                ("Content-Type", "text/plain"),
+                ("Content-Length", len(content)),
+            ]
+
+            return (status_code, headers, content)
+
         self.httpd = httpd.FixtureServer(self.testvars['server_root'])
-        ping_route = [("POST", re.compile('/pings'), self.pings)]
-        self.httpd.routes.extend(ping_route)
+        self.httpd.router.register("POST", '/pings*', pings_handler)
         self.httpd.start()
+
         self.ping_server_url = '{}pings'.format(self.httpd.get_url('/'))
 
         telemetry_prefs = {
             'toolkit.telemetry.server': self.ping_server_url,
             'toolkit.telemetry.initDelay': 1,
             'toolkit.telemetry.minSubsessionLength': 0,
             'datareporting.healthreport.uploadEnabled': True,
             'datareporting.policy.dataSubmissionEnabled': True,
@@ -45,40 +78,50 @@ class TelemetryTestCase(PuppeteerMixin, 
         }
 
         # Firefox will be forced to restart with the prefs enforced.
         self.marionette.enforce_gecko_prefs(telemetry_prefs)
 
         # Wait 5 seconds to ensure that telemetry has reinitialized
         time.sleep(5)
 
-    def wait_for_ping(self, action_func, ping_filter_func):
-        # Filter pings initially using `ping_filter_func`
-        self.ping_list = [p for p in self.ping_list if ping_filter_func(p)]
+    def wait_for_pings(self, action_func, ping_filter_func, count):
+        """Call the given action and wait for pings to come in and return
+        the `count` number of pings, that match the given filter.
+        """
+        # Keep track of the current number of pings
+        current_num_pings = len(self.pings)
 
-        current_num_pings = len(self.ping_list)
-
-        if callable(action_func):
-            action_func()
+        # New list to store new pings that satisfy the filter
+        filtered_pings = []
 
         def wait_func(*args, **kwargs):
-            # Filter pings based on type and reason to make sure right ping is captured.
-            self.ping_list = [p for p in self.ping_list if ping_filter_func(p)]
-            return len(self.ping_list) > current_num_pings
+            # Ignore existing pings in self.pings
+            new_pings = self.pings[current_num_pings:]
+
+            # Filter pings to make sure we wait for the correct ping type
+            filtered_pings[:] = [p for p in new_pings if ping_filter_func(p)]
+
+            return len(filtered_pings) >= count
+
+        self.logger.info(
+            "wait_for_pings running action '{action}'.".format(
+                action=action_func.__name__,
+            )
+        )
+
+        # Call given action and wait for a ping
+        action_func()
 
         try:
             Wait(self.marionette, 60).until(wait_func)
         except Exception as e:
-            self.fail('Error generating ping: {}'.format(e.message))
+            self.fail("Error waiting for ping: {}".format(e.message))
 
-        # TODO: Bug 1380748 - Pings are being cached between test
-        # runs when using --repeat flag in marionette harness
-        # Causes the assert to fail.
-        # assert len(self.ping_list) == 1
-        return self.ping_list.pop()
+        return filtered_pings[:count]
 
     def restart_browser(self):
         """Restarts browser while maintaining the same profile and session."""
         self.restart(clean=False, in_app=True)
 
     def install_addon(self):
         trigger = Process(target=self._install_addon)
         trigger.start()
@@ -107,20 +150,8 @@ class TelemetryTestCase(PuppeteerMixin, 
         ping_data = self.marionette.execute_script(
             'Cu.import("resource://gre/modules/TelemetryController.jsm");'
             'return TelemetryController.getCurrentPingData(true);')
         return ping_data[u'payload'][u'info'][u'subsessionId']
 
     def tearDown(self, *args, **kwargs):
         self.httpd.stop()
         super(TelemetryTestCase, self).tearDown()
-
-    def pings(self, request, response):
-        json_data = json.loads(unpack(request.headers, request.body))
-        self.ping_list.append(json_data)
-        return 200
-
-
-def unpack(headers, data):
-    if "Content-Encoding" in headers and headers["Content-Encoding"] == "gzip":
-        return zlib.decompress(data, zlib.MAX_WBITS | 16)
-    else:
-        return data
--- a/toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py
+++ b/toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py
@@ -32,16 +32,17 @@ class TestMainTabScalars(TelemetryTestCa
             tab3 = self.browser.tabbar.open_tab()
             self.browser.tabbar.switch_to(tab3)
 
             self.browser.tabbar.close_tab(tab3, force=True)
             self.browser.tabbar.close_tab(tab2, force=True)
 
             self.browser.tabbar.switch_to(tab1)
 
-        ping = self.wait_for_ping(self.restart_browser, check_is_shutdown_ping)
+        [ping] = self.wait_for_pings(self.restart_browser, check_is_shutdown_ping, 1)
+
         assert ping["type"] == "main"
         assert ping["clientId"] == self.client_id
 
         scalars = ping["payload"]["processes"]["parent"]["scalars"]
         assert scalars["browser.engagement.max_concurrent_tab_count"] == 3
         assert scalars["browser.engagement.tab_open_event_count"] == 2
         assert scalars["browser.engagement.max_concurrent_window_count"] == 1
--- a/toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py
+++ b/toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py
@@ -7,17 +7,17 @@ import requests
 from telemetry_harness.testcase import TelemetryTestCase
 
 
 class TestPingServer(TelemetryTestCase):
     def test_ping_server_received_ping(self):
         ping_type = "server-test-ping"
         ping_reason = "unit-test"
 
-        def action_func():
+        def send_ping_request():
             """Perform a POST request to the ping server."""
             data = {"type": ping_type, "reason": ping_reason}
             headers = {"Content-type": "application/json", "Accept": "text/plain"}
 
             response = requests.post(self.ping_server_url, json=data, headers=headers)
 
             self.assertEqual(
                 response.status_code,
@@ -26,13 +26,13 @@ class TestPingServer(TelemetryTestCase):
                     response=response
                 ),
             )
             return response
 
         def ping_filter_func(ping):
             return ping["type"] == ping_type
 
-        ping = self.wait_for_ping(action_func, ping_filter_func)
+        [ping] = self.wait_for_pings(send_ping_request, ping_filter_func, 1)
 
         self.assertIsNotNone(ping)
         self.assertEqual(ping["type"], ping_type)
         self.assertEqual(ping["reason"], ping_reason)