Bug 1324470 - Add support for |mach test-info|; r=gps,ekyle
authorGeoff Brown <gbrown@mozilla.com>
Tue, 07 Feb 2017 09:42:29 -0700
changeset 387857 edd70bcf7b44c70fc64fd27850e35a64ca3c96ad
parent 387856 206f2c7479adce00d865c5ea5a64ce5df8dd758c
child 387858 8cf10b012ee2f667abe4b30c0ea91e7f88587cdc
push id7198
push userjlorenzo@mozilla.com
push dateTue, 18 Apr 2017 12:07:49 +0000
treeherdermozilla-beta@d57aa49c3948 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps, ekyle
bugs1324470
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1324470 - Add support for |mach test-info|; r=gps,ekyle The test-info command provides a report of test results for a named test on a set of branches over a time period. Test results include a pass/fail summary, average/min/max duration, and open bugs. Branches and time period can be modified with optional parameters.
testing/mach_commands.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -824,8 +824,284 @@ class ChunkFinder(MachCommandBase):
                 break
 
         if not found:
             raise Exception("Test %s not found." % test_path)
         # Clean up the file
         os.remove(dump_tests)
         if temp_dir:
             shutil.rmtree(temp_dir)
+
+
+@CommandProvider
+class TestInfoCommand(MachCommandBase):
+    from datetime import date, timedelta
+    @Command('test-info', category='testing',
+        description='Display historical test result summary.')
+    @CommandArgument('test_name', nargs='?', metavar='N',
+        help='Test of interest.')
+    @CommandArgument('--branches',
+        default='mozilla-central,mozilla-inbound,autoland',
+        help='Report for named branches (default: mozilla-central,mozilla-inbound,autoland)')
+    @CommandArgument('--start',
+        default=(date.today() - timedelta(7)).strftime("%Y-%m-%d"),
+        help='Start date (YYYY-MM-DD)')
+    @CommandArgument('--end',
+        default=date.today().strftime("%Y-%m-%d"),
+        help='End date (YYYY-MM-DD)')
+
+    def test_info(self, **params):
+
+        import which
+        from mozbuild.base import MozbuildObject
+
+        self.test_name = params['test_name']
+        self.branches = params['branches']
+        self.start = params['start']
+        self.end = params['end']
+
+        if len(self.test_name) < 6:
+            print("'%s' is too short for a test name!" % self.test_name)
+            return
+
+        here = os.path.abspath(os.path.dirname(__file__))
+        build_obj = MozbuildObject.from_environment(cwd=here)
+
+        self._hg = None
+        if conditions.is_hg(build_obj):
+            if self._is_windows():
+                self._hg = which.which('hg.exe')
+            else:
+                self._hg = which.which('hg')
+
+        self._git = None
+        if conditions.is_git(build_obj):
+            if self._is_windows():
+                self._git = which.which('git.exe')
+            else:
+                self._git = which.which('git')
+
+        self.set_test_name()
+        self.report_test_results()
+        self.report_test_durations()
+        self.report_bugs()
+
+    def find_in_hg_or_git(self, test_name):
+        if self._hg:
+            cmd = [self._hg, 'files', '-I', test_name]
+        elif self._git:
+            cmd = [self._git, 'ls-files', test_name]
+        else:
+            return None
+        try:
+            out = subprocess.check_output(cmd).splitlines()
+        except subprocess.CalledProcessError:
+            out = None
+        return out
+
+    def set_test_name(self):
+        # Generating a unified report for a specific test is complicated
+        # by differences in the test name used in various data sources.
+        # Consider:
+        #   - It is often convenient to request a report based only on
+        #     a short file name, rather than the full path;
+        #   - Bugs may be filed in bugzilla against a simple, short test
+        #     name or the full path to the test;
+        #   - In ActiveData, the full path is usually used, but sometimes
+        #     also includes additional path components outside of the
+        #     mercurial repo (common for reftests).
+        # This function attempts to find appropriate names for different
+        # queries based on the specified test name.
+
+        import re
+
+        # full_test_name is full path to file in hg (or git)
+        self.full_test_name = None
+        out = self.find_in_hg_or_git(self.test_name)
+        if out and len(out) == 1:
+            self.full_test_name = out[0]
+        elif out and len(out) > 1:
+            print("Ambiguous test name specified. Found:")
+            for line in out:
+                print(line)
+        else:
+            out = self.find_in_hg_or_git('**/%s*' % self.test_name)
+            if out and len(out) == 1:
+                self.full_test_name = out[0]
+            elif out and len(out) > 1:
+                print("Ambiguous test name. Found:")
+                for line in out:
+                    print(line)
+        if self.full_test_name:
+            print("Found %s in source control." % self.full_test_name)
+        else:
+            print("Unable to validate test name '%s'!" % self.test_name)
+            self.full_test_name = self.test_name
+
+        # short_name is full_test_name without path
+        self.short_name = None
+        name_idx = self.full_test_name.rfind('/')
+        if name_idx > 0:
+            self.short_name = self.full_test_name[name_idx+1:]
+
+        # robo_name is short_name without ".java" - for robocop
+        self.robo_name = None
+        if self.short_name:
+            robo_idx = self.short_name.rfind('.java')
+            if robo_idx > 0:
+                self.robo_name = self.short_name[:robo_idx]
+            if self.short_name == self.test_name:
+                self.short_name = None
+
+        # activedata_test_name is name in ActiveData
+        self.activedata_test_name = None
+        simple_names = [
+            self.full_test_name,
+            self.test_name,
+            self.short_name,
+            self.robo_name
+        ]
+        simple_names = [x for x in simple_names if x]
+        searches = [
+            {"in": {"result.test": simple_names}},
+        ]
+        regex_names = [".*%s.*" % re.escape(x) for x in simple_names if x]
+        for r in regex_names:
+            searches.append({"regexp": {"result.test": r}})
+        query = {
+            "from": "unittest",
+            "format": "list",
+            "limit": 10,
+            "groupby": ["result.test"],
+            "where": {"and": [
+                {"or": searches},
+                {"in": {"build.branch": self.branches.split(',')}},
+                {"gt": {"run.timestamp": {"date": self.start}}},
+                {"lt": {"run.timestamp": {"date": self.end}}}
+            ]}
+        }
+        print("Querying ActiveData...") # Following query can take a long time
+        data = self.submit(query)
+        if data and len(data) > 0:
+            self.activedata_test_name = [
+                d['result']['test']
+                for p in simple_names + regex_names
+                for d in data
+                if re.match(p+"$", d['result']['test'])
+            ][0]  # first match is best match
+        if self.activedata_test_name:
+            print("Found records matching '%s' in ActiveData." %
+                self.activedata_test_name)
+        else:
+            print("Unable to find matching records in ActiveData; using %s!" %
+                self.test_name)
+            self.activedata_test_name = self.test_name
+
+    def get_platform(self, record):
+        platform = record['build']['platform']
+        type = record['build']['type']
+        e10s = "-%s" % record['run']['type'] if 'run' in record else ""
+        return "%s/%s%s:" % (platform, type, e10s)
+
+    def submit(self, query):
+        import requests
+        response = requests.post("http://activedata.allizom.org/query",
+                                 data=json.dumps(query),
+                                 stream=True)
+        response.raise_for_status()
+        data = response.json()["data"]
+        return data
+
+    def report_test_results(self):
+        # Report test pass/fail summary from ActiveData
+        query = {
+            "from": "unittest",
+            "format": "list",
+            "limit": 100,
+            "groupby": ["build.platform", "build.type", "run.type"],
+            "select": [
+                {"aggregate": "count"},
+                {
+                    "name": "failures",
+                    "value": {"case": [
+                        {"when": {"eq": {"result.ok": "F"}}, "then": 1}
+                    ]},
+                    "aggregate": "sum",
+                    "default": 0
+                }
+            ],
+            "where": {"and": [
+                {"eq": {"result.test": self.activedata_test_name}},
+                {"in": {"build.branch": self.branches.split(',')}},
+                {"gt": {"run.timestamp": {"date": self.start}}},
+                {"lt": {"run.timestamp": {"date": self.end}}}
+            ]}
+        }
+        print("\nTest results for %s on %s between %s and %s" %
+            (self.activedata_test_name, self.branches, self.start, self.end))
+        data = self.submit(query)
+        if data and len(data) > 0:
+            data.sort(key=self.get_platform)
+            for record in data:
+                platform = self.get_platform(record)
+                runs = record['count']
+                failures = record['failures']
+                print("%-30s %6d failures in %6d runs" % (
+                    platform, failures, runs))
+        else:
+            print("No test result data found.")
+
+    def report_test_durations(self):
+        # Report test durations summary from ActiveData
+        query = {
+	    "from": "unittest",
+            "format": "list",
+	    "limit": 100,
+	    "groupby": ["build.platform","build.type","run.type"],
+	    "select": [
+		{"value":"result.duration","aggregate":"average","name":"average"},
+		{"value":"result.duration","aggregate":"min","name":"min"},
+		{"value":"result.duration","aggregate":"max","name":"max"},
+		{"aggregate":"count"}
+	    ],
+	    "where": {"and": [
+                {"eq": {"result.ok": "T"}},
+		{"eq": {"result.test": self.activedata_test_name}},
+                {"in": {"build.branch": self.branches.split(',')}},
+                {"gt": {"run.timestamp": {"date": self.start}}},
+                {"lt": {"run.timestamp": {"date": self.end}}}
+	    ]}
+        }
+        data = self.submit(query)
+        print("\nTest durations for %s on %s between %s and %s" %
+            (self.activedata_test_name, self.branches, self.start, self.end))
+        if data and len(data) > 0:
+            data.sort(key=self.get_platform)
+            for record in data:
+                platform = self.get_platform(record)
+                print("%-30s %6.2f s (%.2f s - %.2f s over %d runs)" % (
+                    platform, record['average'], record['min'],
+                    record['max'], record['count']))
+        else:
+            print("No test durations found.")
+
+    def report_bugs(self):
+        # Report open bugs matching test name
+        import requests
+        search = self.full_test_name
+        if self.test_name:
+            search = '%s,%s' % (search, self.test_name)
+        if self.short_name:
+            search = '%s,%s' % (search, self.short_name)
+        if self.robo_name:
+            search = '%s,%s' % (search, self.robo_name)
+        payload = {'quicksearch': search,
+                   'include_fields':'id,summary'}
+        response = requests.get('https://bugzilla.mozilla.org/rest/bug',
+                                payload)
+        response.raise_for_status()
+        json_response = response.json()
+        print("\nBugzilla quick search for '%s':" % search)
+        if 'bugs' in json_response:
+            for bug in json_response['bugs']:
+                print("Bug %s: %s" % (bug['id'], bug['summary']))
+        else:
+            print("No bugs found.")