Backed out changeset ac5d5dec618e (bug 1481590) for Gecko decision task failure
authorNoemi Erli <nerli@mozilla.com>
Wed, 15 Aug 2018 06:13:37 +0300
changeset 486681 7f9ad37449b67138ff06a5ebe04076844c0011c8
parent 486680 13cca9e3e12d0575378a137090f74827a0a9843d
child 486682 5f016db599045a909accd50f942afb96e740790c
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1481590
milestone63.0a1
backs outac5d5dec618e067abf88eb4ba7dc557723ccb003
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset ac5d5dec618e (bug 1481590) for Gecko decision task failure
python/mozbuild/mozbuild/analyze/graph.py
python/mozbuild/mozbuild/analyze/hg.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/test/analyze/test_graph.py
--- a/python/mozbuild/mozbuild/analyze/graph.py
+++ b/python/mozbuild/mozbuild/analyze/graph.py
@@ -106,17 +106,17 @@ class Graph(object):
 
     def file_summaries(self, files):
         for f in files:
             node = self.get_node(self.get_id(f))
             if node is not None:
                 sec = node.cost / 1000.0
                 m, s = sec / 60, sec % 60
                 print ("\n------ Summary for %s ------\
-                    \nTotal Build Time (mm:ss) = %d:%d\nNum Downstream Commands = %d"
+                    \nTotal cost (mm:ss) = %d:%d\nNum Downstream Commands = %d"
                     % (f, m, s, node.num_cmds))
 
     def populate(self):
         # make nodes for files with downstream commands
         files = self.query('SELECT id FROM node WHERE type=0 AND id in \
             (SELECT DISTINCT from_id FROM normal_link)').fetchall()
         res = []
         for (i,) in files:
deleted file mode 100644
--- a/python/mozbuild/mozbuild/analyze/hg.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import bisect
-import gzip
-import json
-import requests
-
-from datetime import datetime, timedelta
-from collections import Counter
-
-import mozpack.path as mozpath
-
-PUSHLOG_CHUNK_SIZE = 500
-
-URL = 'https://hg.mozilla.org/mozilla-central/json-pushes?'
-
-def unix_epoch(date):
-    return (date - datetime(1970,1,1)).total_seconds()
-
-def unix_from_date(n, today):
-    return unix_epoch(today - timedelta(days=n))
-
-def get_lastpid(session):
-    return session.get(URL+'&version=2').json()['lastpushid']
-
-def get_pushlog_chunk(session, start, end):
-    # returns pushes sorted by date
-    res = session.get(URL+'version=1&startID={0}&\
-        endID={1}&full=1'.format(start, end)).json()
-    return sorted(res.items(), key = lambda x: x[1]['date'])
-
-def collect_data(session, date):
-    if date < 1206031764: #first push
-        raise Exception ("No pushes exist before March 20, 2008.")
-    lastpushid = get_lastpid(session)
-    data = []
-    start_id = lastpushid - PUSHLOG_CHUNK_SIZE
-    end_id = lastpushid + 1
-    while True:
-        res = get_pushlog_chunk(session, start_id, end_id)
-        starting_date = res[0][1]['date'] # date of oldest push in chunk
-        dates = [x[1]['date'] for x in res]
-        if starting_date < date:
-            i = bisect.bisect_left(dates, date)
-            data.append(res[i:])
-            return data
-        else:
-            data.append(res)
-            end_id = start_id + 1
-            start_id = start_id - PUSHLOG_CHUNK_SIZE
-
-def get_data(epoch):
-    session = requests.Session()
-    data = collect_data(session, epoch)
-    return {k:v for sublist in data for (k,v) in sublist}
-
-class Pushlog(object):
-
-    def __init__(self, days):
-        info = get_data(unix_from_date(days, datetime.today()))
-        self.pushlog = info
-        self.pids = self.get_pids()
-        self.pushes = self.make_pushes()
-        self.files = [l for p in self.pushes for l in set(p.files)]
-        self.file_set = set(self.files)
-        self.file_count = Counter(self.files)
-
-    def make_pushes(self):
-        pids = self.pids
-        all_pushes = self.pushlog
-        return [Push(pid, all_pushes[str(pid)]) for pid in pids]
-
-    def get_pids(self):
-        keys = self.pushlog.keys()
-        keys.sort()
-        return keys
-
-class Push(object):
-
-    def __init__(self, pid, p_dict):
-        self.id = pid
-        self.date = p_dict['date']
-        self.files = [f for x in p_dict['changesets'] for f in x['files']]
-
-class Report(object):
-
-    def __init__(self, days, path=None, cost_dict=None):
-        obj = Pushlog(days)
-        self.file_set = obj.file_set
-        self.file_count = obj.file_count
-        self.name = str(days) + 'day_report'
-        self.cost_dict = self.get_cost_dict(path, cost_dict)
-
-    def get_cost_dict(self, path, cost_dict):
-        if path is not None:
-            with gzip.open(path) as file:
-                return json.loads(file.read())
-        else:
-            if cost_dict is not None:
-                return cost_dict
-            else:
-                raise Exception
-
-    def organize_data(self):
-        costs = self.cost_dict
-        counts = self.file_count
-        res = []
-        for f in self.file_set:
-            cost = costs.get(f)
-            count = counts.get(f)
-            if cost is not None:
-                res.append((f, cost, count, round(cost*count,3)))
-        return res
-
-    def get_sorted_report(self):
-        res = self.organize_data()
-        res.sort(key=(lambda x: x[3]), reverse=True)
-        return res
-
-    def cut(self, size, lst):
-        if len(lst) <= size:
-            return lst
-        else:
-            return lst[:size]
-
-    def generate_output(self, format, limit, dst):
-        import tablib
-        data = tablib.Dataset(headers=['FILE', 'TIME', 'CHANGES', 'TOTAL'])
-        res = self.get_sorted_report()
-        if limit is not None:
-            res = self.cut(limit, res)
-        for x in res: data.append(x)
-        if format == 'pretty':
-            print (data)
-        else:
-            file_name = self.name + '.' + format
-            content = None
-            data.export(format)
-            if format == 'csv':
-                content = data.csv
-            elif format == 'json':
-                content = data.json
-            else:
-                content = data.html
-            file_path = mozpath.join(dst, file_name)
-            with open(file_path, 'wb') as f:
-                f.write(content)
-            print ("Created report: %s" % file_path)
-
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2653,57 +2653,16 @@ class Repackage(MachCommandBase):
             print('Input file does not exist: %s' % input)
             return 1
 
         if not os.path.exists(os.path.join(self.topobjdir, 'config.status')):
             print('config.status not found.  Please run |mach configure| '
                   'prior to |mach repackage|.')
             return 1
 
-    @SubCommand('analyze', 'all',
-        description='Get a report of files changed within the last n days and their corresponding build cost.')
-    @CommandArgument('--days', '-d', type=int, default=14,
-        help='Number of days to include in the report.')
-    @CommandArgument('--format', default='pretty',
-        choices=['pretty', 'csv', 'json', 'html'],
-        help='Print or export data in the given format.')
-    @CommandArgument('--limit', type=int, default=None,
-        help='Get the top n most expensive files from the report.')
-    @CommandArgument('--path', help='Path to cost_dict.gz',
-        default=None)
-    def analyze_report(self, days, format, limit, path):
-        from mozbuild.analyze.hg import Report
-        self._activate_virtualenv()
-        try:
-            self.virtualenv_manager.install_pip_package('tablib==0.12.1')
-        except Exception:
-            print ('Could not install tablib via pip.')
-            return 1
-        if path is None:
-            # go find tup db and make a cost_dict
-            from mozbuild.analyze.graph import Graph
-            db_path = mozpath.join(self.topsrcdir, '.tup', 'db')
-            if os.path.isfile(db_path):
-                g = Graph(db_path)
-                r = Report(days, cost_dict=g.get_cost_dict())
-                g.close()
-                r.generate_output(format, limit, self.topobjdir)
-            else:
-                res = 'Please specify the location of cost_dict.gz with --path.'
-                print ('Could not find %s to make a cost dictionary.' % db_path, res, sep='\n')
-                return 1
-        else:
-            # path to cost_dict.gz was specified
-            if os.path.isfile(path):
-                r = Report(days, path)
-                r.generate_output(format, limit, self.topobjdir)
-            else:
-                res = 'Please specify the location of cost_dict.gz with --path.'
-                print ('Could not find cost_dict.gz at %s' % path, res, sep='\n')
-                return 1
         from mozbuild.repackaging.dmg import repackage_dmg
         repackage_dmg(input, output)
 
     @SubCommand('repackage', 'installer',
                 description='Repackage into a Windows installer exe')
     @CommandArgument('--tag', type=str, required=True,
         help='The .tag file used to build the installer')
     @CommandArgument('--setupexe', type=str, required=True,
@@ -2738,27 +2697,22 @@ class Repackage(MachCommandBase):
         help='Output filename')
     def repackage_mar(self, input, mar, output):
         from mozbuild.repackaging.mar import repackage_mar
         repackage_mar(self.topsrcdir, input, mar, output)
 
 @CommandProvider
 class Analyze(MachCommandBase):
     """ Get information about a file in the build graph """
-    @Command('analyze', category='misc',
-        description='Analyze the build graph.')
-    def analyze(self):
-        print("Usage: ./mach analyze [files|report] [args...]")
-
-    @SubCommand('analyze', 'files',
-        description='Get incremental build cost for file(s) from the tup database.')
+    @Command('summarize', category='misc',
+        description='Get incremental build cost for a file (or files) from the tup database.')
     @CommandArgument('--path', help='Path to tup db',
         default=None)
-    @CommandArgument('files', nargs='*', help='Files to analyze')
-    def analyze_files(self, path, files):
+    @CommandArgument('files', nargs='*', help='Files to summarize')
+    def summarize(self, path, files):
         from mozbuild.analyze.graph import Graph
         if path is None:
             path = mozpath.join(self.topsrcdir, '.tup', 'db')
         if os.path.isfile(path):
             g = Graph(path)
             g.file_summaries(files)
             g.close()
         else:
--- a/python/mozbuild/mozbuild/test/analyze/test_graph.py
+++ b/python/mozbuild/mozbuild/test/analyze/test_graph.py
@@ -22,17 +22,17 @@ CREATE_NORMAL_LINK = """CREATE TABLE nor
 
 NODE_DATA = [(1, 0 ,2, -1, '.'),
         (2, 100, 0, 1, 'Base64.cpp'),
         (3, 200, 0, 1, 'nsArray.cpp'),
         (4, 100, 0, 1, 'nsWildCard.h'),
         (5, -1, 1, 9426, 'CDD Unified_cpp_xpcom_io0.cpp'),
         (6, -1, 1, 5921, 'CXX Unified_cpp_xpcom_ds0.cpp'),
         (7, -1, 1, 11077, 'CXX /builds/worker/workspace/build/src/dom/\
-            plugins/base/nsNPAPIPlugin.cpp'),
+            plugins/base/snNPAPIPlugin.cpp'),
         (8, -1, 1, 7677, 'CXX Unified_cpp_xpcom_io1.cpp'),
         (9, -1, 1, 8672, 'CXX Unified_cpp_modules_libjar0.cpp'),
         (10, -1, 4, 1, 'Unified_cpp_xpcom_io0.o'),
         (11, -1, 4, 1, 'Unified_cpp_xpcom_dso.o'),
         (12, -1, 4, 1, 'nsNPAPIPlugin.o'),
         (13, -1, 4, 1, 'Unified_cpp_xpcom_io1.o'),
         (14, -1, 4, 1, 'Unified_cpp_modules_libjar0.o'),
         (15, -1, 1, 52975, 'LINK libxul.so'),
@@ -127,9 +127,9 @@ class TestGraph(unittest.TestCase):
         self.assertEqual(g.get_node(27).path, '')
         # node is a file
         self.assertEqual(g.get_node(2).path, 'xpcom/io/Base64.cpp')
         self.assertEqual(g.get_node(3).path, 'xpcom/ds/nsArray.cpp')
         self.assertEqual(g.get_node(4).path, 'xpcom/io/nsWildCard.h')
         self.assertEqual(g.get_node(28).path, 'dummy node')
 
 if __name__ == '__main__':
-  mozunit.main()
+  mozunit.main()
\ No newline at end of file