Bug 1372857 - allow xperf in debug mode to upload more data to blobber. r=rwood
authorJoel Maher <jmaher@mozilla.com>
Thu, 15 Jun 2017 09:44:09 -0400
changeset 415281 301cf943d0e01c61110a950e4f32c0cb421b7dc2
parent 415280 1556492464a3fa0993563452a33262d07c748634
child 415282 a8600c9c47ec7d9d8b2f765f07607c740dda8e3c
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1372857
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1372857 - allow xperf in debug mode to upload more data to blobber. r=rwood MozReview-Commit-ID: 55Kfcg3SBMO
testing/talos/talos/xtalos/etlparser.py
testing/talos/talos/xtalos/xperf_whitelist.json
--- a/testing/talos/talos/xtalos/etlparser.py
+++ b/testing/talos/talos/xtalos/etlparser.py
@@ -7,16 +7,17 @@
 import csv
 import re
 import os
 import sys
 import xtalos
 import subprocess
 import json
 import mozfile
+import shutil
 
 
 EVENTNAME_INDEX = 0
 PROCESS_INDEX = 2
 THREAD_ID_INDEX = 3
 DISKBYTES_COL = "Size"
 FNAME_COL = "FileName"
 IMAGEFUNC_COL = "Image!Function"
@@ -33,16 +34,24 @@ net_events = {
     "TcpDataTransferSend": "send",
     "UdpEndpointSendMessages": "send"
 }
 gThreads = {}
 gConnectionIDs = {}
 gHeaders = {}
 
 
+def uploadFile(filename):
+    mud = os.environ.get('MOZ_UPLOAD_DIR', None)
+    if mud:
+        print("uploading raw file %s via blobber" % filename)
+        mud_filename = os.path.join(mud, filename)
+        shutil.copyfile(filename, "%s.log" % mud_filename)
+
+
 def filterOutHeader(data):
     # -1 means we have not yet found the header
     # 0 means we are in the header
     # 1+ means that we are past the header
     state = -1
     for row in data:
 
         if not len(row):
@@ -287,60 +296,75 @@ def etlparser(xperf_path, etl_filename, 
             fileSummary(row, stage, files)
             trackThreadFileIO(row, io, stage)
         elif event.endswith("Event/Classic") and \
                 row[THREAD_ID_INDEX] in gThreads:
             stage = updateStage(row, stage)
         elif event.startswith("Microsoft-Windows-TCPIP"):
             trackThreadNetIO(row, io, stage)
 
-    # remove the csv file
-    if not debug:
+    if debug:
+        uploadFile(csvname)
+    else:
         mozfile.remove(csvname)
 
     output = "thread, stage, counter, value\n"
     for cntr in sorted(io.iterkeys()):
         output += "%s, %s\n" % (", ".join(cntr), str(io[cntr]))
     if outputFile:
         fname = "%s_thread_stats%s" % os.path.splitext(outputFile)
         with open(fname, "w") as f:
             f.write(output)
+
+        if debug:
+            uploadFile(fname)
     else:
         print(output)
 
     whitelist = loadWhitelist(whitelist_file)
 
     header = ("filename, tid, stage, readcount, readbytes, writecount,"
               " writebytes")
     outFile.write(header + "\n")
 
     # Filter out stages, threads, and whitelisted files that we're not
     # interested in
     filekeys = filter(lambda x: (all_stages or x[2] == stages[0]) and
                                 (all_threads or x[1].endswith("(main)")) and
                                 (all_stages and x[2] != stages[0] or
                                  not checkWhitelist(x[0], whitelist)),
                       files.iterkeys())
+    if debug:
+        # in debug, we want stages = [startup+normal] and all threads, not just (main)
+        # we will use this data to upload fileIO info to blobber only for debug mode
+        outputData = filter(lambda x: (all_stages or x[2] in [stages[0], stages[1]]) and
+                                      (all_stages and x[2] not in [stages[0], stages[1]] or
+                                       not checkWhitelist(x[0], whitelist)),
+                            files.iterkeys())
+    else:
+        outputData = filekeys
 
     # output data
-    for row in filekeys:
+    for row in outputData:
         output = "%s, %s, %s, %s, %s, %s, %s\n" % (
             row[0],
             row[1],
             row[2],
             files[row]['DiskReadCount'],
             files[row]['DiskReadBytes'],
             files[row]['DiskWriteCount'],
             files[row]['DiskWriteBytes']
         )
         outFile.write(output)
 
     if outputFile:
         # close the file handle
         outFile.close()
+        if debug:
+            uploadFile(outputFile)
 
     # We still like to have the outputfile to record the raw data, now
     # filter out acceptable files/ranges
     filename = None
     wl_temp = {}
     dirname = os.path.dirname(__file__)
     if os.path.exists(os.path.join(dirname, 'xperf_whitelist.json')):
         filename = os.path.join(dirname, 'xperf_whitelist.json')
@@ -450,22 +474,18 @@ def etlparser(xperf_path, etl_filename, 
             print("TEST-UNEXPECTED-FAIL : xperf: %s" % error)
 
         # We detect if browser_failures.txt exists to exit and turn the job
         # orange
         if error_filename:
             with open(error_filename, 'w') as errorFile:
                 errorFile.write('\n'.join(errors))
 
-# TODO: commented this out from bug 1205005, we have a memory error in blobber
-# upload
-#        mud = os.environ.get('MOZ_UPLOAD_DIR', None)
-#        if mud:
-#            mud_filename = os.path.join(mud, etl_filename)
-#            os.rename(etl_filename, mud_filename)
+        if debug:
+            uploadFile(etl_filename)
 
 
 def etlparser_from_config(config_file, **kwargs):
     """start from a YAML config file"""
 
     # option defaults
     args = {'xperf_path': 'xperf.exe',
             'etl_filename': 'test.etl',
--- a/testing/talos/talos/xtalos/xperf_whitelist.json
+++ b/testing/talos/talos/xtalos/xperf_whitelist.json
@@ -20,25 +20,41 @@
  "{firefox}\\browser\\features\\activity-stream@mozilla.org.xpi": {"mincount": 0, "maxcount": 100, "minbytes": 0, "maxbytes": 10000000},
  "{firefox}\\browser\\features\\screenshots@mozilla.org.xpi": {"mincount": 0, "maxcount": 100, "minbytes": 0, "maxbytes": 10000000},
  "{talos}\\talos\\tests\\tp5n\\tp5n.manifest": {"mincount": 0, "maxcount": 8, "minbytes": 0, "maxbytes": 32786},
  "{talos}\\talos\\tests\\tp5n\\tp5n.manifest.develop": {"mincount": 0, "maxcount": 8, "minbytes": 0, "maxbytes": 32786},
  "{profile}\\localstore.rdf": {"mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192},
  "{firefox}\\dependentlibs.list": {"mincount": 4, "maxcount": 4, "minbytes": 16384, "maxbytes": 16384},
  "{profile}\\content-prefs.sqlite": {"mincount": 6, "maxcount": 6, "minbytes": 65768, "maxbytes": 65768},
  "{profile}\\extensions.ini": {"mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192},
+ "{profile}\\containers.json": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 512},
+ "{profile}\\extensions.json": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 512},
+ "{profile}\\times.json": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 512},
+ "{profile}\\sessionstore-backups\\recovery.js": {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 5602},
+ "{profile}\\sessioncheckpoints.json.tmp": {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 512},
+ "{profile}\\sessioncheckpoints.json": {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 512},
  "{profile}\\extensions\\pageloader@mozilla.org\\chrome.manifest": {"mincount": 2, "maxcount": 2, "minbytes": 600, "maxbytes": 600},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\mozillafilelogger.js": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 125200},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\memory.js": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 127000},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\profiler.js": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 127000},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\pageloader.js": {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 127000},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\pageloader.xul": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 127000},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\report.js": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 127000},
+ "{profile}\\extensions\\pageloader@mozilla.org\\chrome\\quit.js": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 127000},
  "{profile}\\extensions\\talos-powers@mozilla.org\\chrome.manifest": {"mincount": 2, "maxcount": 2, "minbytes": 600, "maxbytes": 600},
  "{profile}\\extensions\\talos-powers@mozilla.org\\chrome\\talos-powers-content.js": {"mincount": 2, "maxcount": 2, "minbytes": 2000, "maxbytes": 2000},
  "{firefox}\\profiles.ini": {"mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192},
+ "{firefox}\\firefox.exe": {"mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192},
+ "{firefox}\\gmp-clearkey\\0.1\\manifest.json": {"mincount": 2, "maxcount": 2, "minbytes": 512, "maxbytes": 512},
  "c:\\program files\\nvidia corporation\\3d vision\\nvstereoapii.dll": {"mincount": 0, "maxcount": 4, "minbytes": 2, "maxbytes": 33792},
  "c:\\programdata\\nvidia corporation\\drs\\nvdrssel.bin": {"mincount": 2, "maxcount": 2, "minbytes": 2, "maxbytes": 2},
  "c:\\programdata\\nvidia corporation\\drs\\nvapptimestamps": {"mincount": 22, "maxcount": 22, "minbytes": 704, "maxbytes": 704},
  "{profile}\\places.sqlite": {"mincount": 8, "maxcount": 8, "minbytes": 196808, "maxbytes": 196808},
  "c:\\windows\\system32\\dwrite.dll": {"mincount": 4, "maxcount": 4, "minbytes": 16384, "maxbytes": 90112},
+ "c:\\windows\\system32\\audioses.dll": {"mincount": 4, "maxcount": 4, "minbytes": 16384, "maxbytes": 32768},
  "c:\\users\\desktop.ini": {"mincount": 2, "maxcount": 2, "minbytes": 352, "maxbytes": 352},
  "{desktop}\\desktop.ini": {"mincount": 6, "maxcount": 6, "minbytes": 1692, "maxbytes": 1692},
  "c:\\windows\\fonts\\staticcache.dat": {"mincount": 2, "maxcount": 2, "minbytes": 120, "maxbytes": 120},
  "c:\\windows\\system32\\spool\\drivers\\color\\srgb color space profile.icm": {"mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192},
  "{profile}\\prefs.js": {"mincount": 4, "maxcount": 4, "minbytes": 19612, "maxbytes": 20136},
  "{profile}\\places.sqlite-shm": {"mincount": 2, "maxcount": 2, "minbytes": 65536, "maxbytes": 65536},
  "{profile}\\places.sqlite-wal": {"mincount": 0, "maxcount": 32, "minbytes": 0, "maxbytes": 131072},
  "{profile}\\permissions.sqlite": {"mincount": 14, "maxcount": 14, "minbytes": 4424, "maxbytes": 4424},
@@ -52,9 +68,14 @@
  "{profile}\\pluginreg.dat": {"mincount": 2, "maxcount": 2, "minbytes": 1892, "maxbytes": 1892},
  "{firefox}\\defaults\\pref\\channel-prefs.js": {"mincount": 4, "maxcount": 4, "minbytes": 1432, "maxbytes": 1432},
  "{profile}\\cert8.db": {"mincount": 0, "maxcount": 12, "minbytes": 0, "maxbytes": 65536},
  "{profile}\\key3.db": {"mincount": 0, "maxcount": 12, "minbytes": 0, "maxbytes": 32768},
  "{appdata}\\local\\temp": {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 278528},
  "{profile}\\favicons.sqlite": {"mincount": 8, "maxcount": 12, "minbytes": 196808, "maxbytes": 393216},
  "{profile}\\favicons.sqlite-shm": {"mincount": 0, "maxcount": 4, "minbytes": 65536, "maxbytes": 131072},
  "{profile}\\favicons.sqlite-wal": {"mincount": 0, "maxcount": 32, "minbytes": 0, "maxbytes": 131072},
+ "c:\\windows\\system32\\msctf.dll":  {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 65536},
+ "{profile}\\handlers.json":  {"mincount": 0, "maxcount": 4, "minbytes": 0, "maxbytes": 1388},
+ "{profile}\\startupcache":  {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 8192},
+ "{profile}\\sessionstore.js":  {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 4096},
+ "{profile}\\addonstartup.json.lz4":  {"mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 8192}
 }