author | Deepa <deepsrijit1105@gmail.com> |
Wed, 15 Feb 2017 01:20:56 +0530 | |
changeset 343194 | 868a4dc22d7a40d78a620c2939be59392b7822be |
parent 343193 | 35d16d04d67d7fb3739b2dbf6fc38b1c1846bd27 |
child 343195 | 9228f08cacdee37c5e741ff1abc844714a55257b |
push id | 31372 |
push user | cbook@mozilla.com |
push date | Thu, 16 Feb 2017 12:16:10 +0000 |
treeherder | mozilla-central@2737f66ad6ac [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahal |
bugs | 1280572 |
milestone | 54.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/layout/tools/reftest/mach_commands.py +++ b/layout/tools/reftest/mach_commands.py @@ -79,18 +79,18 @@ class ReftestRunner(MozbuildObject): args.extraProfileFiles.append(os.path.join(self.topobjdir, "dist", "plugins")) args.symbolsPath = os.path.join(self.topobjdir, "crashreporter-symbols") if not args.tests: args.tests = [os.path.join(*default_manifest[args.suite])] if args.suite == "jstestbrowser": args.extraProfileFiles.append(os.path.join(self.topobjdir, "dist", - "test-stage", "jsreftest", - "tests", "user.js")) + "test-stage", "jsreftest", + "tests", "user.js")) self.log_manager.enable_unstructured() try: rv = runreftest.run_test_harness(parser, args) finally: self.log_manager.disable_unstructured() return rv
--- a/layout/tools/reftest/output.py +++ b/layout/tools/reftest/output.py @@ -1,14 +1,13 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import json -import os import threading from mozlog.formatters import TbplFormatter from mozrunner.utils import get_stack_fixer_function class ReftestFormatter(TbplFormatter): """ @@ -44,28 +43,28 @@ class ReftestFormatter(TbplFormatter): status_msg += "UNEXPECTED-%s" % status else: if status != "PASS": status_msg += "KNOWN-" status_msg += status if extra.get('status_msg') == 'Random': status_msg += "(EXPECTED RANDOM)" - output_text = "%s | %s | %s" % (status_msg, test, data.get("message", "")) - if "reftest_screenshots" in extra: screenshots = extra["reftest_screenshots"] + image_1 = screenshots[0]["screenshot"] + image_2 = screenshots[2]["screenshot"] + if len(screenshots) == 3: output_text += ("\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n" - "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (screenshots[0]["screenshot"], - screenshots[2]["screenshot"]) + "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % ( + image_1, image_2) elif len(screenshots) == 1: - output_text += "\nREFTEST IMAGE: data:image/png;base64,%(image1)s" % screenshots[0]["screenshot"] - + output_text += "\nREFTEST IMAGE: data:image/png;base64,%(image1)s" % image_1 output_text += "\nREFTEST TEST-END | %s" % test return "%s\n" % output_text def process_output(self, data): return "%s\n" % data["data"] def suite_end(self, data):
--- a/layout/tools/reftest/reftest/__init__.py +++ b/layout/tools/reftest/reftest/__init__.py @@ -29,16 +29,17 @@ FAILURE_TYPES = ( 'asserts-if', ) PREF_ITEMS = ( 'pref', 'test-pref', 'ref-pref', ) + class ReftestManifest(object): """Represents a parsed reftest manifest. We currently only capture file information because that is the only thing tools require. """ def __init__(self, finder=None): self.path = None
--- a/layout/tools/reftest/reftestcommandline.py +++ b/layout/tools/reftest/reftestcommandline.py @@ -1,14 +1,13 @@ import argparse import os -import sys from collections import OrderedDict from urlparse import urlparse - +import mozinfo import mozlog here = os.path.abspath(os.path.dirname(__file__)) class ReftestArgumentsParser(argparse.ArgumentParser): def __init__(self, **kwargs): super(ReftestArgumentsParser, self).__init__(**kwargs) @@ -30,17 +29,18 @@ class ReftestArgumentsParser(argparse.Ar default=None, help="absolute path to directory containing XRE (probably xulrunner)") self.add_argument("--symbols-path", action="store", type=str, dest="symbolsPath", default=None, - help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols") + help="absolute path to directory containing breakpad symbols, " + "or the URL of a zip file containing symbols") self.add_argument("--debugger", action="store", dest="debugger", help="use the given debugger to launch the application") self.add_argument("--debugger-args", action="store", @@ -67,17 +67,18 @@ class ReftestArgumentsParser(argparse.Ar default=[], help="copy specified files/dirs to testing profile") self.add_argument("--timeout", action="store", dest="timeout", type=int, default=5 * 60, # 5 minutes per bug 479518 - help="reftest will timeout in specified number of seconds. [default %(default)s].") + help="reftest will timeout in specified number of seconds. " + "[default %(default)s].") self.add_argument("--leak-threshold", action="store", type=int, dest="defaultLeakThreshold", default=0, help="fail if the number of bytes leaked in default " "processes through refcounted objects (or bytes " @@ -114,17 +115,18 @@ class ReftestArgumentsParser(argparse.Ar action="store_true", default=False, help="skip tests marked as slow when running") self.add_argument("--ignore-window-size", dest="ignoreWindowSize", action="store_true", default=False, - help="ignore the window size, which may cause spurious failures and passes") + help="ignore the window size, which may cause spurious " + "failures and passes") self.add_argument("--install-extension", action="append", dest="extensionsToInstall", default=[], help="install the specified extension in the testing profile. " "The extension file's name should be <id>.xpi where <id> is " "the extension's id as indicated in its install.rdf. " @@ -213,20 +215,20 @@ class ReftestArgumentsParser(argparse.Ar help="Path to the special powers extension") self.add_argument("--suite", choices=["reftest", "crashtest", "jstestbrowser"], default=None, help=argparse.SUPPRESS) self.add_argument("--cleanup-crashes", - action = "store_true", - dest = "cleanupCrashes", - default = False, - help = "Delete pending crash reports before running tests.") + action="store_true", + dest="cleanupCrashes", + default=False, + help="Delete pending crash reports before running tests.") self.add_argument("tests", metavar="TEST_PATH", nargs="*", help="Path to test file, manifest file, or directory containing tests") mozlog.commandline.add_logging_group(self) @@ -255,17 +257,18 @@ class ReftestArgumentsParser(argparse.Ar options.suite = suite return self.error("Failed to determine test suite; supply --suite to set this explicitly") def validate(self, options, reftest): if not options.tests: # Can't just set this in the argument parser because mach will set a default - self.error("Must supply at least one path to a manifest file, test directory, or test file to run.") + self.error("Must supply at least one path to a manifest file, " + "test directory, or test file to run.") if options.suite is None: self.set_default_suite(options) if options.totalChunks is not None and options.thisChunk is None: self.error( "thisChunk must be specified when totalChunks is specified") @@ -370,17 +373,19 @@ class RemoteArgumentsParser(ReftestArgum xrePath="", utilityPath="", localLogName=None) self.add_argument("--remote-app-path", action="store", type=str, dest="remoteAppPath", - help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.") + help="Path to remote executable relative to device root using only " + "forward slashes. Either this or app must be specified, " + "but not both.") self.add_argument("--adbpath", action="store", type=str, dest="adb_path", default="adb", help="path to adb") @@ -403,17 +408,18 @@ class RemoteArgumentsParser(ReftestArgum dest="devicePort", help="port of remote device to test") self.add_argument("--remote-product-name", action="store", type=str, dest="remoteProductName", default="fennec", - help="Name of product to test - either fennec or firefox, defaults to fennec") + help="Name of product to test - either fennec or firefox, " + "defaults to fennec") self.add_argument("--remote-webserver", action="store", type=str, dest="remoteWebServer", help="IP Address of the webserver hosting the reftest content") self.add_argument("--http-port", @@ -428,37 +434,40 @@ class RemoteArgumentsParser(ReftestArgum dest="sslPort", help="Port for https traffic to the web server") self.add_argument("--remote-logfile", action="store", type=str, dest="remoteLogFile", default="reftest.log", - help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.") + help="Name of log file on the device relative to device root. " + "PLEASE USE ONLY A FILENAME.") self.add_argument("--pidfile", action="store", type=str, dest="pidFile", default="", help="name of the pidfile to generate") self.add_argument("--dm_trans", action="store", type=str, dest="dm_trans", default="sut", - help="the transport to use to communicate with device: [adb|sut]; default=sut") + help="the transport to use to communicate with device: " + "[adb|sut]; default=sut") self.add_argument("--remoteTestRoot", action="store", type=str, dest="remoteTestRoot", - help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)") + help="remote directory to use as test root " + "(eg. /mnt/sdcard/tests or /data/local/tests)") self.add_argument("--httpd-path", action="store", type=str, dest="httpdPath", help="path to the httpd.js file") self.add_argument("--no-device-info", @@ -475,17 +484,18 @@ class RemoteArgumentsParser(ReftestArgum options.remoteProfile = options.remoteTestRoot + "/profile" if options.remoteWebServer is None: options.remoteWebServer = self.get_ip() # Verify that our remotewebserver is set properly if options.remoteWebServer == '127.0.0.1': self.error("ERROR: Either you specified the loopback for the remote webserver or ", - "your local IP cannot be detected. Please provide the local ip in --remote-webserver") + "your local IP cannot be detected. " + "Please provide the local ip in --remote-webserver") if not options.httpPort: options.httpPort = automation.DEFAULT_HTTP_PORT if not options.sslPort: options.sslPort = automation.DEFAULT_SSL_PORT # One of remoteAppPath (relative path to application) or the app (executable) must be @@ -534,15 +544,16 @@ class RemoteArgumentsParser(ReftestArgum options.httpdPath = os.path.join(options.utilityPath, "components") if not options.ignoreWindowSize: parts = automation._devicemanager.getInfo( 'screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): - self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % ( - width, height)) + self.error("ERROR: Invalid screen resolution %sx%s, " + "please adjust to 1366x1050 or higher" % ( + width, height)) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False return options
--- a/layout/tools/reftest/remotereftest.py +++ b/layout/tools/reftest/remotereftest.py @@ -30,18 +30,18 @@ class RemoteReftestResolver(ReftestResol elif os.path.exists(os.path.abspath(path)): rv = os.path.abspath(path) else: print >> sys.stderr, "Could not find manifest %s" % script_abs_path sys.exit(1) return os.path.normpath(rv) def manifestURL(self, options, path): - # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot - # It's possible for this url to have a leading "..", but reftest.js will fix that up + # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' + # webroot. It's possible for this url to have a leading "..", but reftest.js will fix that relPath = os.path.relpath(path, SCRIPT_DIRECTORY) return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath) class ReftestServer: """ Web server used to serve Reftests, for closer fidelity to the real web. It is virtually identical to the server used in mochitest and will only be used for running reftests remotely. @@ -53,44 +53,47 @@ class ReftestServer: self._utilityPath = options.utilityPath self._xrePath = options.xrePath self._profileDir = options.serverProfilePath self.webServer = options.remoteWebServer self.httpPort = options.httpPort self.scriptDir = scriptDir self.pidFile = options.pidFile self._httpdPath = os.path.abspath(options.httpdPath) - self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort } + self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { + "server": self.webServer, "port": self.httpPort} def start(self): "Run the Refest server, returning the process ID of the server." - env = self.automation.environment(xrePath = self._xrePath) + env = self.automation.environment(xrePath=self._xrePath) env["XPCOM_DEBUG_BREAK"] = "warn" if self.automation.IS_WIN32: env["PATH"] = env["PATH"] + ";" + self._xrePath args = ["-g", self._xrePath, "-v", "170", "-f", os.path.join(self._httpdPath, "httpd.js"), - "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR ='%(server)s';" % - {"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer }, + "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = " + "'%(port)s'; const _SERVER_ADDR ='%(server)s';" % { + "profile": self._profileDir.replace('\\', '\\\\'), "port": self.httpPort, + "server": self.webServer}, "-f", os.path.join(self.scriptDir, "server.js")] xpcshell = os.path.join(self._utilityPath, "xpcshell" + self.automation.BIN_SUFFIX) if not os.access(xpcshell, os.F_OK): raise Exception('xpcshell not found at %s' % xpcshell) if self.automation.elf_arm(xpcshell): raise Exception('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell) - self._process = self.automation.Process([xpcshell] + args, env = env) + self._process = self.automation.Process([xpcshell] + args, env=env) pid = self._process.pid if pid < 0: print "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server." return 2 self.automation.log.info("INFO | remotereftests.py | Server pid: %d", pid) if (self.pidFile != ""): f = open(self.pidFile + ".xpcshell.pid", 'w') @@ -103,33 +106,35 @@ class ReftestServer: aliveFile = os.path.join(self._profileDir, "server_alive.txt") i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: - print "TEST-UNEXPECTED-FAIL | remotereftests.py | Timed out while waiting for server startup." + print ("TEST-UNEXPECTED-FAIL | remotereftests.py | " + "Timed out while waiting for server startup.") self.stop() return 1 def stop(self): if hasattr(self, '_process'): try: c = urllib2.urlopen(self.shutdownURL) c.read() c.close() rtncode = self._process.poll() - if (rtncode == None): + if (rtncode is None): self._process.terminate() except: self._process.kill() + class RemoteReftest(RefTest): use_marionette = False remoteApp = '' resolver_cls = RemoteReftestResolver def __init__(self, automation, devicemanager, options, scriptDir): RefTest.__init__(self) self.automation = automation @@ -150,17 +155,17 @@ class RemoteReftest(RefTest): self._populate_logger(options) outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. outputHandler.write = outputHandler.__call__ self.automation._processArgs['messageLogger'] = outputHandler - def findPath(self, paths, filename = None): + def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None @@ -169,42 +174,45 @@ class RemoteReftest(RefTest): remoteXrePath = options.xrePath remoteUtilityPath = options.utilityPath localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform - if (hostos == 'mac' or hostos == 'darwin'): + if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True - paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, os.path.join('..', self.automation._product)] + paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, + os.path.join('..', self.automation._product)] options.xrePath = self.findPath(paths) - if options.xrePath == None: - print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name) + if options.xrePath is None: + print ("ERROR: unable to find xulrunner path for %s, " + "please specify with --xre-path" % (os.name)) return 1 paths.append("bin") paths.append(os.path.join("..", "bin")) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if (options.utilityPath): paths.insert(0, options.utilityPath) options.utilityPath = self.findPath(paths, xpcshell) - if options.utilityPath == None: - print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name) + if options.utilityPath is None: + print ("ERROR: unable to find utility path for %s, " + "please specify with --utility-path" % (os.name)) return 1 options.serverProfilePath = tempfile.mkdtemp() self.server = ReftestServer(localAutomation, options, self.scriptDir) retVal = self.server.start() if retVal: return retVal retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) @@ -313,22 +321,24 @@ class RemoteReftest(RefTest): self._devicemanager.removeDir(self.remoteProfile) self._devicemanager.removeDir(self.remoteTestRoot) RefTest.cleanup(self, profileDir) if (self.pidFile != ""): try: os.remove(self.pidFile) os.remove(self.pidFile + ".xpcshell.pid") except: - print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile + print ("Warning: cleaning up pidfile '%s' was unsuccessful " + "from the test harness" % self.pidFile) def run_test_harness(parser, options): - if options.dm_trans == 'sut' and options.deviceIP == None: - print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option" + if options.dm_trans == 'sut' and options.deviceIP is None: + print ("Error: If --dm_trans = sut, you must provide a device IP to " + "connect to via the --deviceIP option") return 1 dm_args = { 'deviceRoot': options.remoteTestRoot, 'host': options.deviceIP, 'port': options.devicePort, } @@ -338,17 +348,18 @@ def run_test_harness(parser, options): if not dm_args['host']: dm_args['deviceSerial'] = options.deviceSerial dm_cls = mozdevice.DroidADB try: dm = dm_cls(**dm_args) except mozdevice.DMError: traceback.print_exc() - print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state." + print ("Automation Error: exception while initializing devicemanager. " + "Most likely the device is not in a testable state.") return 1 automation = RemoteAutomation(None) automation.setDeviceManager(dm) if options.remoteProductName: automation.setProduct(options.remoteProductName) @@ -382,18 +393,19 @@ def run_test_harness(parser, options): procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.printDeviceInfo: reftest.printDeviceInfo() -#an example manifest name to use on the cli -# manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" +# an example manifest name to use on the cli +# manifest = "http://" + options.remoteWebServer + +# "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: dm.recordLogcat() retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1
--- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -128,16 +128,17 @@ class ReftestThread(threading.Thread): haveSuppressedSummaryLine = True break if haveSuppressedSummaryLine: continue if summaryHeadRegex.search(line) is None: yield line + class ReftestResolver(object): def defaultManifest(self, suite): return {"reftest": "reftest.list", "crashtest": "crashtests.list", "jstestbrowser": "jstests.list"}[suite] def directoryManifest(self, suite, path): return os.path.join(path, self.defaultManifest(suite)) @@ -310,17 +311,18 @@ class RefTest(object): if not self.use_marionette: addons.append(options.reftestExtensionPath) if options.specialPowersExtensionPath is not None: if not self.use_marionette: addons.append(options.specialPowersExtensionPath) # SpecialPowers requires insecure automation-only features that we # put behind a pref. - prefs['security.turn_off_all_security_so_that_viruses_can_take_over_this_computer'] = True + prefs['security.turn_off_all_security_so_that_viruses' + '_can_take_over_this_computer'] = True for pref in prefs: prefs[pref] = mozprofile.Preferences.cast(prefs[pref]) # Install distributed extensions, if application has any. distExtDir = os.path.join(options.app[:options.app.rfind(os.sep)], "distribution", "extensions") if os.path.isdir(distExtDir): @@ -506,18 +508,19 @@ class RefTest(object): print 'REFTEST INFO | ' + text + ': ' + str(summaryObj['total']) + ' (' + details + ')' return int(any(t.retcode != 0 for t in threads)) def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo): """handle process output timeout""" # TODO: bug 913975 : _processOutput should call self.processOutputLine # one more time one timeout (I think) - self.log.error("%s | application timed out after %d seconds with no output" % (self.lastTestSeen, int(timeout))) - self.log.error("Force-terminating active process(es)."); + self.log.error("%s | application timed out after %d seconds with no output" % ( + self.lastTestSeen, int(timeout))) + self.log.error("Force-terminating active process(es).") self.killAndGetStack( proc, utilityPath, debuggerInfo, dump_screen=not debuggerInfo) def dumpScreen(self, utilityPath): if self.haveDumpedScreen: self.log.info("Not taking screenshot here: see the one that was previously logged") return self.haveDumpedScreen = True @@ -677,18 +680,17 @@ class RefTest(object): timeout=options.timeout + 30.0, symbolsPath=options.symbolsPath, options=options, debuggerInfo=debuggerInfo) self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s')) mozleak.process_leak_log(self.leakLogFile, leak_thresholds=options.leakThresholds, stack_fixer=get_stack_fixer_function(options.utilityPath, - options.symbolsPath), - ) + options.symbolsPath)) finally: self.cleanup(profileDir) return status def copyExtraFilesToProfile(self, options, profile): "Copy extra files or dirs specified on the command line to the testing profile." profileDir = profile.profile if not os.path.exists(os.path.join(profileDir, "hyphenation")):
--- a/tools/lint/flake8.lint +++ b/tools/lint/flake8.lint @@ -171,16 +171,17 @@ def lint(files, **lintargs): return results LINTER = { 'name': "flake8", 'description': "Python linter", 'include': [ + 'layout/tools/reftest', 'python/mozlint', 'security/manager', 'taskcluster', 'testing/firefox-ui', 'testing/marionette/client', 'testing/marionette/harness', 'testing/marionette/puppeteer', 'testing/mozbase',