author Nathan Froyd <>
Fri, 18 Apr 2014 11:03:00 -0400
changeset 171344 8cbc1c4cb9608c0b1cfb3e3d9931064965552515
parent 155390 ded6ef58e8a3530cfde0b443653b9240a9402d53
child 171521 e866dd05cb12478855470f6173b51090440ec032
permissions -rw-r--r--
Bug 997820 - Disable telemetry in tests. r=ted, a=test-only

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

Runs the reftest test harness.

import re, sys, shutil, os, os.path
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
sys.path.insert(0, SCRIPT_DIRECTORY)

from automation import Automation
from automationutils import *
from optparse import OptionParser
from tempfile import mkdtemp

import mozprofile

class RefTest(object):

  oldcwd = os.getcwd()

  def __init__(self, automation):
    self.automation = automation

  def getFullPath(self, path):
    "Get an absolute path relative to self.oldcwd."
    return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))

  def getManifestPath(self, path):
    "Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"
    path = self.getFullPath(path)
    if os.path.isdir(path):
      defaultManifestPath = os.path.join(path, 'reftest.list')
      if os.path.exists(defaultManifestPath):
        path = defaultManifestPath
        defaultManifestPath = os.path.join(path, 'crashtests.list')
        if os.path.exists(defaultManifestPath):
          path = defaultManifestPath
    return path

  def makeJSString(self, s):
    return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)

  def createReftestProfile(self, options, manifest, server='localhost',
      Sets up a profile for reftest.
      'manifest' is the path to the reftest.list file we want to test with.  This is used in
      the remote subclass in so we can write it to a preference for the 
      bootstrap extension.

    locations = mozprofile.permissions.ServerLocations()
    locations.add_host(server, port=0)
    locations.add_host('<file>', port=0)

    # Set preferences for communication between our command line arguments
    # and the reftest harness.  Preferences that are required for reftest
    # to work should instead be set in reftest-cmdline.js .
    prefs = {}
    prefs['reftest.timeout'] = options.timeout * 1000
    if options.totalChunks:
      prefs['reftest.totalChunks'] = options.totalChunks
    if options.thisChunk:
      prefs['reftest.thisChunk'] = options.thisChunk
    if options.logFile:
      prefs['reftest.logFile'] = options.logFile
    if options.ignoreWindowSize:
      prefs['reftest.ignoreWindowSize'] = True
    if options.filter:
      prefs['reftest.filter'] = options.filter
    prefs['reftest.focusFilterMode'] = options.focusFilterMode

    # Ensure that telemetry is disabled, so we don't connect to the telemetry
    # server in the middle of the tests.
    prefs['toolkit.telemetry.enabled'] = False

    for v in options.extraPrefs:
      thispref = v.split('=')
      if len(thispref) < 2:
        print "Error: syntax error in --setpref=" + v
      prefs[thispref[0]] = mozprofile.Preferences.cast(thispref[1].strip())

    # install the reftest extension bits into the profile
    addons = []
    addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))

    # I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with
    # release engineering and landing on multiple branches at once.
    if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):
      addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))

    # Install distributed extensions, if application has any.
    distExtDir = os.path.join([ :], "distribution", "extensions")
    if os.path.isdir(distExtDir):
      for f in os.listdir(distExtDir):
        addons.append(os.path.join(distExtDir, f))

    # Install custom extensions.
    for f in options.extensionsToInstall:

    profile = mozprofile.profile.Profile(
    self.copyExtraFilesToProfile(options, profile)
    return profile

  def buildBrowserEnv(self, options, profileDir):
    browserEnv = self.automation.environment(xrePath = options.xrePath)
    browserEnv["XPCOM_DEBUG_BREAK"] = "stack"

    for v in options.environment:
      ix = v.find("=")
      if ix <= 0:
        print "Error: syntax error in --setenv=" + v
        return None
      browserEnv[v[:ix]] = v[ix + 1:]    

    # Enable leaks detection to its own log file.
    self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
    browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
    return browserEnv

  def cleanup(self, profileDir):
    if profileDir:
      shutil.rmtree(profileDir, True)

  def runTests(self, testPath, options, cmdlineArgs = None):
    debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs,

    profileDir = None
      reftestlist = self.getManifestPath(testPath)
      if cmdlineArgs == None:
        cmdlineArgs = ['-reftest', reftestlist]
      profile = self.createReftestProfile(options, reftestlist)
      profileDir = profile.profile # name makes more sense

      # browser environment
      browserEnv = self.buildBrowserEnv(options, profileDir)"REFTEST INFO | | Running tests: start.\n")
      status = self.automation.runApp(None, browserEnv,, profileDir,
                                 utilityPath = options.utilityPath,
                                 # give the JS harness 30 seconds to deal
                                 # with its own timeouts
                                 timeout=options.timeout + 30.0)
      processLeakLog(self.leakLogFile, options.leakThreshold)"\nREFTEST INFO | | Running tests: end.")
    return status

  def copyExtraFilesToProfile(self, options, profile):
    "Copy extra files or dirs specified on the command line to the testing profile."
    profileDir = profile.profile
    for f in options.extraProfileFiles:
      abspath = self.getFullPath(f)
      if os.path.isfile(abspath):
        if os.path.basename(abspath) == 'user.js':
          extra_prefs = mozprofile.Preferences.read_prefs(abspath)
          shutil.copy2(abspath, profileDir)
      elif os.path.isdir(abspath):
        dest = os.path.join(profileDir, os.path.basename(abspath))
        shutil.copytree(abspath, dest)
        self.automation.log.warning("WARNING | | Failed to copy %s to profile", abspath)

class ReftestOptions(OptionParser):

  def __init__(self, automation):
    self._automation = automation
    defaults = {}

    # we want to pass down everything from automation.__all__
                            [getattr(self._automation, x) for x in self._automation.__all__])))
                    action = "store", type = "string", dest = "app",
                    default = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP),
                    help = "absolute path to application, overriding default")
                    action = "append", dest = "extraProfileFiles",
                    default = [],
                    help = "copy specified files/dirs to testing profile")
                    action = "store", dest = "timeout", type = "int", 
                    default = 5 * 60, # 5 minutes per bug 479518
                    help = "reftest will timeout in specified number of seconds. [default %default s].")
                    action = "store", type = "int", dest = "leakThreshold",
                    default = 0,
                    help = "fail if the number of bytes leaked through "
                           "refcounted objects (or bytes in classes with "
                           "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
                           "than the given number")
                    action = "store", type = "string", dest = "utilityPath",
                    default = self._automation.DIST_BIN,
                    help = "absolute path to directory containing utility "
                           "programs (xpcshell, ssltunnel, certutil)")
    defaults["utilityPath"] = self._automation.DIST_BIN

                    type = "int", dest = "totalChunks",
                    help = "how many chunks to split the tests up into")
    defaults["totalChunks"] = None

                    type = "int", dest = "thisChunk",
                    help = "which chunk to run between 1 and --total-chunks")
    defaults["thisChunk"] = None

                    action = "store", type = "string", dest = "logFile",
                    default = None,
                    help = "file to log output to in addition to stdout")
    defaults["logFile"] = None
                    dest = "skipSlowTests", action = "store_true",
                    help = "skip tests marked as slow when running")
    defaults["skipSlowTests"] = False

                    dest = "ignoreWindowSize", action = "store_true",
                    help = "ignore the window size, which may cause spurious failures and passes")
    defaults["ignoreWindowSize"] = False

                    action = "append", dest = "extensionsToInstall",
                    help = "install the specified extension in the testing profile. "
                           "The extension file's name should be <id>.xpi where <id> is "
                           "the extension's id as indicated in its install.rdf. "
                           "An optional path can be specified too.")
    defaults["extensionsToInstall"] = []

                    action = "append", type = "string",
                    dest = "environment", metavar = "NAME=VALUE",
                    help = "sets the given variable in the application's "
    defaults["environment"] = []

                    action = "store", type="string", dest = "filter",
                    help = "specifies a regular expression (as could be passed to the JS "
                           "RegExp constructor) to test against URLs in the reftest manifest; "
                           "only test items that have a matching test URL will be run.")
    defaults["filter"] = None

                    action = "store", type = "string", dest = "focusFilterMode",
                    help = "filters tests to run by whether they require focus. "
                           "Valid values are `all', `needs-focus', or `non-needs-focus'. "
                           "Defaults to `all'.")
    defaults["focusFilterMode"] = "all"


  def verifyCommonOptions(self, options, reftest):
    if options.totalChunks is not None and options.thisChunk is None:
      self.error("thisChunk must be specified when totalChunks is specified")

    if options.totalChunks:
      if not 1 <= options.thisChunk <= options.totalChunks:
        self.error("thisChunk must be between 1 and totalChunks")

    if options.logFile:
      options.logFile = reftest.getFullPath(options.logFile)

    if options.xrePath is not None:
      if not os.access(options.xrePath, os.F_OK):
        self.error("--xre-path '%s' not found" % options.xrePath)
      if not os.path.isdir(options.xrePath):
        self.error("--xre-path '%s' is not a directory" % options.xrePath)
      options.xrePath = reftest.getFullPath(options.xrePath)

    return options

def main():
  automation = Automation()
  parser = ReftestOptions(automation)
  reftest = RefTest(automation)

  options, args = parser.parse_args()
  if len(args) != 1:
    print >>sys.stderr, "No reftest.list specified."

  options = parser.verifyCommonOptions(options, reftest) = reftest.getFullPath(
  if not os.path.exists(
    print """Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/reftest/""" \
            % {"app":}

  if options.xrePath is None:
    options.xrePath = os.path.dirname(

  if options.symbolsPath and not isURL(options.symbolsPath):
    options.symbolsPath = reftest.getFullPath(options.symbolsPath)
  options.utilityPath = reftest.getFullPath(options.utilityPath)

  sys.exit(reftest.runTests(args[0], options))

if __name__ == "__main__":