author Benjamin Smedberg <>
Wed, 27 Jan 2010 13:22:25 -0500
changeset 46626 15fc08aacb4e7f5339447fb672f18f6740de0907
parent 46621 15030b53f5f8e7e958fe751257bfc61353d5c65e
parent 46625 c04a2911a953c72a2a925d65554da9c5c7a32439
child 46633 95990e1a50e5647fc6c3b3dcc73ca92c518f8251
permissions -rw-r--r--
Merge backouts related to bug 535564 and bug 542337

# Version: MPL 1.1/GPL 2.0/LGPL 2.1
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
# The Original Code is code.
# The Initial Developer of the Original Code is
# Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2008
# the Initial Developer. All Rights Reserved.
# Contributor(s):
#   Robert Sayre <>
#   Jeff Walden <>
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
# ***** END LICENSE BLOCK *****

import codecs
from datetime import datetime, timedelta
import itertools
import logging
import os
import re
import select
import shutil
import signal
import subprocess
import sys
import threading
import tempfile

#expand _DIST_BIN = __XPC_BIN_PATH__
#expand _IS_WIN32 = len("__WIN32__") != 0
#expand _IS_MAC = __IS_MAC__ != 0
#expand _IS_LINUX = __IS_LINUX__ != 0
#ifdef IS_CYGWIN
#expand _IS_CYGWIN = __IS_CYGWIN__ == 1
_IS_CYGWIN = False
#expand _IS_CAMINO = __IS_CAMINO__ != 0
#expand _BIN_SUFFIX = __BIN_SUFFIX__
#expand _PERL = __PERL__

#expand _DEFAULT_APP = "./" + __BROWSER_PATH__


class SyntaxError(Exception):
  "Signifies a syntax error on a particular line in server-locations.txt."

  def __init__(self, lineno, msg = None):
    self.lineno = lineno
    self.msg = msg

  def __str__(self):
    s = "Syntax error on line " + str(self.lineno)
    if self.msg:
      s += ": %s." % self.msg
      s += "."
    return s

class Location:
  "Represents a location line in server-locations.txt."

  def __init__(self, scheme, host, port, options):
    self.scheme = scheme = host
    self.port = port
    self.options = options

class Automation(object):
  Runs the browser from a script, and provides useful utilities
  for setting up the browser environment.

  IS_WIN32 = _IS_WIN32

  UNIXISH = not IS_WIN32 and not IS_MAC


  SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  sys.path.insert(0, SCRIPT_DIR)
  automationutils = __import__('automationutils')

  # timeout, in seconds

  log = logging.getLogger()

  def __init__(self):

    # We use the logging system here primarily because it'll handle multiple
    # threads, which is needed to process the output of the server and application
    # processes simultaneously.
    handler = logging.StreamHandler(sys.stdout)

  def __all__(self):
    return [

  class Process(subprocess.Popen):
    Represents our view of a subprocess.
    It adds a kill() method which allows it to be stopped explicitly.

    def kill(self):
      if Automation().IS_WIN32:
        import platform
        pid = "%i" %
        if platform.release() == "2000":
          # Windows 2000 needs 'kill.exe' from the 
          #'Windows 2000 Resource Kit tools'. (See bug 475455.)
            subprocess.Popen(["kill", "-f", pid]).wait()
  "TEST-UNEXPECTED-FAIL | | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
          # Windows XP and later.
          subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
        os.kill(, signal.SIGKILL)

  def readLocations(self, locationsPath = "server-locations.txt"):
    Reads the locations at which the Mochitest HTTP server is available from

    locationFile =, "r", "UTF-8")

    # Perhaps more detail than necessary, but it's the easiest way to make sure
    # we get exactly the format we want.  See server-locations.txt for the exact
    # format guaranteed here.
    lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
    locations = []
    lineno = 0
    seenPrimary = False
    for line in locationFile:
      lineno += 1
      if line.startswith("#") or line == "\n":
      match = lineRe.match(line)
      if not match:
        raise SyntaxError(lineno)

      options ="options")
      if options:
        options = options.split(",")
        if "primary" in options:
          if seenPrimary:
            raise SyntaxError(lineno, "multiple primary locations")
          seenPrimary = True
        options = []

                      "port"), options))

    if not seenPrimary:
      raise SyntaxError(lineno + 1, "missing primary location")

    return locations

  def initializeProfile(self, profileDir, extraPrefs = []):
    "Sets up the standard testing profile."

    # Start with a clean slate.
    shutil.rmtree(profileDir, True)

    prefs = []

    part = """\
user_pref("browser.dom.window.dump.enabled", true);
user_pref("dom.allow_scripts_to_close_windows", true);
user_pref("dom.disable_open_during_load", false);
user_pref("dom.max_script_run_time", 0); // no slow script dialogs
user_pref("dom.max_chrome_script_run_time", 0);
user_pref("dom.popup_maximum", -1);
user_pref("signed.applets.codebase_principal_support", true);
user_pref("security.warn_submit_insecure", false);
user_pref("", false);
user_pref("shell.checkDefaultClient", false);
user_pref("browser.warnOnQuit", false);
user_pref("accessibility.typeaheadfind.autostart", false);
user_pref("javascript.options.showInConsole", true);
user_pref("layout.debug.enable_data_xbl", true);
user_pref("browser.EULA.override", true);
user_pref("javascript.options.jit.content", true);
user_pref("gfx.color_management.force_srgb", true);
user_pref("network.manage-offline-status", false);
user_pref("test.mousescroll", true);
user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
user_pref("network.http.prompt-temp-redirect", false);
user_pref("media.cache_size", 100);
user_pref("security.warn_viewing_mixed", false);

user_pref("geo.wifi.uri", "http://localhost:8888/tests/dom/tests/mochitest/geolocation/network_geolocation.sjs");
user_pref("geo.wifi.testing", true);

user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others

// Make url-classifier updates so rare that they won't affect tests
user_pref("urlclassifier.updateinterval", 172800);
// Point the url-classifier to the local testing server for fast failures
user_pref("browser.safebrowsing.provider.0.gethashURL", "http://localhost:8888/safebrowsing-dummy/gethash");
user_pref("browser.safebrowsing.provider.0.keyURL", "http://localhost:8888/safebrowsing-dummy/newkey");
user_pref("browser.safebrowsing.provider.0.lookupURL", "http://localhost:8888/safebrowsing-dummy/lookup");
user_pref("browser.safebrowsing.provider.0.updateURL", "http://localhost:8888/safebrowsing-dummy/update");

    locations = self.readLocations()

    # Grant God-power to all the privileged servers on which tests run.
    privileged = filter(lambda loc: "privileged" in loc.options, locations)
    for (i, l) in itertools.izip(itertools.count(1), privileged):
      part = """
          "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
           UniversalPreferencesRead UniversalPreferencesWrite \
user_pref("capability.principal.codebase.p%(i)", "%(origin)s");
user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
"""  % { "i": i,
         "origin": (l.scheme + "://" + + ":" + l.port) }

    # We need to proxy every server but the primary one.
    origins = ["'%s://%s:%s'" % (l.scheme,, l.port)
              for l in filter(lambda l: "primary" not in l.options, locations)]
    origins = ", ".join(origins)

    pacURL = """data:text/plain,
function FindProxyForURL(url, host)
  var origins = [%(origins)s];
  var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
                         '://' +
                         '(?:[^/@]*@)?' +
                         '(.*?)' +
  var matches = regex.exec(url);
  if (!matches)
    return 'DIRECT';
  var isHttp = matches[1] == 'http';
  var isHttps = matches[1] == 'https';
  if (!matches[3])
    if (isHttp) matches[3] = '80';
    if (isHttps) matches[3] = '443';
  var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
  if (origins.indexOf(origin) < 0)
    return 'DIRECT';
  if (isHttp)
    return 'PROXY';
  if (isHttps)
    return 'PROXY';
  return 'DIRECT';
}""" % { "origins": origins }
    pacURL = "".join(pacURL.splitlines())

    part = """
user_pref("network.proxy.type", 2);
user_pref("network.proxy.autoconfig_url", "%(pacURL)s");

user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
""" % {"pacURL": pacURL}

    for v in extraPrefs:
      thispref = v.split("=")
      if len(thispref) < 2:
        print "Error: syntax error in --setpref=" + v
      part = 'user_pref("%s", %s);\n' % (thispref[0], thispref[1])

    # write the preferences
    prefsFile = open(profileDir + "/" + "user.js", "a")

  def addCommonOptions(self, parser):
    "Adds command-line options which are common to mochitest and reftest."

                      action = "append", type = "string",
                      default = [],
                      dest = "extraPrefs", metavar = "PREF=VALUE",
                      help = "defines an extra user preference")  

  def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
    pwfilePath = os.path.join(profileDir, ".crtdbpw")
    pwfile = open(pwfilePath, "w")

    # Create head of the ssltunnel configuration file
    sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
    sslTunnelConfig = open(sslTunnelConfigPath, "w")
    sslTunnelConfig.write("certdbdir:%s\n" % certPath)
    sslTunnelConfig.write("listen:*:4443:pgo server certificate\n")

    # Configure automatic certificate and bind custom certificates, client authentication
    locations = self.readLocations()
    for loc in locations:
      if loc.scheme == "https" and "nocert" not in loc.options:
        customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
        clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
        for option in loc.options:
          match = customCertRE.match(option)
          if match:
            customcert ="nickname");
            sslTunnelConfig.write("listen:%s:%s:4443:%s\n" %
                      (, loc.port, customcert))

          match = clientAuthRE.match(option)
          if match:
            clientauth ="clientauth");
            sslTunnelConfig.write("clientauth:%s:%s:4443:%s\n" %
                      (, loc.port, clientauth))


    # Pre-create the certification database for the profile
    env = self.environment(xrePath = xrePath)
    certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
    pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)

    status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
    if status != 0:
      return status

    # Walk the cert directory and add custom CAs and client certs
    files = os.listdir(certPath)
    for item in files:
      root, ext = os.path.splitext(item)
      if ext == ".ca":
        trustBits = "CT,,"
        if root.endswith("-object"):
          trustBits = "CT,,CT"
        self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
                    "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
                    env = env).wait()
      if ext == ".client":
        self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
                    pwfilePath, "-d", profileDir], 
                    env = env).wait()

    return 0

  def environment(self, env = None, xrePath = None, crashreporter = True):
    if xrePath == None:
      xrePath = self.DIST_BIN
    if env == None:
      env = dict(os.environ)

    ldLibraryPath = os.path.abspath(os.path.join(self.SCRIPT_DIR, xrePath))
    if self.UNIXISH or self.IS_MAC:
      envVar = "LD_LIBRARY_PATH"
      if self.IS_MAC:
        envVar = "DYLD_LIBRARY_PATH"
      else: # unixish
        env['MOZILLA_FIVE_HOME'] = xrePath
      if envVar in env:
        ldLibraryPath = ldLibraryPath + ":" + env[envVar]
      env[envVar] = ldLibraryPath
    elif self.IS_WIN32:
      env["PATH"] = env["PATH"] + ";" + ldLibraryPath

    if crashreporter:
      env['MOZ_CRASHREPORTER'] = '1'

    return env

  if IS_WIN32:
    ctypes = __import__('ctypes')
    wintypes = __import__('ctypes.wintypes')
    time = __import__('time')
    msvcrt = __import__('msvcrt')
    PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
    GetLastError = ctypes.windll.kernel32.GetLastError

    def readWithTimeout(self, f, timeout):
      """Try to read a line of output from the file object |f|.
      |f| must be a  pipe, like the |stdout| member of a subprocess.Popen
      object created with stdout=PIPE. If no output
      is received within |timeout| seconds, return a blank line.
      Returns a tuple (line, did_timeout), where |did_timeout| is True
      if the read timed out, and False otherwise."""
      if timeout is None:
        # shortcut to allow callers to pass in "None" for no timeout.
        return (f.readline(), False)
      x = self.msvcrt.get_osfhandle(f.fileno())
      l = self.ctypes.c_long()
      done = self.time.time() + timeout
      while self.time.time() < done:
        if self.PeekNamedPipe(x, None, 0, None, self.ctypes.byref(l), None) == 0:
          err = self.GetLastError()
          if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
            return ('', False)
            log.error("readWithTimeout got error: %d", err)
        if l.value > 0:
          # we're assuming that the output is line-buffered,
          # which is not unreasonable
          return (f.readline(), False)
      return ('', True)

    def isPidAlive(self, pid):
      STILL_ACTIVE = 259
      pHandle = self.ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
      if not pHandle:
        return False
      pExitCode = ctypes.wintypes.DWORD()
      self.ctypes.windll.kernel32.GetExitCodeProcess(pHandle, self.ctypes.byref(pExitCode))
      if (pExitCode.value == STILL_ACTIVE):
        return True
        return False

    def killPid(self, pid):
      PROCESS_TERMINATE = 0x0001
      pHandle = self.ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
      if not pHandle:
      success = self.ctypes.windll.kernel32.TerminateProcess(pHandle, 1)

    errno = __import__('errno')

    def readWithTimeout(self, f, timeout):
      """Try to read a line of output from the file object |f|. If no output
      is received within |timeout| seconds, return a blank line.
      Returns a tuple (line, did_timeout), where |did_timeout| is True
      if the read timed out, and False otherwise."""
      (r, w, e) =[f], [], [], timeout)
      if len(r) == 0:
        return ('', True)
      return (f.readline(), False)

    def isPidAlive(self, pid):
        # kill(pid, 0) checks for a valid PID without actually sending a signal
        # The method throws OSError if the PID is invalid, which we catch below.
        os.kill(pid, 0)

        # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
        # the process terminates before we get to this point.
        wpid, wstatus = os.waitpid(pid, os.WNOHANG)
        if wpid == 0:
          return True

        return False
      except OSError, err:
        # Catch the errors we might expect from os.kill/os.waitpid, 
        # and re-raise any others
        if err.errno == self.errno.ESRCH or err.errno == self.errno.ECHILD:
          return False

    def killPid(self, pid):
      os.kill(pid, signal.SIGKILL)

  def triggerBreakpad(self, proc, utilityPath):
    """Attempt to kill this process in a way that triggers Breakpad crash
    reporting, if we know how for this platform. Otherwise just .kill() it."""
    if self.CRASHREPORTER:
      if self.UNIXISH:
        # SEGV will get picked up by Breakpad's signal handler
        os.kill(, signal.SIGSEGV)
      elif self.IS_WIN32:
        # We should have a "crashinject" program in our utility path
        crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
        if os.path.exists(crashinject) and subprocess.Popen([crashinject, str(]).wait() == 0:
    #TODO: kill the process such that it triggers Breakpad on OS X (bug 525296)"Can't trigger Breakpad, just killing process")

  def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime):
    """ Look for timeout or crashes and return the status after the process terminates """
    stackFixerProcess = None
    didTimeout = False
    if proc.stdout is None:"TEST-INFO: Not logging stdout or stderr due to debugger connection")
      logsource = proc.stdout
      if self.IS_DEBUG_BUILD:
        stackFixerCommand = None
        if self.IS_MAC:
          stackFixerCommand = ""
        elif self.IS_LINUX:
          stackFixerCommand = ""
        if stackFixerCommand is not None:
          stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, stackFixerCommand)], 
          logsource = stackFixerProcess.stdout

      (line, didTimeout) = self.readWithTimeout(logsource, timeout)
      hitMaxTime = False
      while line != "" and not didTimeout:
        (line, didTimeout) = self.readWithTimeout(logsource, timeout)
        if not hitMaxTime and maxTime and - startTime > timedelta(seconds = maxTime):
          # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
          hitMaxTime = True
"TEST-UNEXPECTED-FAIL | | application ran for longer than allowed maximum time of %d seconds", int(maxTime))
          self.triggerBreakpad(proc, utilityPath)
      if didTimeout:"TEST-UNEXPECTED-FAIL | | application timed out after %d seconds with no output", int(timeout))
        self.triggerBreakpad(proc, utilityPath)

    status = proc.wait()
    if status != 0 and not didTimeout and not hitMaxTime:"TEST-UNEXPECTED-FAIL | | Exited with code %d during test run", status)
    if stackFixerProcess is not None:
      fixerStatus = stackFixerProcess.wait()
      if fixerStatus != 0 and not didTimeout and not hitMaxTime:"TEST-UNEXPECTED-FAIL | | Stack fixer process exited with code %d during test run", fixerStatus)
    return status

  def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
    """ build the application command line """

    cmd = app
    if self.IS_MAC and not self.IS_CAMINO and not cmd.endswith("-bin"):
      cmd += "-bin"
    cmd = os.path.abspath(cmd)

    args = []

    if debuggerInfo:
      cmd = os.path.abspath(debuggerInfo["path"])

    if self.IS_MAC:

    if self.IS_CYGWIN:
      profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
      profileDirectory = profileDir + "/"

    args.extend(("-no-remote", "-profile", profileDirectory))
    if testURL is not None:
      if self.IS_CAMINO:
        args.extend(("-url", testURL))
    return cmd, args

  def checkForZombies(self, processLog):
    """ Look for hung processes """
    if not os.path.exists(processLog):'INFO | | PID log not found: %s', processLog)
    else:'INFO | | Reading PID log: %s', processLog)
      processList = []
      pidRE = re.compile(r'launched child process (\d+)$')
      processLogFD = open(processLog)
      for line in processLogFD:
        m =
        if m:

      for processPID in processList:"INFO | | Checking for orphan process with PID: %d", processPID)
        if self.isPidAlive(processPID):
"TEST-UNEXPECTED-FAIL | | child process %d still alive after shutdown", processPID)

  def runApp(self, testURL, env, app, profileDir, extraArgs,
             runSSLTunnel = False, utilityPath = None,
             xrePath = None, certPath = None,
             debuggerInfo = None, symbolsPath = None,
             timeout = -1, maxTime = None):
    Run the app, log the duration it took to execute, return the status code.
    Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.

    if utilityPath == None:
      utilityPath = self.DIST_BIN
    if xrePath == None:
      xrePath = self.DIST_BIN
    if certPath == None:
      certPath = self.CERTS_SRC_DIR
    if timeout == -1:
      timeout = self.DEFAULT_TIMEOUT

    # copy env so we don't munge the caller's environment
    env = dict(env);
    env["NO_EM_RESTART"] = "1"
    tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
    env["MOZ_PROCESS_LOG"] = processLog

    if self.IS_TEST_BUILD and runSSLTunnel:
      # create certificate database for the profile
      certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
      if certificateStatus != 0:"TEST-UNEXPECTED FAIL | | Certificate integration failed")
        return certificateStatus

      # start ssltunnel to provide https:// URLs capability
      ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
      ssltunnelProcess = self.Process([ssltunnel, 
                               os.path.join(profileDir, "ssltunnel.cfg")], 
                               env = self.environment(xrePath = xrePath))"INFO | | SSL tunnel pid: %d",

    cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
    startTime =

    # Don't redirect stdout and stderr if an interactive debugger is attached
    if debuggerInfo and debuggerInfo["interactive"]:
      outputPipe = None
      outputPipe = subprocess.PIPE

    proc = self.Process([cmd] + args,
                 env = self.environment(env, xrePath = xrePath,
                                   crashreporter = not debuggerInfo),
                 stdout = outputPipe,
                 stderr = subprocess.STDOUT)"INFO | | Application pid: %d",

    status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime)"INFO | | Application ran for: %s", str( - startTime))

    # Do a final check for zombie child processes.
    self.automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)

    if os.path.exists(processLog):

    if self.IS_TEST_BUILD and runSSLTunnel:

    return status