Bug 778133 - Update mozbase in m-c for bug 776931. r=jhammel
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 02 Aug 2012 17:17:34 -0400
changeset 101261 f36cf6866998430dfadda7b138350083ae374537
parent 101260 ba9e4b81d7f60615ad111586a184e3bbcdb9a1c7
child 101262 3a17236e908482aa5f48580e5e5b5102bb2b7333
push id12924
push userryanvm@gmail.com
push dateThu, 02 Aug 2012 21:20:47 +0000
treeherdermozilla-inbound@3a17236e9084 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhammel
bugs778133, 776931
milestone17.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 778133 - Update mozbase in m-c for bug 776931. r=jhammel
testing/mozbase/README
testing/mozbase/mozprocess/mozprocess/processhandler.py
testing/mozbase/mozprocess/tests/mozprocess1.py
testing/mozbase/mozprocess/tests/mozprocess2.py
testing/mozbase/mozprofile/mozprofile/addons.py
testing/mozbase/mozprofile/mozprofile/permissions.py
testing/mozbase/mozprofile/mozprofile/profile.py
testing/mozbase/mozprofile/tests/permissions.py
testing/mozbase/mozprofile/tests/server_locations.py
testing/mozbase/mozrunner/mozrunner/runner.py
testing/mozbase/test-manifest.ini
testing/peptest/MANIFEST.in
testing/peptest/peptest/extension/build.xml
testing/peptest/peptest/extension/resource/mozmill/README.md
testing/peptest/peptest/pepprocess.py
testing/peptest/peptest/runpeptests.py
--- a/testing/mozbase/README
+++ b/testing/mozbase/README
@@ -1,9 +1,27 @@
-This is the git repo for the Mozilla mozbase suite of python utilities.
+This is the git repository for the Mozilla mozbase suite of python utilities.
+
+MozBase is composed of several python packages. These packages work
+together to form the basis of a test harness.
+
+* Firefox is launched via [mozrunner](https://github.com/mozilla/mozbase/tree/master/mozrunner)
+** which sets up a profile with preferences and extensions using
+   [mozprofile ](https://github.com/mozilla/mozbase/tree/master/mozprofile)
+** and runs the application under test using [mozprocess](https://github.com/mozilla/mozbase/tree/master/mozprocess)
+* [mozInstall](https://github.com/mozilla/mozbase/tree/master/mozinstall) is used to install the test application
+* A test harness may direct Firefox to load web pages. These may be
+  served using [mozhttpd](https://github.com/mozilla/mozbase/tree/master/mozhttpd) for testing
+* The machine environment is introspected by [mozinfo](https://github.com/mozilla/mozbase/tree/master/mozinfo)
+* A test manifest may be read to determine the tests to be run. These
+  manifests are processed by [ManifestDestiny](https://github.com/mozilla/mozbase/tree/master/manifestdestiny)
+* For mozbile testing, the test runner communicates to the test agent
+  using [mozdevice](https://github.com/mozilla/mozbase/tree/master/mozdevice)
+  (Note that the canonical location of mozdevice is
+  mozilla-central: http://mxr.mozilla.org/mozilla-central/source/build/mobile/)
 
 Learn more about mozbase here:
 https://wiki.mozilla.org/Auto-tools/Projects/MozBase
 
 Bugs live at
 https://bugzilla.mozilla.org/buglist.cgi?resolution=---&component=Mozbase&product=Testing and https://bugzilla.mozilla.org/buglist.cgi?resolution=---&status_whiteboard_type=allwordssubstr&query_format=advanced&status_whiteboard=mozbase
 
 To file a bug, go to
--- a/testing/mozbase/mozprocess/mozprocess/processhandler.py
+++ b/testing/mozbase/mozprocess/mozprocess/processhandler.py
@@ -557,16 +557,17 @@ falling back to not using job objects fo
         will be logged.
         """
         self.cmd = cmd
         self.args = args
         self.cwd = cwd
         self.didTimeout = False
         self._ignore_children = ignore_children
         self.keywordargs = kwargs
+        self.outThread = None
 
         if env is None:
             env = os.environ.copy()
         self.env = env
 
         # handlers
         self.processOutputLineHandlers = list(processOutputLine)
         self.onTimeoutHandlers = list(onTimeout)
@@ -642,55 +643,79 @@ falling back to not using job objects fo
         for handler in self.onTimeoutHandlers:
             handler()
 
     def onFinish(self):
         """Called when a process finishes without a timeout."""
         for handler in self.onFinishHandlers:
             handler()
 
-    def waitForFinish(self, timeout=None, outputTimeout=None):
+    def processOutput(self, timeout=None, outputTimeout=None):
         """
         Handle process output until the process terminates or times out.
 
         If timeout is not None, the process will be allowed to continue for
         that number of seconds before being killed.
 
         If outputTimeout is not None, the process will be allowed to continue
         for that number of seconds without producing any output before
         being killed.
         """
-
-        if not hasattr(self, 'proc'):
-            self.run()
-
-        self.didTimeout = False
-        logsource = self.proc.stdout
+        def _processOutput():
+            if not hasattr(self, 'proc'):
+                self.run()
 
-        lineReadTimeout = None
-        if timeout:
-            lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
-        elif outputTimeout:
-            lineReadTimeout = outputTimeout
+            self.didTimeout = False
+            logsource = self.proc.stdout
 
-        (line, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
-        while line != "" and not self.didTimeout:
-            self.processOutputLine(line.rstrip())
+            lineReadTimeout = None
             if timeout:
                 lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
+            elif outputTimeout:
+                lineReadTimeout = outputTimeout
+
             (line, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
+            while line != "" and not self.didTimeout:
+                self.processOutputLine(line.rstrip())
+                if timeout:
+                    lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
+                (line, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
+
+            if self.didTimeout:
+                self.proc.kill()
+                self.onTimeout()
+            else:
+                self.onFinish()
+        
+        if not self.outThread:
+            self.outThread = threading.Thread(target=_processOutput)
+            self.outThread.daemon = True
+            self.outThread.start()
 
-        if self.didTimeout:
-            self.proc.kill()
-            self.onTimeout()
-        else:
-            self.onFinish()
+
+    def waitForFinish(self, timeout=None):
+        """
+        Waits until all output has been read and the process is 
+        terminated.
 
-        status = self.proc.wait()
-        return status
+        If timeout is not None, will return after timeout seconds.
+        This timeout is only for waitForFinish and doesn't affect
+        the didTimeout or onTimeout properties.
+        """
+        if self.outThread:
+            # Thread.join() blocks the main thread until outThread is finished
+            # wake up once a second in case a keyboard interrupt is sent
+            count = 0
+            while self.outThread.isAlive():
+                self.outThread.join(timeout=1)
+                count += 1
+                if timeout and count > timeout:
+                    return
+
+        return self.proc.wait()
 
 
     ### Private methods from here on down. Thar be dragons.
 
     if mozinfo.isWin:
         # Windows Specific private functions are defined in this block
         PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
         GetLastError = ctypes.windll.kernel32.GetLastError
@@ -757,17 +782,16 @@ class LogOutput(object):
             self.file = file(self.filename, 'a')
         self.file.write(line + '\n')
         self.file.flush()
 
     def __del__(self):
         if self.file is not None:
             self.file.close()
 
-
 ### front end class with the default handlers
 
 class ProcessHandler(ProcessHandlerMixin):
 
     def __init__(self, cmd, logfile=None, storeOutput=True, **kwargs):
         """
         If storeOutput=True, the output produced by the process will be saved
         as self.output.
--- a/testing/mozbase/mozprocess/tests/mozprocess1.py
+++ b/testing/mozbase/mozprocess/tests/mozprocess1.py
@@ -41,30 +41,30 @@ def check_for_process(processName):
         output -- if process exists, stdout of the process, '' otherwise
     """
     output = ''
     if sys.platform == "win32":
         # On windows we use tasklist
         p1 = subprocess.Popen(["tasklist"], stdout=subprocess.PIPE)
         output = p1.communicate()[0]
         detected = False
-        for line in output:
+        for line in output.splitlines():
             if processName in line:
                 detected = True
                 break
     else:
-        p1 = subprocess.Popen(["ps", "-A"], stdout=subprocess.PIPE)
+        p1 = subprocess.Popen(["ps", "-ef"], stdout=subprocess.PIPE)
         p2 = subprocess.Popen(["grep", processName], stdin=p1.stdout, stdout=subprocess.PIPE)
         p1.stdout.close()
         output = p2.communicate()[0]
         detected = False
-        for line in output:
+        for line in output.splitlines():
             if "grep %s" % processName in line:
                 continue
-            elif processName in line:
+            elif processName in line and not 'defunct' in line: 
                 detected = True
                 break
 
     return detected, output
 
 
 class ProcTest1(unittest.TestCase):
 
@@ -76,47 +76,50 @@ class ProcTest1(unittest.TestCase):
         unittest.TestCase.__init__(self, *args, **kwargs)
 
     def test_process_normal_finish(self):
         """Process is started, runs to completion while we wait for it"""
 
         p = processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"],
                                           cwd=here)
         p.run()
+        p.processOutput()
         p.waitForFinish()
 
         detected, output = check_for_process(self.proclaunch)
         self.determine_status(detected,
                               output,
                               p.proc.returncode,
                               p.didTimeout)
 
     def test_process_waittimeout(self):
         """ Process is started, runs but we time out waiting on it
             to complete
         """
         p = processhandler.ProcessHandler([self.proclaunch, "process_waittimeout.ini"],
                                           cwd=here)
         p.run()
-        p.waitForFinish(timeout=10)
+        p.processOutput(timeout=10) 
+        p.waitForFinish()
 
         detected, output = check_for_process(self.proclaunch)
         self.determine_status(detected,
                               output,
                               p.proc.returncode,
                               p.didTimeout,
                               False,
                               ['returncode', 'didtimeout'])
 
     def test_process_kill(self):
         """ Process is started, we kill it
         """
         p = processhandler.ProcessHandler([self.proclaunch, "process_normal_finish.ini"],
                                           cwd=here)
         p.run()
+        p.processOutput()
         p.kill()
 
         detected, output = check_for_process(self.proclaunch)
         self.determine_status(detected,
                               output,
                               p.proc.returncode,
                               p.didTimeout)
 
@@ -134,24 +137,24 @@ class ProcTest1(unittest.TestCase):
             output -- string of data from detected process, can be ''
             returncode -- return code from process, defaults to 0
             didtimeout -- True if process timed out, defaults to False
             isalive -- Use True to indicate we pass if the process exists; however, by default
                        the test will pass if the process does not exist (isalive == False)
             expectedfail -- Defaults to [], used to indicate a list of fields that are expected to fail
         """
         if 'returncode' in expectedfail:
-            self.assertTrue(returncode, "Detected an expected non-zero return code")
-        else:
+            self.assertTrue(returncode, "Detected an unexpected return code of: %s" % returncode)
+        elif not isalive:
             self.assertTrue(returncode == 0, "Detected non-zero return code of: %d" % returncode)
 
         if 'didtimeout' in expectedfail:
-            self.assertTrue(didtimeout, "Process timed out as expected")
+            self.assertTrue(didtimeout, "Detected that process didn't time out")
         else:
             self.assertTrue(not didtimeout, "Detected that process timed out")
 
-        if detected:
-            self.assertTrue(isalive, "Detected process is still running, process output: %s" % output)
+        if isalive:
+            self.assertTrue(detected, "Detected process is not running, process output: %s" % output)
         else:
-            self.assertTrue(not isalive, "Process ended")
+            self.assertTrue(not detected, "Detected process is still running, process output: %s" % output)
 
 if __name__ == '__main__':
     unittest.main()
--- a/testing/mozbase/mozprocess/tests/mozprocess2.py
+++ b/testing/mozbase/mozprocess/tests/mozprocess2.py
@@ -46,75 +46,97 @@ def check_for_process(processName):
         output -- if process exists, stdout of the process, '' otherwise
     """
     output = ''
     if sys.platform == "win32":
         # On windows we use tasklist
         p1 = subprocess.Popen(["tasklist"], stdout=subprocess.PIPE)
         output = p1.communicate()[0]
         detected = False
-        for line in output:
+        for line in output.splitlines():
             if processName in line:
                 detected = True
                 break
     else:
-        p1 = subprocess.Popen(["ps", "-A"], stdout=subprocess.PIPE)
+        p1 = subprocess.Popen(["ps", "-ef"], stdout=subprocess.PIPE)
         p2 = subprocess.Popen(["grep", processName], stdin=p1.stdout, stdout=subprocess.PIPE)
         p1.stdout.close()
         output = p2.communicate()[0]
         detected = False
-        for line in output:
+        for line in output.splitlines():
             if "grep %s" % processName in line:
                 continue
-            elif processName in line:
+            elif processName in line and not 'defunct' in line:
                 detected = True
                 break
 
     return detected, output
 
 class ProcTest2(unittest.TestCase):
 
     def __init__(self, *args, **kwargs):
 
         # Ideally, I'd use setUpClass but that only exists in 2.7.
         # So, we'll do this make step now.
         self.proclaunch = make_proclaunch(here)
         unittest.TestCase.__init__(self, *args, **kwargs)
 
-    def test_process_waittimeout(self):
+    def test_process_waitnotimeout(self):
         """ Process is started, runs to completion before our wait times out
         """
         p = processhandler.ProcessHandler([self.proclaunch,
                                           "process_waittimeout_10s.ini"],
                                           cwd=here)
         p.run()
-        p.waitForFinish(timeout=30)
+        p.processOutput(timeout=30)
+        p.waitForFinish()
 
         detected, output = check_for_process(self.proclaunch)
         self.determine_status(detected,
                               output,
                               p.proc.returncode,
                               p.didTimeout)
 
-    def test_process_waitnotimeout(self):
+    def test_process_wait(self):
         """ Process is started runs to completion while we wait indefinitely
         """
 
         p = processhandler.ProcessHandler([self.proclaunch,
                                           "process_waittimeout_10s.ini"],
                                           cwd=here)
         p.run()
         p.waitForFinish()
 
         detected, output = check_for_process(self.proclaunch)
         self.determine_status(detected,
                               output,
                               p.proc.returncode,
                               p.didTimeout)
 
+    def test_process_waittimeout(self):
+        """
+        Process is started, then waitForFinish is called and times out.
+        Process is still running and didn't timeout
+        """
+        p = processhandler.ProcessHandler([self.proclaunch,
+                                          "process_waittimeout_10s.ini"],
+                                          cwd=here)
+
+        p.run()
+        p.processOutput()
+        p.waitForFinish(timeout=5)
+
+        detected, output = check_for_process(self.proclaunch)
+        self.determine_status(detected,
+                              output,
+                              p.proc.returncode,
+                              p.didTimeout,
+                              True,
+                              [])
+
     def determine_status(self,
                          detected=False,
                          output = '',
                          returncode = 0,
                          didtimeout = False,
                          isalive=False,
                          expectedfail=[]):
         """
@@ -124,24 +146,24 @@ class ProcTest2(unittest.TestCase):
             output -- string of data from detected process, can be ''
             returncode -- return code from process, defaults to 0
             didtimeout -- True if process timed out, defaults to False
             isalive -- Use True to indicate we pass if the process exists; however, by default
                        the test will pass if the process does not exist (isalive == False)
             expectedfail -- Defaults to [], used to indicate a list of fields that are expected to fail
         """
         if 'returncode' in expectedfail:
-            self.assertTrue(returncode, "Detected an expected non-zero return code")
-        else:
+            self.assertTrue(returncode, "Detected an unexpected return code of: %s" % returncode)
+        elif not isalive:
             self.assertTrue(returncode == 0, "Detected non-zero return code of: %d" % returncode)
 
         if 'didtimeout' in expectedfail:
-            self.assertTrue(didtimeout, "Process timed out as expected")
+            self.assertTrue(didtimeout, "Detected that process didn't time out")
         else:
             self.assertTrue(not didtimeout, "Detected that process timed out")
 
-        if detected:
-            self.assertTrue(isalive, "Detected process is still running, process output: %s" % output)
+        if isalive:
+            self.assertTrue(detected, "Detected process is not running, process output: %s" % output)
         else:
-            self.assertTrue(not isalive, "Process ended")
+            self.assertTrue(not detected, "Detected process is still running, process output: %s" % output)
 
 if __name__ == '__main__':
     unittest.main()
--- a/testing/mozbase/mozprofile/mozprofile/addons.py
+++ b/testing/mozbase/mozprofile/mozprofile/addons.py
@@ -167,18 +167,21 @@ class AddonManager(object):
             os.close(fd)
             tmpfile = path
         else:
             tmpfile = None
 
         # if the addon is a directory, install all addons in it
         addons = [path]
         if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
-            assert os.path.isdir(path), "Addon '%s' cannot be installed" % path
-            addons = [os.path.join(path, x) for x in os.listdir(path)]
+            # If the path doesn't exist, then we don't really care, just return
+            if not os.path.isdir(path):
+                return
+            addons = [os.path.join(path, x) for x in os.listdir(path) if
+                    os.path.isdir(os.path.join(path, x))]
 
         # install each addon
         for addon in addons:
             tmpdir = None
             xpifile = None
             if addon.endswith('.xpi'):
                 tmpdir = tempfile.mkdtemp(suffix = '.' + os.path.split(addon)[-1])
                 compressed_file = zipfile.ZipFile(addon, 'r')
@@ -196,17 +199,17 @@ class AddonManager(object):
                 addon = tmpdir
 
             # determine the addon id
             addon_details = AddonManager.addon_details(addon)
             addon_id = addon_details.get('id')
             assert addon_id, 'The addon id could not be found: %s' % addon
 
             # copy the addon to the profile
-            extensions_path = os.path.join(self.profile, 'extensions')
+            extensions_path = os.path.join(self.profile, 'extensions', 'staged')
             addon_path = os.path.join(extensions_path, addon_id)
             if not unpack and not addon_details['unpack'] and xpifile:
                 if not os.path.exists(extensions_path):
                     os.makedirs(extensions_path)
                 shutil.copy(xpifile, addon_path + '.xpi')
             else:
                 dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
                 self._addon_dirs.append(addon_path)
--- a/testing/mozbase/mozprofile/mozprofile/permissions.py
+++ b/testing/mozbase/mozprofile/mozprofile/permissions.py
@@ -123,19 +123,17 @@ class ServerLocations(object):
     def __len__(self):
         return len(self._locations)
 
     def add(self, location, suppress_callback=False):
         if "primary" in location.options:
             if self.hasPrimary:
                 raise MultiplePrimaryLocationsError()
             self.hasPrimary = True
-        for loc in self._locations:
-            if loc.isEqual(location):
-                raise DuplicateLocationError(location.url())
+
         self._locations.append(location)
         if self.add_callback and not suppress_callback:
             self.add_callback([location])
 
     def add_host(self, host, port='80', scheme='http', options='privileged'):
         if isinstance(options, basestring):
             options = options.split(',')
         self.add(Location(scheme, host, port, options))
@@ -145,17 +143,17 @@ class ServerLocations(object):
         Reads the file (in the format of server-locations.txt) and add all
         valid locations to the self._locations array.
 
         If check_for_primary is True, a MissingPrimaryLocationError
         exception is raised if no primary is found.
 
         This format:
         http://mxr.mozilla.org/mozilla-central/source/build/pgo/server-locations.txt
-        The only exception is that the port, if not defined, defaults to 80.
+        The only exception is that the port, if not defined, defaults to 80 or 443.
 
         FIXME: Shouldn't this default to the protocol-appropriate port?  Is
         there any reason to have defaults at all?
         """
 
         locationFile = codecs.open(filename, "r", "UTF-8")
         lineno = 0
         new_locations = []
@@ -180,17 +178,21 @@ class ServerLocations(object):
             if '://' not in server:
                 server = 'http://' + server
             scheme, netloc, path, query, fragment = urlparse.urlsplit(server)
             # get the host and port
             try:
                 host, port = netloc.rsplit(':', 1)
             except ValueError:
                 host = netloc
-                port = '80'
+                default_ports = {'http': '80',
+                                 'https': '443',
+                                 'ws': '443',
+                                 'wss': '443'}
+                port = default_ports.get(scheme, '80')
 
             try:
                 location = Location(scheme, host, port, options)
                 self.add(location, suppress_callback=True)
             except LocationError, e:
                 raise LocationsSyntaxError(lineno, e)
 
             new_locations.append(location)
@@ -263,18 +265,18 @@ class Permissions(object):
         """
 
         # Grant God-power to all the privileged servers on which tests run.
         prefs = []
         privileged = [i for i in self._locations if "privileged" in i.options]
         for (i, l) in itertools.izip(itertools.count(1), privileged):
             prefs.append(("capability.principal.codebase.p%s.granted" % i, "UniversalXPConnect"))
 
-            # TODO: do we need the port?
-            prefs.append(("capability.principal.codebase.p%s.id" % i, l.scheme + "://" + l.host))
+            prefs.append(("capability.principal.codebase.p%s.id" % i, "%s://%s:%s" %
+                        (l.scheme, l.host, l.port)))
             prefs.append(("capability.principal.codebase.p%s.subjectName" % i, ""))
 
         if proxy:
             user_prefs = self.pac_prefs()
         else:
             user_prefs = []
 
         return prefs, user_prefs
@@ -284,37 +286,38 @@ class Permissions(object):
         return preferences for Proxy Auto Config. originally taken from
         http://mxr.mozilla.org/mozilla-central/source/build/automation.py.in
         """
 
         prefs = []
 
         # We need to proxy every server but the primary one.
         origins = ["'%s'" % l.url()
-                   for l in self._locations
-                   if "primary" not in l.options]
+                   for l in self._locations]
+
         origins = ", ".join(origins)
 
-        # TODO: this is not a reliable way to determine the Proxy host
         for l in self._locations:
             if "primary" in l.options:
                 webServer = l.host
-                httpPort  = l.port
-                sslPort   = 443
+                port = l.port
 
         # TODO: this should live in a template!
+        # TODO: So changing the 5th line of the regex below from (\\\\\\\\d+)
+        # to (\\\\d+) makes this code work. Not sure why there would be this
+        # difference between automation.py.in and this file.
         pacURL = """data:text/plain,
 function FindProxyForURL(url, host)
 {
   var origins = [%(origins)s];
   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
                          '://' +
                          '(?:[^/@]*@)?' +
                          '(.*?)' +
-                         '(?::(\\\\\\\\d+))?/');
+                         '(?::(\\\\d+))?/');
   var matches = regex.exec(url);
   if (!matches)
     return 'DIRECT';
   var isHttp = matches[1] == 'http';
   var isHttps = matches[1] == 'https';
   var isWebSocket = matches[1] == 'ws';
   var isWebSocketSSL = matches[1] == 'wss';
   if (!matches[3])
@@ -325,25 +328,22 @@ function FindProxyForURL(url, host)
   if (isWebSocket)
     matches[1] = 'http';
   if (isWebSocketSSL)
     matches[1] = 'https';
 
   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
   if (origins.indexOf(origin) < 0)
     return 'DIRECT';
-  if (isHttp)
-    return 'PROXY %(remote)s:%(httpport)s';
-  if (isHttps || isWebSocket || isWebSocketSSL)
-    return 'PROXY %(remote)s:%(sslport)s';
+  if (isHttp || isHttps || isWebSocket || isWebSocketSSL)
+    return 'PROXY %(remote)s:%(port)s';
   return 'DIRECT';
 }""" % { "origins": origins,
          "remote":  webServer,
-         "httpport":httpPort,
-         "sslport": sslPort }
+         "port": port }
         pacURL = "".join(pacURL.splitlines())
 
         prefs.append(("network.proxy.type", 2))
         prefs.append(("network.proxy.autoconfig_url", pacURL))
 
         return prefs
 
     def clean_db(self):
--- a/testing/mozbase/mozprofile/mozprofile/profile.py
+++ b/testing/mozbase/mozprofile/mozprofile/profile.py
@@ -22,17 +22,17 @@ class Profile(object):
     sets preferences and handles cleanup."""
 
     def __init__(self,
                  profile=None, # Path to the profile
                  addons=None,  # String of one or list of addons to install
                  addon_manifests=None,  # Manifest for addons, see http://ahal.ca/blog/2011/bulk-installing-fx-addons/
                  preferences=None, # Dictionary or class of preferences
                  locations=None, # locations to proxy
-                 proxy=False, # setup a proxy
+                 proxy=None, # setup a proxy - dict of server-loc,server-port,ssl-port
                  restore=True # If true remove all installed addons preferences when cleaning up
                  ):
 
         # if true, remove installed addons/prefs afterwards
         self.restore = restore
 
         # prefs files written to
         self.written_prefs = set()
--- a/testing/mozbase/mozprofile/tests/permissions.py
+++ b/testing/mozbase/mozprofile/tests/permissions.py
@@ -76,38 +76,39 @@ http://127.0.0.1:8888           privileg
         cur.execute("select * from sqlite_master where type='table'")
         entries = cur.fetchall()
         self.assertEqual(len(entries), 0)
         
     def test_nw_prefs(self):
         perms = Permissions(self.profile_dir, self.locations_file.name)
 
         prefs, user_prefs = perms.network_prefs(False)
+
         self.assertEqual(len(user_prefs), 0)
         self.assertEqual(len(prefs), 6)
 
         self.assertEqual(prefs[0], ('capability.principal.codebase.p1.granted',
                                     'UniversalXPConnect'))
         self.assertEqual(prefs[1], ('capability.principal.codebase.p1.id',
-                                    'http://mochi.test'))
+                                    'http://mochi.test:8888'))
         self.assertEqual(prefs[2], ('capability.principal.codebase.p1.subjectName', ''))
 
         self.assertEqual(prefs[3], ('capability.principal.codebase.p2.granted',
                                     'UniversalXPConnect'))
         self.assertEqual(prefs[4], ('capability.principal.codebase.p2.id',
-                                    'http://127.0.0.1'))
+                                    'http://127.0.0.1:8888'))
         self.assertEqual(prefs[5], ('capability.principal.codebase.p2.subjectName', ''))
 
 
         prefs, user_prefs = perms.network_prefs(True)
         self.assertEqual(len(user_prefs), 2)
         self.assertEqual(user_prefs[0], ('network.proxy.type', 2))
         self.assertEqual(user_prefs[1][0], 'network.proxy.autoconfig_url')
 
-        origins_decl = "var origins = ['http://127.0.0.1:80', 'http://127.0.0.1:8888'];"
+        origins_decl = "var origins = ['http://mochi.test:8888', 'http://127.0.0.1:80', 'http://127.0.0.1:8888'];"
         self.assertTrue(origins_decl in user_prefs[1][1])
 
-        proxy_check = "if (isHttp)    return 'PROXY mochi.test:8888';  if (isHttps || isWebSocket || isWebSocketSSL)    return 'PROXY mochi.test:443';"
+        proxy_check = "if (isHttp || isHttps || isWebSocket || isWebSocketSSL)    return 'PROXY mochi.test:8888';"
         self.assertTrue(proxy_check in user_prefs[1][1])
 
 
 if __name__ == '__main__':
     unittest.main()
--- a/testing/mozbase/mozprofile/tests/server_locations.py
+++ b/testing/mozbase/mozprofile/tests/server_locations.py
@@ -77,18 +77,21 @@ http://example.org:80           privileg
         self.assertEqual(len(locations), 7)
         self.compare_location(i.next(), 'http', 'mozilla.org', '80',
                               ['privileged'])
 
         # test some errors
         self.assertRaises(MultiplePrimaryLocationsError, locations.add_host,
                           'primary.test', options='primary')
 
-        self.assertRaises(DuplicateLocationError, locations.add_host,
-                          '127.0.0.1')
+        # We no longer throw these DuplicateLocation Error
+        try:
+            locations.add_host('127.0.0.1')
+        except DuplicateLocationError:
+            self.assertTrue(False, "Should no longer throw DuplicateLocationError")
 
         self.assertRaises(BadPortLocationError, locations.add_host, '127.0.0.1',
                           port='abc')
 
         # test some errors in locations file
         f = self.create_temp_file(self.locations_no_primary)
 
         exc = None
--- a/testing/mozbase/mozrunner/mozrunner/runner.py
+++ b/testing/mozbase/mozrunner/mozrunner/runner.py
@@ -9,17 +9,16 @@
 import mozinfo
 import optparse
 import os
 import platform
 import subprocess
 import sys
 import ConfigParser
 
-from threading import Thread
 from utils import get_metadata_from_egg
 from utils import findInPath
 from mozprofile import *
 from mozprocess.processhandler import ProcessHandler
 
 if mozinfo.isMac:
     from plistlib import readPlist
 
@@ -146,57 +145,68 @@ class Runner(object):
                 except:
                     repository['%s_%s' % (file, id)] = None
 
         return repository
 
     def is_running(self):
         return self.process_handler is not None
 
-    def start(self, debug_args=None, interactive=False):
+    def start(self, debug_args=None, interactive=False, timeout=None, outputTimeout=None):
         """
         Run self.command in the proper environment.
         - debug_args: arguments for the debugger
+        - interactive: uses subprocess.Popen directly
+        - read_output: sends program output to stdout [default=False]
+        - timeout: see process_handler.waitForFinish
+        - outputTimeout: see process_handler.waitForFinish
         """
 
         # ensure you are stopped
         self.stop()
 
         # ensure the profile exists
         if not self.profile.exists():
             self.profile.reset()
             assert self.profile.exists(), "%s : failure to reset profile" % self.__class__.__name__
 
         cmd = self._wrap_command(self.command+self.cmdargs)
 
         # attach a debugger, if specified
         if debug_args:
             cmd = list(debug_args) + cmd
 
-        #
         if interactive:
             self.process_handler = subprocess.Popen(cmd, env=self.env)
             # TODO: other arguments
         else:
             # this run uses the managed processhandler
             self.process_handler = self.process_class(cmd, env=self.env, **self.kp_kwargs)
             self.process_handler.run()
 
-            # Spin a thread to handle reading the output
-            self.outThread = OutputThread(self.process_handler)
-            self.outThread.start()
+            # start processing output from the process
+            self.process_handler.processOutput(timeout, outputTimeout)
 
-    def wait(self, timeout=None, outputTimeout=None):
-        """Wait for the app to exit."""
+    def wait(self, timeout=None):
+        """
+        Wait for the app to exit.
+
+        If timeout is not None, will return after timeout seconds.
+        Use is_running() to determine whether or not a timeout occured.
+        Timeout is ignored if interactive was set to True.
+        """
         if self.process_handler is None:
             return
         if isinstance(self.process_handler, subprocess.Popen):
             self.process_handler.wait()
         else:
-            self.process_handler.waitForFinish(timeout=timeout, outputTimeout=outputTimeout)
+            self.process_handler.waitForFinish(timeout)
+            if not getattr(self.process_handler.proc, 'returncode', False):
+                # waitForFinish timed out
+                return
         self.process_handler = None
 
     def stop(self):
         """Kill the app"""
         if self.process_handler is None:
             return
         self.process_handler.kill()
         self.process_handler = None
@@ -242,23 +252,16 @@ class FirefoxRunner(Runner):
 
 class ThunderbirdRunner(Runner):
     """Specialized Runner subclass for running Thunderbird"""
     profile_class = ThunderbirdProfile
 
 runners = {'firefox': FirefoxRunner,
            'thunderbird': ThunderbirdRunner}
 
-class OutputThread(Thread):
-    def __init__(self, prochandler):
-        Thread.__init__(self)
-        self.ph = prochandler
-    def run(self):
-        self.ph.waitForFinish()
-
 class CLI(MozProfileCLI):
     """Command line interface."""
 
     module = "mozrunner"
 
     def __init__(self, args=sys.argv[1:]):
         """
         Setup command line parser and parse arguments
--- a/testing/mozbase/test-manifest.ini
+++ b/testing/mozbase/test-manifest.ini
@@ -3,12 +3,13 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # mozbase test manifest, in the format of
 # https://github.com/mozilla/mozbase/blob/master/manifestdestiny/README.txt
 
 # run with
 # https://github.com/mozilla/mozbase/blob/master/test.py
 
+[include:manifestdestiny/tests/manifest.ini]
 [include:mozprocess/tests/manifest.ini]
 [include:mozprofile/tests/manifest.ini]
 [include:mozhttpd/tests/manifest.ini]
 [include:mozdevice/tests/manifest.ini]
--- a/testing/peptest/MANIFEST.in
+++ b/testing/peptest/MANIFEST.in
@@ -1,5 +1,1 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-recursive-include peptest/extension *
\ No newline at end of file
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

recursive-include peptest/extension *
\ No newline at end of file
--- a/testing/peptest/peptest/extension/build.xml
+++ b/testing/peptest/peptest/extension/build.xml
@@ -1,14 +1,13 @@
 <?xml version="1.0"?>
 <!-- This Source Code Form is subject to the terms of the Mozilla Public
    - License, v. 2.0. If a copy of the MPL was not distributed with this
    - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
 
-
 <!--
 build.xml adapted from Shawn Wilsher's rtse
 (http://shawnwilsher.com/extensions/rtse/)
  -->
 
 <project name="pep" default="pepxpi">
   <tstamp>
     <format property="build.number" pattern="yyyyMMdd" offset="-1" unit="hour"/>
--- a/testing/peptest/peptest/extension/resource/mozmill/README.md
+++ b/testing/peptest/peptest/extension/resource/mozmill/README.md
@@ -1,9 +1,9 @@
 <!-- This Source Code Form is subject to the terms of the Mozilla Public
    - License, v. 2.0. If a copy of the MPL was not distributed with this
    - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
 
 These folders are pulled from:
 https://github.com/mozilla/mozmill/tree/master/mozmill/mozmill/extension/resource
 
-To update them, simply checkout the mozmill repo at https://github.com/mozautomation/mozmill, 
+To update them, simply checkout the mozmill repo at https://github.com/mozilla/mozmill, 
 then copy and paste the 'driver' and 'stdlib' folders to this location. 
--- a/testing/peptest/peptest/pepprocess.py
+++ b/testing/peptest/peptest/pepprocess.py
@@ -57,43 +57,40 @@ class PepProcess(ProcessHandler):
                 self.logger.testStart(results.currentTest)
             elif level == 'TEST-END':
                 metric = results.get_metric(results.currentTest)
                 if len(tokens) > 4:
                     threshold = float(tokens[4].rstrip())
                 else:
                     threshold = 0.0
 
-                msg = results.currentTest \
-                      + ' | fail threshold: ' + str(threshold)
+                msg = '%s | fail threshold: %s' % (results.currentTest, threshold)
                 if metric > threshold:
-                    msg += ' < metric: ' + str(metric)
+                    msg += ' < metric: %s' % metric
                     self.logger.testFail(msg)
                 else:
-                    msg += ' >= metric: ' + str(metric)
+                    msg += ' >= metric: %s' % metric
                     self.logger.testPass(msg)
 
-                self.logger.testEnd(
-                        results.currentTest +
-                        ' | finished in: ' + tokens[3].rstrip() + ' ms')
+                self.logger.testEnd('%s | finished in: %s ms' %
+                        (results.currentTest, tokens[3].rstrip()))
                 results.currentTest = None
             elif level == 'ACTION-START':
                 results.currentAction = tokens[3].rstrip()
-                self.logger.debug(level + ' | ' + results.currentAction)
+                self.logger.debug('%s | %s' % (level, results.currentAction))
             elif level == 'ACTION-END':
-                self.logger.debug(level + ' | ' + results.currentAction)
+                self.logger.debug('%s | %s' % (level, results.currentAction))
                 results.currentAction = None
             elif level in ['DEBUG', 'INFO', 'WARNING', 'ERROR']:
                 line = line[len('PEP ' + level)+1:]
                 getattr(self.logger, level.lower())(line.rstrip())
                 if level == 'ERROR':
                     results.fails[str(results.currentTest)].append(0)
             else:
                 line = line[len('PEP'):]
                 self.logger.debug(line.rstrip())
         elif tokens[0] == 'MOZ_EVENT_TRACE' and results.currentAction is not None:
             # The output is generated from EventTracer
             # Format is 'MOZ_EVENT_TRACE sample <TIMESTAMP> <VALUE>
             # <VALUE> is the unresponsive time in ms
-            self.logger.warning(
-                    results.currentTest + ' | ' + results.currentAction +
-                    ' | unresponsive time: ' + tokens[3].rstrip() + ' ms')
+            self.logger.warning('%s | %s | unresponsive time: %s ms' %
+                (results.currentTest, results.currentAction, tokens[3].rstrip()))
             results.fails[results.currentTest].append(int(tokens[3].rstrip()))
--- a/testing/peptest/peptest/runpeptests.py
+++ b/testing/peptest/peptest/runpeptests.py
@@ -44,17 +44,18 @@ class Peptest():
         # create the profile
         enable_proxy = False
         locations = ServerLocations()
         if self.options.proxyLocations:
             if not self.options.serverPath:
                 self.logger.warning('Can\'t set up proxy without server path')
             else:
                 enable_proxy = True
-                locations.read(self.options.proxyLocations, False)
+                for proxyLocation in self.options.proxyLocations:
+                    locations.read(proxyLocation, False)
                 locations.add_host(host='127.0.0.1',
                                    port=self.options.serverPort,
                                    options='primary,privileged')
 
         self.profile = self.profile_class(profile=self.options.profilePath,
                                           addons=[os.path.join(here, 'extension')],
                                           locations=locations,
                                           proxy=enable_proxy)
@@ -108,18 +109,18 @@ class Peptest():
                                         cmdargs=cmdargs,
                                         env=env,
                                         process_class=PepProcess)
 
     def start(self):
         self.logger.debug('Starting Peptest')
 
         # start firefox
-        self.runner.start()
-        self.runner.wait(outputTimeout=self.options.timeout)
+        self.runner.start(outputTimeout=self.options.timeout)
+        self.runner.wait()
         crashed = self.checkForCrashes(results.currentTest)
         self.stop()
 
         if crashed or results.has_fails():
             return 1
         return 0
 
     def runServer(self):
@@ -288,21 +289,21 @@ class PeptestOptions(OptionParser):
 
         self.add_option("-p", "--profile-path", action="store",
                         type="string", dest="profilePath",
                         default=None,
                         help="path to the profile to use. "
                              "If none specified, a temporary profile is created")
 
         self.add_option("--proxy",
-                        action="store", type="string", dest="proxyLocations",
+                        action="append", type="string", dest="proxyLocations",
                         default=None,
-                        help="path to a server-location file specifying "
-                             "domains to proxy. --server-path must also be "
-                             "specified.")
+                        help="a list of paths to server-location files specifying "
+                             "domains to proxy (set with multiple --proxy agruments). "
+                             "--server-path must also be specified.")
 
         self.add_option("--proxy-host-dirs",
                         action="store_true", dest="proxyHostDirs",
                         default=False,
                         help="proxied requests are served from directories "
                              "named by requested host. --proxy must also be "
                              "specified.")