author | Jeff Hammel <jhammel@mozilla.com> |
Fri, 20 Jul 2012 20:19:38 -0400 | |
changeset 100000 | b5d3474e21bffc4a6934c81afe358562fe57a296 |
parent 99999 | e4dc553bbf42c8ac7ebe9830ed0b8991b726ccb4 |
child 100001 | 1be98690ab783d815dddada97efe6602aab660a5 |
push id | 12291 |
push user | ryanvm@gmail.com |
push date | Sat, 21 Jul 2012 00:19:38 +0000 |
treeherder | mozilla-inbound@045c11dd41a6 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahalberstadt |
bugs | 775127 |
milestone | 17.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py +++ b/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py @@ -691,18 +691,18 @@ class TestManifest(ManifestParser): """ apply logic to manifests; this is your integration layer :) specific harnesses may subclass from this if they need more logic """ def filter(self, values, tests): """ filter on a specific list tag, e.g.: - run-if.os = win linux - skip-if.os = mac + run-if = os == win linux + skip-if = os == mac """ # tags: run_tag = 'run-if' skip_tag = 'skip-if' fail_tag = 'fail-if' # loop over test
--- a/testing/mozbase/manifestdestiny/setup.py +++ b/testing/mozbase/manifestdestiny/setup.py @@ -8,17 +8,17 @@ # and result in a useless package from setuptools import setup, find_packages import sys import os here = os.path.dirname(os.path.abspath(__file__)) try: - filename = os.path.join(here, 'README.txt') + filename = os.path.join(here, 'README.md') description = file(filename).read() except: description = '' PACKAGE_NAME = "ManifestDestiny" PACKAGE_VERSION = "0.5.4" setup(name=PACKAGE_NAME,
--- a/testing/mozbase/manifestdestiny/tests/test.py +++ b/testing/mozbase/manifestdestiny/tests/test.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """tests for ManifestDestiny""" import doctest import os @@ -17,17 +19,17 @@ def run_tests(raise_on_error=False, repo # doctest arguments directory = os.path.dirname(os.path.abspath(__file__)) extraglobs = {} doctest_args = dict(extraglobs=extraglobs, module_relative=False, raise_on_error=raise_on_error) if report_first: doctest_args['optionflags'] = doctest.REPORT_ONLY_FIRST_FAILURE - + # gather tests directory = os.path.dirname(os.path.abspath(__file__)) tests = [ test for test in os.listdir(directory) if test.endswith('.txt') and test.startswith('test_')] os.chdir(directory) # run the tests for test in tests: @@ -69,11 +71,11 @@ def main(args=sys.argv[1:]): if failed: sys.exit(1) # error if not quiet: # print results print "manifestparser.py: All tests pass!" for test in sorted(results.keys()): result = results[test] print "%s: failed=%s, attempted=%s" % (test, result[0], result[1]) - + if __name__ == '__main__': main()
--- a/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py +++ b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py @@ -12,16 +12,17 @@ import threading import posixpath import socket import sys import os import urllib import urlparse import re import iface +import time from SocketServer import ThreadingMixIn class EasyServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): allow_reuse_address = True acceptable_errors = (errno.EPIPE, errno.ECONNABORTED) def handle_error(self, request, client_address): error = sys.exc_value @@ -58,19 +59,26 @@ class Request(object): else: self.body = None class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): docroot = os.getcwd() # current working directory at time of import proxy_host_dirs = False + request_log = [] + log_requests = False request = None def _try_handler(self, method): + if self.log_requests: + self.request_log.append({ 'method': method, + 'path': self.request.path, + 'time': time.time() }) + handlers = [handler for handler in self.urlhandlers if handler['method'] == method] for handler in handlers: m = re.match(handler['path'], self.request.path) if m: (response_code, headerdict, data) = \ handler['function'](self.request, *m.groups()) self.send_response(response_code) @@ -176,30 +184,34 @@ class MozHttpd(object): For example, the request "GET http://foo.bar/dir/file.html" would (assuming no handlers match) serve <docroot>/dir/file.html if proxy_host_dirs is False, or <docroot>/foo.bar/dir/file.html if it is True. """ def __init__(self, host="127.0.0.1", port=8888, docroot=None, - urlhandlers=None, proxy_host_dirs=False): + urlhandlers=None, proxy_host_dirs=False, log_requests=False): self.host = host self.port = int(port) self.docroot = docroot if not urlhandlers and not docroot: self.docroot = os.getcwd() self.proxy_host_dirs = proxy_host_dirs self.httpd = None self.urlhandlers = urlhandlers or [] + self.log_requests = log_requests + self.request_log = [] class RequestHandlerInstance(RequestHandler): docroot = self.docroot urlhandlers = self.urlhandlers proxy_host_dirs = self.proxy_host_dirs + request_log = self.request_log + log_requests = self.log_requests self.handler_class = RequestHandlerInstance def start(self, block=False): """ Start the server. If block is True, the call will not return. If block is False, the server will be started on a separate thread that can be terminated by a call to .stop() @@ -239,18 +251,16 @@ def main(args=sys.argv[1:]): dest='external_ip', default=False, help="find and use external ip for host") parser.add_option('-d', '--docroot', dest='docroot', default=os.getcwd(), help="directory to serve files from [DEFAULT: %default]") options, args = parser.parse_args(args) if args: parser.error("mozhttpd does not take any arguments") - parser.print_help() - parser.exit() if options.external_ip: host = iface.get_lan_ip() else: host = options.host # create the server server = MozHttpd(host=host, port=options.port, docroot=options.docroot)
--- a/testing/mozbase/mozhttpd/setup.py +++ b/testing/mozbase/mozhttpd/setup.py @@ -6,22 +6,22 @@ import os from setuptools import setup, find_packages try: here = os.path.dirname(os.path.abspath(__file__)) description = file(os.path.join(here, 'README.md')).read() except IOError: description = None -version = '0.3' +PACKAGE_VERSION = '0.3' deps = [] setup(name='mozhttpd', - version=version, + version=PACKAGE_VERSION, description="basic python webserver, tested with talos", long_description=description, classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Joel Maher', author_email='tools@lists.mozilla.org', url='https://github.com/mozilla/mozbase/tree/master/mozhttpd', license='MPL',
--- a/testing/mozbase/mozhttpd/tests/api.py +++ b/testing/mozbase/mozhttpd/tests/api.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozhttpd import urllib2 import os import unittest @@ -187,16 +189,22 @@ class ApiTest(unittest.TestCase): except AttributeError: pass # python 2.4 self.assertTrue('Directory listing for' in f.read()) # Make sure API methods still work self.try_get(server_port, '') self.try_get(server_port, '?foo=bar') +class ProxyTest(unittest.TestCase): + + def tearDown(self): + # reset proxy opener in case it changed + urllib2.install_opener(None) + def test_proxy(self): docroot = tempfile.mkdtemp() hosts = ('mozilla.com', 'mozilla.org') unproxied_host = 'notmozilla.org' def url(host): return 'http://%s/' % host index_filename = 'index.html' def index_contents(host): return '%s index' % host
--- a/testing/mozbase/mozhttpd/tests/filelisting.py +++ b/testing/mozbase/mozhttpd/tests/filelisting.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozhttpd import urllib2 import os import unittest
--- a/testing/mozbase/mozhttpd/tests/manifest.ini +++ b/testing/mozbase/mozhttpd/tests/manifest.ini @@ -1,2 +1,3 @@ [filelisting.py] [api.py] +[requestlog.py]
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/requestlog.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozhttpd +import urllib2 +import os +import unittest +import time + +here = os.path.dirname(os.path.abspath(__file__)) + +class RequestLogTest(unittest.TestCase): + + def check_logging(self, log_requests=False): + filelist = os.listdir(here) + + httpd = mozhttpd.MozHttpd(port=0, docroot=here, log_requests=log_requests) + httpd.start(block=False) + url = "http://%s:%s/" % ('127.0.0.1', httpd.httpd.server_port) + f = urllib2.urlopen(url) + data = f.read() + + return httpd.request_log + + def test_logging_enabled(self): + request_log = self.check_logging(log_requests=True) + + self.assertEqual(len(request_log), 1) + + log_entry = request_log[0] + self.assertEqual(log_entry['method'], 'GET') + self.assertEqual(log_entry['path'], '/') + self.assertEqual(type(log_entry['time']), float) + + def test_logging_disabled(self): + request_log = self.check_logging(log_requests=False) + + self.assertEqual(len(request_log), 0) + +if __name__ == '__main__': + unittest.main()
--- a/testing/mozbase/mozinfo/setup.py +++ b/testing/mozbase/mozinfo/setup.py @@ -1,34 +1,34 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os from setuptools import setup, find_packages -version = '0.3.3' +PACKAGE_VERSION = '0.3.3' # get documentation from the README try: here = os.path.dirname(os.path.abspath(__file__)) description = file(os.path.join(here, 'README.md')).read() except (OSError, IOError): description = '' # dependencies deps = [] try: import json except ImportError: deps = ['simplejson'] setup(name='mozinfo', - version=version, + version=PACKAGE_VERSION, description="file for interface to transform introspected system information to a format pallatable to Mozilla", long_description=description, classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Jeff Hammel', author_email='jhammel@mozilla.com', url='https://wiki.mozilla.org/Auto-tools', license='MPL',
--- a/testing/mozbase/mozinstall/README.md +++ b/testing/mozbase/mozinstall/README.md @@ -1,13 +1,13 @@ [Mozinstall](https://github.com/mozilla/mozbase/tree/master/mozinstall) is a python package for installing and uninstalling Mozilla applications on various platforms. -For example, depending on the platform, Firefox can be distributed as a +For example, depending on the platform, Firefox can be distributed as a zip, tar.bz2, exe, or dmg file or cloned from a repository. Mozinstall takes the hassle out of extracting and/or running these files and for convenience returns the full path to the install directory. In the case that mozinstall is invoked from the command line, the binary path will be printed to stdout. To remove an installed application the uninstaller can be used. It requires the installation path of the application and will remove all the installed files. On Windows the uninstaller will be tried first.
--- a/testing/mozbase/mozinstall/mozinstall/mozinstall.py +++ b/testing/mozbase/mozinstall/mozinstall/mozinstall.py @@ -213,33 +213,30 @@ def _extract(src, dest): Arguments: src -- archive which has to be extracted dest -- the path to extract to """ if zipfile.is_zipfile(src): bundle = zipfile.ZipFile(src) + + # FIXME: replace with zip.extractall() when we require python 2.6 namelist = bundle.namelist() - - if hasattr(bundle, 'extractall'): - # zipfile.extractall doesn't exist in Python 2.5 - bundle.extractall(path=dest) - else: - for name in namelist: - filename = os.path.realpath(os.path.join(dest, name)) - if name.endswith('/'): - os.makedirs(filename) - else: - path = os.path.dirname(filename) - if not os.path.isdir(path): - os.makedirs(path) - dest = open(filename, 'wb') - dest.write(bundle.read(name)) - dest.close() + for name in bundle.namelist(): + filename = os.path.realpath(os.path.join(dest, name)) + if name.endswith('/'): + os.makedirs(filename) + else: + path = os.path.dirname(filename) + if not os.path.isdir(path): + os.makedirs(path) + _dest = open(filename, 'wb') + _dest.write(bundle.read(name)) + _dest.close() elif tarfile.is_tarfile(src): bundle = tarfile.open(src) namelist = bundle.getnames() if hasattr(bundle, 'extractall'): # tarfile.extractall doesn't exist in Python 2.4 bundle.extractall(path=dest)
--- a/testing/mozbase/mozinstall/setup.py +++ b/testing/mozbase/mozinstall/setup.py @@ -6,22 +6,22 @@ import os from setuptools import setup, find_packages try: here = os.path.dirname(os.path.abspath(__file__)) description = file(os.path.join(here, 'README.md')).read() except IOError: description = None -version = '1.1' +PACKAGE_VERSION = '1.2' deps = ['mozinfo==0.3.3'] setup(name='mozInstall', - version=version, + version=PACKAGE_VERSION, description="This is a utility package for installing and uninstalling " "Mozilla applications on various platforms.", long_description=description, # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=['Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English',
--- a/testing/mozbase/mozlog/README.md +++ b/testing/mozbase/mozlog/README.md @@ -1,17 +1,17 @@ [Mozlog](https://github.com/mozilla/mozbase/tree/master/mozlog) -is a python package intended to simplify and standardize logs in the Mozilla universe. -It wraps around python's [logging](http://docs.python.org/library/logging.html) +is a python package intended to simplify and standardize logs in the Mozilla universe. +It wraps around python's [logging](http://docs.python.org/library/logging.html) module and adds some additional functionality. # Usage -Import mozlog instead of [logging](http://docs.python.org/library/logging.html) -(all functionality in the logging module is also available from the mozlog module). +Import mozlog instead of [logging](http://docs.python.org/library/logging.html) +(all functionality in the logging module is also available from the mozlog module). To get a logger, call mozlog.getLogger passing in a name and the path to a log file. If no log file is specified, the logger will log to stdout. import mozlog logger = mozlog.getLogger('LOG_NAME', 'log_file_path') logger.setLevel(mozlog.DEBUG) logger.info('foo') logger.testPass('bar')
--- a/testing/mozbase/mozprocess/mozprocess/processhandler.py +++ b/testing/mozbase/mozprocess/mozprocess/processhandler.py @@ -9,24 +9,28 @@ import select import signal import subprocess import sys import threading import time import traceback from Queue import Queue from datetime import datetime, timedelta +__all__ = ['ProcessHandlerMixin', 'ProcessHandler'] -__all__ = ['ProcessHandlerMixin', 'ProcessHandler'] +# Set the MOZPROCESS_DEBUG environment variable to 1 to see some debugging output +MOZPROCESS_DEBUG = os.getenv("MOZPROCESS_DEBUG") if mozinfo.isWin: import ctypes, ctypes.wintypes, msvcrt - from ctypes import sizeof, addressof, c_ulong, byref, POINTER, WinError + from ctypes import sizeof, addressof, c_ulong, byref, POINTER, WinError, c_longlong import winprocess - from qijo import JobObjectAssociateCompletionPortInformation, JOBOBJECT_ASSOCIATE_COMPLETION_PORT + from qijo import JobObjectAssociateCompletionPortInformation,\ + JOBOBJECT_ASSOCIATE_COMPLETION_PORT, JobObjectExtendedLimitInformation,\ + JOBOBJECT_BASIC_LIMIT_INFORMATION, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, IO_COUNTERS class ProcessHandlerMixin(object): """Class which represents a process to be executed.""" class Process(subprocess.Popen): """ Represents our view of a subprocess. It adds a kill() method which allows it to be stopped explicitly. @@ -206,16 +210,45 @@ class ProcessHandlerMixin(object): joacp = JOBOBJECT_ASSOCIATE_COMPLETION_PORT(winprocess.COMPKEY_JOBOBJECT, self._io_port) winprocess.SetInformationJobObject(self._job, JobObjectAssociateCompletionPortInformation, addressof(joacp), sizeof(joacp) ) + # Allow subprocesses to break away from us - necessary for + # flash with protected mode + jbli = JOBOBJECT_BASIC_LIMIT_INFORMATION( + c_longlong(0), # per process time limit (ignored) + c_longlong(0), # per job user time limit (ignored) + winprocess.JOB_OBJECT_LIMIT_BREAKAWAY_OK, + 0, # min working set (ignored) + 0, # max working set (ignored) + 0, # active process limit (ignored) + None, # affinity (ignored) + 0, # Priority class (ignored) + 0, # Scheduling class (ignored) + ) + + iocntr = IO_COUNTERS() + jeli = JOBOBJECT_EXTENDED_LIMIT_INFORMATION( + jbli, # basic limit info struct + iocntr, # io_counters (ignored) + 0, # process mem limit (ignored) + 0, # job mem limit (ignored) + 0, # peak process limit (ignored) + 0) # peak job limit (ignored) + + winprocess.SetInformationJobObject(self._job, + JobObjectExtendedLimitInformation, + addressof(jeli), + sizeof(jeli) + ) + # Assign the job object to the process winprocess.AssignProcessToJobObject(self._job, int(hp)) # It's overkill, but we use Queue to signal between threads # because it handles errors more gracefully than event or condition. self._process_events = Queue() # Spin up our thread for managing the IO Completion Port @@ -250,16 +283,19 @@ falling back to not using job objects fo except KeyboardInterrupt: raise KeyboardInterrupt def _poll_iocompletion_port(self): # Watch the IO Completion port for status self._spawned_procs = {} countdowntokill = 0 + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC Self.pid value is: %s" % self.pid + while True: msgid = c_ulong(0) compkey = c_ulong(0) pid = c_ulong(0) portstatus = winprocess.GetQueuedCompletionStatus(self._io_port, byref(msgid), byref(compkey), byref(pid), @@ -291,50 +327,62 @@ falling back to not using job objects fo # Timeouts are expected, just keep on polling continue else: print >> sys.stderr, "Error Code %s trying to query IO Completion Port, exiting" % errcode raise WinError(errcode) break if compkey.value == winprocess.COMPKEY_TERMINATE.value: + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC compkeyterminate detected" # Then we're done break # Check the status of the IO Port and do things based on it if compkey.value == winprocess.COMPKEY_JOBOBJECT.value: if msgid.value == winprocess.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO: # No processes left, time to shut down # Signal anyone waiting on us that it is safe to shut down + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC job object msg active processes zero" self._process_events.put({self.pid: 'FINISHED'}) break elif msgid.value == winprocess.JOB_OBJECT_MSG_NEW_PROCESS: # New Process started # Add the child proc to our list in case our parent flakes out on us # without killing everything. if pid.value != self.pid: self._spawned_procs[pid.value] = 1 + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC new process detected with pid value: %s" % pid.value elif msgid.value == winprocess.JOB_OBJECT_MSG_EXIT_PROCESS: + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC process id %s exited normally" % pid.value # One process exited normally if pid.value == self.pid and len(self._spawned_procs) > 0: # Parent process dying, start countdown timer countdowntokill = datetime.now() elif pid.value in self._spawned_procs: # Child Process died remove from list del(self._spawned_procs[pid.value]) elif msgid.value == winprocess.JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS: # One process existed abnormally + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC process id %s existed abnormally" % pid.value if pid.value == self.pid and len(self._spawned_procs) > 0: # Parent process dying, start countdown timer countdowntokill = datetime.now() elif pid.value in self._spawned_procs: # Child Process died remove from list del self._spawned_procs[pid.value] else: # We don't care about anything else + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC We got a message %s" % msgid.value pass def _wait(self): # First, check to see if the process is still running if self._handle: self.returncode = winprocess.GetExitCodeProcess(self._handle) else: @@ -372,16 +420,18 @@ falling back to not using job objects fo if err is not None: raise OSError(err) else: # Not managing with job objects, so all we can reasonably do # is call waitforsingleobject and hope for the best + if MOZPROCESS_DEBUG: + print "DBG::MOZPROC NOT USING JOB OBJECTS!!!" # First, make sure we have not already ended if self.returncode != winprocess.STILL_ACTIVE: self._cleanup() return self.returncode rc = None if self._handle: rc = winprocess.WaitForSingleObject(self._handle, -1)
--- a/testing/mozbase/mozprocess/mozprocess/wpk.py +++ b/testing/mozbase/mozprocess/mozprocess/wpk.py @@ -47,34 +47,8 @@ def get_pids(process_name): return pids def kill_pid(pid): process = windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid) if process: windll.kernel32.TerminateProcess(process, 0) windll.kernel32.CloseHandle(process) - -if __name__ == '__main__': - import subprocess - import time - - # This test just opens a new notepad instance and kills it. - - name = 'notepad' - - old_pids = set(get_pids(name)) - subprocess.Popen([name]) - time.sleep(0.25) - new_pids = set(get_pids(name)).difference(old_pids) - - if len(new_pids) != 1: - raise Exception('%s was not opened or get_pids() is ' - 'malfunctioning' % name) - - kill_pid(tuple(new_pids)[0]) - - newest_pids = set(get_pids(name)).difference(old_pids) - - if len(newest_pids) != 0: - raise Exception('kill_pid() is malfunctioning') - - print "Test passed."
--- a/testing/mozbase/mozprocess/setup.py +++ b/testing/mozbase/mozprocess/setup.py @@ -1,16 +1,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os from setuptools import setup, find_packages -PACKAGE_VERSION = '0.3' +PACKAGE_VERSION = '0.4' # take description from README here = os.path.dirname(os.path.abspath(__file__)) try: description = file(os.path.join(here, 'README.md')).read() except (OSError, IOError): description = ''
--- a/testing/mozbase/mozprocess/tests/mozprocess1.py +++ b/testing/mozbase/mozprocess/tests/mozprocess1.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os import subprocess import sys import unittest @@ -15,16 +17,19 @@ here = os.path.dirname(os.path.abspath(_ def make_proclaunch(aDir): """ Makes the proclaunch executable. Params: aDir - the directory in which to issue the make commands Returns: the path to the proclaunch executable that is generated """ + # Ideally make should take care of this, but since it doesn't - on windows, + # anyway, let's just call out both targets explicitly. + p = subprocess.call(["make", "-C", "iniparser"], cwd=aDir) p = subprocess.call(["make"], cwd=aDir) if sys.platform == "win32": exepath = os.path.join(aDir, "proclaunch.exe") else: exepath = os.path.join(aDir, "proclaunch") return exepath def check_for_process(processName):
--- a/testing/mozbase/mozprocess/tests/mozprocess2.py +++ b/testing/mozbase/mozprocess/tests/mozprocess2.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os import subprocess import sys import unittest @@ -20,16 +22,19 @@ here = os.path.dirname(os.path.abspath(_ def make_proclaunch(aDir): """ Makes the proclaunch executable. Params: aDir - the directory in which to issue the make commands Returns: the path to the proclaunch executable that is generated """ + # Ideally make should take care of this, but since it doesn't - on windows, + # anyway, let's just call out both targets explicitly. + p = subprocess.call(["make", "-C", "iniparser"], cwd=aDir) p = subprocess.call(["make"], cwd=aDir) if sys.platform == "win32": exepath = os.path.join(aDir, "proclaunch.exe") else: exepath = os.path.join(aDir, "proclaunch") return exepath def check_for_process(processName):
--- a/testing/mozbase/mozprofile/mozprofile/prefs.py +++ b/testing/mozbase/mozprofile/mozprofile/prefs.py @@ -203,11 +203,8 @@ class Preferences(object): print >> f, pref_string % (key, 'false') elif isinstance(value, basestring): print >> f, pref_string % (key, repr(string(value))) else: print >> f, pref_string % (key, value) # should be numeric! if isinstance(_file, basestring): f.close() - -if __name__ == '__main__': - pass
--- a/testing/mozbase/mozprofile/mozprofile/profile.py +++ b/testing/mozbase/mozprofile/mozprofile/profile.py @@ -1,16 +1,18 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. __all__ = ['Profile', 'FirefoxProfile', 'ThunderbirdProfile'] import os +import time import tempfile +import uuid from addons import AddonManager from permissions import Permissions from shutil import rmtree try: import simplejson except ImportError: import json as simplejson @@ -31,17 +33,18 @@ class Profile(object): # if true, remove installed addons/prefs afterwards self.restore = restore # prefs files written to self.written_prefs = set() # our magic markers - self.delimeters = ('#MozRunner Prefs Start', '#MozRunner Prefs End') + nonce = '%s %s' % (str(time.time()), uuid.uuid4()) + self.delimeters = ('#MozRunner Prefs Start %s' % nonce,'#MozRunner Prefs End %s' % nonce) # Handle profile creation self.create_new = not profile if profile: # Ensure we have a full path to the profile self.profile = os.path.abspath(os.path.expanduser(profile)) if not os.path.exists(self.profile): os.makedirs(self.profile)
--- a/testing/mozbase/mozprofile/setup.py +++ b/testing/mozbase/mozprofile/setup.py @@ -1,17 +1,17 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os import sys from setuptools import setup, find_packages -version = '0.4' +PACKAGE_VERSION = '0.4' # we only support python 2 right now assert sys.version_info[0] == 2 deps = ["ManifestDestiny >= 0.5.4"] # version-dependent dependencies try: import json @@ -26,17 +26,17 @@ except ImportError: # take description from README here = os.path.dirname(os.path.abspath(__file__)) try: description = file(os.path.join(here, 'README.md')).read() except (OSError, IOError): description = '' setup(name='mozprofile', - version=version, + version=PACKAGE_VERSION, description="Handling of Mozilla Gecko based application profiles", long_description=description, classifiers=['Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python',
--- a/testing/mozbase/mozprofile/tests/manifest.ini +++ b/testing/mozbase/mozprofile/tests/manifest.ini @@ -1,5 +1,6 @@ [addonid.py] [server_locations.py] [test_preferences.py] [permissions.py] [bug758250.py] +[test_nonce.py]
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/test_nonce.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +""" +test nonce in prefs delimeters +see https://bugzilla.mozilla.org/show_bug.cgi?id=722804 +""" + +import os +import tempfile +import time +import unittest +from mozprofile.prefs import Preferences +from mozprofile.profile import Profile + +class PreferencesNonceTest(unittest.TestCase): + + def test_nonce(self): + + # make a profile with one preference + path = tempfile.mktemp() + profile = Profile(path, + preferences={'foo': 'bar'}, + restore=False) + user_js = os.path.join(profile.profile, 'user.js') + self.assertTrue(os.path.exists(user_js)) + + # ensure the preference is correct + prefs = Preferences.read_prefs(user_js) + self.assertEqual(dict(prefs), {'foo': 'bar'}) + + del profile + + # augment the profile with a second preference + profile = Profile(path, + preferences={'fleem': 'baz'}, + restore=True) + prefs = Preferences.read_prefs(user_js) + self.assertEqual(dict(prefs), {'foo': 'bar', 'fleem': 'baz'}) + + # cleanup the profile; + # this should remove the new preferences but not the old + profile.cleanup() + prefs = Preferences.read_prefs(user_js) + self.assertEqual(dict(prefs), {'foo': 'bar'}) + +if __name__ == '__main__': + unittest.main()
--- a/testing/mozbase/mozprofile/tests/test_preferences.py +++ b/testing/mozbase/mozprofile/tests/test_preferences.py @@ -80,17 +80,17 @@ browser.startup.homepage = http://github # test a specific section _prefs = {'browser.startup.homepage': 'http://github.com/'} commandline[-1] = commandline[-1] + ':foo' self.compare_generated(_prefs, commandline) # cleanup os.remove(name) - + def test_reset_should_remove_added_prefs(self): """Check that when we call reset the items we expect are updated""" profile = Profile() prefs_file = os.path.join(profile.profile, 'user.js') # we shouldn't have any initial preferences initial_prefs = Preferences.read_prefs(prefs_file) @@ -98,18 +98,20 @@ browser.startup.homepage = http://github initial_prefs = file(prefs_file).read().strip() self.assertFalse(initial_prefs) # add some preferences prefs1 = [("mr.t.quotes", "i aint getting on no plane!")] profile.set_preferences(prefs1) self.assertEqual(prefs1, Preferences.read_prefs(prefs_file)) lines = file(prefs_file).read().strip().splitlines() - self.assertTrue('#MozRunner Prefs Start' in lines) - self.assertTrue('#MozRunner Prefs End' in lines) + self.assertTrue(bool([line for line in lines + if line.startswith('#MozRunner Prefs Start')])) + self.assertTrue(bool([line for line in lines + if line.startswith('#MozRunner Prefs End')])) profile.reset() self.assertNotEqual(prefs1, \ Preferences.read_prefs(os.path.join(profile.profile, 'user.js')),\ "I pity the fool who left my pref") def test_magic_markers(self): """ensure our magic markers are working""" @@ -124,27 +126,31 @@ browser.startup.homepage = http://github self.assertFalse(initial_prefs) # add some preferences prefs1 = [("browser.startup.homepage", "http://planet.mozilla.org/"), ("zoom.minPercent", 30)] profile.set_preferences(prefs1) self.assertEqual(prefs1, Preferences.read_prefs(prefs_file)) lines = file(prefs_file).read().strip().splitlines() - self.assertTrue('#MozRunner Prefs Start' in lines) - self.assertTrue('#MozRunner Prefs End' in lines) + self.assertTrue(bool([line for line in lines + if line.startswith('#MozRunner Prefs Start')])) + self.assertTrue(bool([line for line in lines + if line.startswith('#MozRunner Prefs End')])) # add some more preferences prefs2 = [("zoom.maxPercent", 300), ("webgl.verbose", 'false')] profile.set_preferences(prefs2) self.assertEqual(prefs1 + prefs2, Preferences.read_prefs(prefs_file)) lines = file(prefs_file).read().strip().splitlines() - self.assertTrue(lines.count('#MozRunner Prefs Start') == 2) - self.assertTrue(lines.count('#MozRunner Prefs End') == 2) + self.assertTrue(len([line for line in lines + if line.startswith('#MozRunner Prefs Start')]) == 2) + self.assertTrue(len([line for line in lines + if line.startswith('#MozRunner Prefs End')]) == 2) # now clean it up profile.clean_preferences() final_prefs = Preferences.read_prefs(prefs_file) self.assertFalse(final_prefs) lines = file(prefs_file).read().strip().splitlines() self.assertTrue('#MozRunner Prefs Start' not in lines) self.assertTrue('#MozRunner Prefs End' not in lines)
--- a/testing/mozbase/mozrunner/mozrunner/utils.py +++ b/testing/mozbase/mozrunner/mozrunner/utils.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """ utility functions for mozrunner """
--- a/testing/mozbase/mozrunner/setup.py +++ b/testing/mozbase/mozrunner/setup.py @@ -2,28 +2,28 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os import sys from setuptools import setup, find_packages PACKAGE_NAME = "mozrunner" -PACKAGE_VERSION = '5.7' +PACKAGE_VERSION = '5.8' desc = """Reliable start/stop/configuration of Mozilla Applications (Firefox, Thunderbird, etc.)""" # take description from README here = os.path.dirname(os.path.abspath(__file__)) try: description = file(os.path.join(here, 'README.md')).read() except (OSError, IOError): description = '' deps = ['mozinfo == 0.3.3', - 'mozprocess == 0.3', + 'mozprocess == 0.4', 'mozprofile == 0.4', ] # we only support python 2 right now assert sys.version_info[0] == 2 setup(name=PACKAGE_NAME, version=PACKAGE_VERSION,
--- a/testing/mozbase/setup_development.py +++ b/testing/mozbase/setup_development.py @@ -35,64 +35,78 @@ all_packages = [i for i in os.listdir(he def cycle_check(order, dependencies): """ensure no cyclic dependencies""" order_dict = dict([(j, i) for i, j in enumerate(order)]) for package, deps in dependencies.items(): index = order_dict[package] for d in deps: assert index > order_dict[d], "Cyclic dependencies detected" -def dependencies(directory): - """ - get the dependencies of a package directory containing a setup.py - returns the package name and the list of dependencies - """ +def info(directory): + "get the package setup.py information" + assert os.path.exists(os.path.join(directory, 'setup.py')) # setup the egg info call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE) # get the .egg-info directory - egg_info = [i for i in os.listdir(directory) - if i.endswith('.egg-info')] + egg_info = [entry for entry in os.listdir(directory) + if entry.endswith('.egg-info')] assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info) egg_info = os.path.join(directory, egg_info[0]) assert os.path.isdir(egg_info), "%s is not a directory" % egg_info - # read the dependencies - requires = os.path.join(egg_info, 'requires.txt') - if os.path.exists(requires): - dependencies = [i.strip() for i in file(requires).readlines() if i.strip()] - else: - dependencies = [] - # read the package information pkg_info = os.path.join(egg_info, 'PKG-INFO') info_dict = {} for line in file(pkg_info).readlines(): if not line or line[0].isspace(): continue # XXX neglects description assert ':' in line key, value = [i.strip() for i in line.split(':', 1)] info_dict[key] = value + return info_dict + +def get_dependencies(directory): + "returns the package name and dependencies given a package directory" + + # get the package metadata + info_dict = info(directory) + + # get the .egg-info directory + egg_info = [entry for entry in os.listdir(directory) + if entry.endswith('.egg-info')][0] + + # read the dependencies + requires = os.path.join(directory, egg_info, 'requires.txt') + if os.path.exists(requires): + dependencies = [line.strip() + for line in file(requires).readlines() + if line.strip()] + else: + dependencies = [] # return the information return info_dict['Name'], dependencies -def sanitize_dependency(dep): - """ - remove version numbers from deps - """ +def dependency_info(dep): + "return dictionary of dependency information from a dependency string" + retval = dict(Name=None, Type=None, Version=None) for joiner in ('==', '<=', '>='): if joiner in dep: - dep = dep.split(joiner, 1)[0].strip() - return dep # XXX only one joiner allowed right now - return dep - + retval['Type'] = joiner + name, version = [i.strip() for i in dep.split(joiner, 1)] + retval['Name'] = name + retval['Version'] = version + break + else: + retval['name'] = dep.strip() + return retval def unroll_dependencies(dependencies): """ unroll a set of dependencies to a flat list dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']), 'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']), 'packageC': set(['packageE']), @@ -139,57 +153,57 @@ def main(args=sys.argv[1:]): packages = sorted(all_packages) # ensure specified packages are in the list assert set(packages).issubset(all_packages), "Packages should be in %s (You gave: %s)" % (all_packages, packages) if options.list_dependencies: # list the package dependencies for package in packages: - print '%s: %s' % dependencies(os.path.join(here, package)) + print '%s: %s' % get_dependencies(os.path.join(here, package)) parser.exit() # gather dependencies # TODO: version conflict checking deps = {} alldeps = {} mapping = {} # mapping from subdir name to package name # core dependencies for package in packages: - key, value = dependencies(os.path.join(here, package)) - deps[key] = [sanitize_dependency(dep) for dep in value] + key, value = get_dependencies(os.path.join(here, package)) + deps[key] = [dependency_info(dep)['Name'] for dep in value] mapping[package] = key # keep track of all dependencies for non-mozbase packages for dep in value: - alldeps[sanitize_dependency(dep)] = ''.join(dep.split()) + alldeps[dependency_info(dep)['Name']] = ''.join(dep.split()) # indirect dependencies flag = True while flag: flag = False for value in deps.values(): for dep in value: if dep in all_packages and dep not in deps: - key, value = dependencies(os.path.join(here, dep)) + key, value = get_dependencies(os.path.join(here, dep)) deps[key] = [sanitize_dependency(dep) for dep in value] for dep in value: alldeps[sanitize_dependency(dep)] = ''.join(dep.split()) mapping[package] = key flag = True break if flag: break # get the remaining names for the mapping for package in all_packages: if package in mapping: continue - key, value = dependencies(os.path.join(here, package)) + key, value = get_dependencies(os.path.join(here, package)) mapping[package] = key # unroll dependencies unrolled = unroll_dependencies(deps) # make a reverse mapping: package name -> subdirectory reverse_mapping = dict([(j,i) for i, j in mapping.items()])
--- a/testing/mozbase/test-manifest.ini +++ b/testing/mozbase/test-manifest.ini @@ -1,8 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + # mozbase test manifest, in the format of # https://github.com/mozilla/mozbase/blob/master/manifestdestiny/README.txt # run with # https://github.com/mozilla/mozbase/blob/master/test.py [include:mozprocess/tests/manifest.ini] [include:mozprofile/tests/manifest.ini]
--- a/testing/mozbase/test.py +++ b/testing/mozbase/test.py @@ -1,10 +1,14 @@ #!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + """ run mozbase tests from a manifest, by default https://github.com/mozilla/mozbase/blob/master/test-manifest.ini """ import imp import manifestparser import os