author | Jonathan Griffin <jgriffin@mozilla.com> |
Tue, 22 Oct 2013 15:42:05 -0700 | |
changeset 151726 | e199bf0b32574e69b511762e657cec3e6f3ab925 |
parent 151725 | fa13474d7b16433307c5f090c152c25e3e8a5292 |
child 151728 | 820620c8a288fbd4ce80da9ec48d85851a6bf30e |
push id | 25504 |
push user | philringnalda@gmail.com |
push date | Wed, 23 Oct 2013 02:50:41 +0000 |
treeherder | mozilla-central@21d97baadc05 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahal |
bugs | 917750 |
milestone | 27.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozbase/mozcrash/setup.py +++ b/testing/mozbase/mozcrash/setup.py @@ -1,18 +1,18 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup -PACKAGE_VERSION = '0.9' +PACKAGE_VERSION = '0.10' # dependencies -deps = ['mozfile >= 0.3', +deps = ['mozfile >= 0.12', 'mozlog'] setup(name='mozcrash', version=PACKAGE_VERSION, description="Library for printing stack traces from minidumps left behind by crashed processes", long_description="see http://mozbase.readthedocs.org/", classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla',
--- a/testing/mozbase/mozfile/mozfile/mozfile.py +++ b/testing/mozbase/mozfile/mozfile/mozfile.py @@ -1,8 +1,10 @@ +# -*- coding: utf-8 -*- + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from contextlib import contextmanager import os import shutil import tarfile @@ -12,16 +14,17 @@ import urllib2 import zipfile __all__ = ['extract_tarball', 'extract_zip', 'extract', 'is_url', 'load', 'rmtree', + 'tree', 'NamedTemporaryFile', 'TemporaryDirectory'] ### utilities for extracting archives def extract_tarball(src, dest): """extract a .tar file""" @@ -102,16 +105,18 @@ def extract(src, dest=None): if index != -1: root = os.path.join(dest, name[:index]) if root not in top_level_files: top_level_files.append(root) return top_level_files +### utilities for directory trees + def rmtree(dir): """Removes the specified directory tree This is a replacement for shutil.rmtree that works better under windows.""" # (Thanks to Bear at the OSAF for the code.) if not os.path.exists(dir): return @@ -151,16 +156,107 @@ def rmtree(dir): rmtree(full_name) else: if os.path.isfile(full_name): os.chmod(full_name, 0700) os.remove(full_name) os.rmdir(dir) +def depth(directory): + """returns the integer depth of a directory or path relative to '/' """ + + directory = os.path.abspath(directory) + level = 0 + while True: + directory, remainder = os.path.split(directory) + level += 1 + if not remainder: + break + return level + +# ASCII delimeters +ascii_delimeters = { + 'vertical_line' : '|', + 'item_marker' : '+', + 'last_child' : '\\' + } + +# unicode delimiters +unicode_delimeters = { + 'vertical_line' : '│', + 'item_marker' : '├', + 'last_child' : '└' + } + +def tree(directory, + item_marker=unicode_delimeters['item_marker'], + vertical_line=unicode_delimeters['vertical_line'], + last_child=unicode_delimeters['last_child'], + sort_key=lambda x: x.lower()): + """ + display tree directory structure for `directory` + """ + + retval = [] + indent = [] + last = {} + top = depth(directory) + + for dirpath, dirnames, filenames in os.walk(directory, topdown=True): + + abspath = os.path.abspath(dirpath) + basename = os.path.basename(abspath) + parent = os.path.dirname(abspath) + level = depth(abspath) - top + + # sort articles of interest + for resource in (dirnames, filenames): + resource[:] = sorted(resource, key=sort_key) + + files_end = item_marker + dirpath_marker = item_marker + + if level > len(indent): + indent.append(vertical_line) + indent = indent[:level] + + if dirnames: + files_end = item_marker + last[abspath] = dirnames[-1] + else: + files_end = last_child + + if last.get(parent) == os.path.basename(abspath): + # last directory of parent + dirpath_mark = last_child + indent[-1] = ' ' + elif not indent: + dirpath_mark = '' + else: + dirpath_mark = item_marker + + # append the directory and piece of tree structure + # if the top-level entry directory, print as passed + retval.append('%s%s%s'% (''.join(indent[:-1]), + dirpath_mark, + basename if retval else directory)) + # add the files + if filenames: + last_file = filenames[-1] + retval.extend([('%s%s%s' % (''.join(indent), + files_end if filename == last_file else item_marker, + filename)) + for index, filename in enumerate(filenames)]) + + return '\n'.join(retval) + + +### utilities for temporary resources + class NamedTemporaryFile(object): """ Like tempfile.NamedTemporaryFile except it works on Windows in the case where you open the created file a second time. This behaves very similarly to tempfile.NamedTemporaryFile but may not behave exactly the same. For example, this function does not prevent fd inheritance by children. @@ -204,16 +300,33 @@ class NamedTemporaryFile(object): def __del__(self): if self.__dict__['_unlinked']: return self.file.__exit__(None, None, None) if self.__dict__['_delete']: os.unlink(self.__dict__['_path']) +@contextmanager +def TemporaryDirectory(): + """ + create a temporary directory using tempfile.mkdtemp, and then clean it up. + + Example usage: + with TemporaryDirectory() as tmp: + open(os.path.join(tmp, "a_temp_file"), "w").write("data") + + """ + tempdir = tempfile.mkdtemp() + try: + yield tempdir + finally: + shutil.rmtree(tempdir) + + ### utilities dealing with URLs def is_url(thing): """ Return True if thing looks like a URL. """ parsed = urlparse.urlparse(thing) @@ -234,23 +347,8 @@ def load(resource): resource = resource[len('file://'):] if not is_url(resource): # if no scheme is given, it is a file path return file(resource) return urllib2.urlopen(resource) -@contextmanager -def TemporaryDirectory(): - """ - create a temporary directory using tempfile.mkdtemp, and then clean it up. - - Example usage: - with TemporaryDirectory() as tmp: - open(os.path.join(tmp, "a_temp_file"), "w").write("data") - - """ - tempdir = tempfile.mkdtemp() - try: - yield tempdir - finally: - shutil.rmtree(tempdir)
--- a/testing/mozbase/mozfile/setup.py +++ b/testing/mozbase/mozfile/setup.py @@ -1,15 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup -PACKAGE_VERSION = '0.10' +PACKAGE_VERSION = '0.12' setup(name='mozfile', version=PACKAGE_VERSION, description="Library of file utilities for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org/", classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Tools team',
--- a/testing/mozbase/mozhttpd/mozhttpd/handlers.py +++ b/testing/mozbase/mozhttpd/mozhttpd/handlers.py @@ -1,16 +1,13 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -try: - import json -except ImportError: - import simplejson as json +import json def json_response(func): """ Translates results of 'func' into a JSON response. """ def wrap(*a, **kw): (code, data) = func(*a, **kw) json_data = json.dumps(data) return (code, { 'Content-type': 'application/json', 'Content-Length': len(json_data) }, json_data)
--- a/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py +++ b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py @@ -90,31 +90,47 @@ class RequestHandler(SimpleHTTPServer.Si self.send_header(keyword, value) self.end_headers() self.wfile.write(data) return True return False + def _find_path(self): + """Find the on-disk path to serve this request from, + using self.path_mappings and self.docroot. + Return (url_path, disk_path).""" + path_components = filter(None, self.request.path.split('/')) + for prefix, disk_path in self.path_mappings.iteritems(): + prefix_components = filter(None, prefix.split('/')) + if len(path_components) < len(prefix_components): + continue + if path_components[:len(prefix_components)] == prefix_components: + return ('/'.join(path_components[len(prefix_components):]), + disk_path) + if self.docroot: + return self.request.path, self.docroot + return None + def parse_request(self): retval = SimpleHTTPServer.SimpleHTTPRequestHandler.parse_request(self) self.request = Request(self.path, self.headers, self.rfile) return retval def do_GET(self): if not self._try_handler('GET'): - if self.docroot: + res = self._find_path() + if res: + self.path, self.disk_root = res # don't include query string and fragment, and prepend # host directory if required. if self.request.netloc and self.proxy_host_dirs: self.path = '/' + self.request.netloc + \ - self.request.path - else: - self.path = self.request.path + self.path SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) else: self.send_response(404) self.end_headers() self.wfile.write('') def do_POST(self): # if we don't have a match, we always fall through to 404 (this may @@ -137,17 +153,17 @@ class RequestHandler(SimpleHTTPServer.Si def translate_path(self, path): # this is taken from SimpleHTTPRequestHandler.translate_path(), # except we serve from self.docroot instead of os.getcwd(), and # parse_request()/do_GET() have already stripped the query string and # fragment and mangled the path for proxying, if required. path = posixpath.normpath(urllib.unquote(self.path)) words = path.split('/') words = filter(None, words) - path = self.docroot + path = self.disk_root for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path @@ -162,16 +178,17 @@ class RequestHandler(SimpleHTTPServer.Si class MozHttpd(object): """ :param host: Host from which to serve (default 127.0.0.1) :param port: Port from which to serve (default 8888) :param docroot: Server root (default os.getcwd()) :param urlhandlers: Handlers to specify behavior against method and path match (default None) + :param path_mappings: A dict mapping URL prefixes to additional on-disk paths. :param proxy_host_dirs: Toggle proxy behavior (default False) :param log_requests: Toggle logging behavior (default False) Very basic HTTP server class. Takes a docroot (path on the filesystem) and a set of urlhandler dictionaries of the form: :: @@ -196,32 +213,40 @@ class MozHttpd(object): from <self.docroot>/<host>/. For example, the request "GET http://foo.bar/dir/file.html" would (assuming no handlers match) serve <docroot>/dir/file.html if proxy_host_dirs is False, or <docroot>/foo.bar/dir/file.html if it is True. """ - def __init__(self, host="127.0.0.1", port=0, docroot=None, - urlhandlers=None, proxy_host_dirs=False, log_requests=False): + def __init__(self, + host="127.0.0.1", + port=0, + docroot=None, + urlhandlers=None, + path_mappings=None, + proxy_host_dirs=False, + log_requests=False): self.host = host self.port = int(port) self.docroot = docroot - if not urlhandlers and not docroot: + if not (urlhandlers or docroot or path_mappings): self.docroot = os.getcwd() self.proxy_host_dirs = proxy_host_dirs self.httpd = None self.urlhandlers = urlhandlers or [] + self.path_mappings = path_mappings or {} self.log_requests = log_requests self.request_log = [] class RequestHandlerInstance(RequestHandler): docroot = self.docroot urlhandlers = self.urlhandlers + path_mappings = self.path_mappings proxy_host_dirs = self.proxy_host_dirs request_log = self.request_log log_requests = self.log_requests self.handler_class = RequestHandlerInstance def start(self, block=False): """
--- a/testing/mozbase/mozhttpd/setup.py +++ b/testing/mozbase/mozhttpd/setup.py @@ -1,16 +1,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup -PACKAGE_VERSION = '0.6' -deps = ['moznetwork >= 0.1'] +PACKAGE_VERSION = '0.7' +deps = ['moznetwork >= 0.24'] setup(name='mozhttpd', version=PACKAGE_VERSION, description="Python webserver intended for use with Mozilla testing", long_description="see http://mozbase.readthedocs.org/", classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Testing Team',
--- a/testing/mozbase/mozhttpd/tests/api.py +++ b/testing/mozbase/mozhttpd/tests/api.py @@ -4,20 +4,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozhttpd import urllib2 import os import unittest import re -try: - import json -except ImportError: - import simplejson as json +import json import tempfile here = os.path.dirname(os.path.abspath(__file__)) class ApiTest(unittest.TestCase): resource_get_called = 0 resource_post_called = 0 resource_del_called = 0
--- a/testing/mozbase/mozhttpd/tests/manifest.ini +++ b/testing/mozbase/mozhttpd/tests/manifest.ini @@ -1,5 +1,6 @@ [api.py] [baseurl.py] [basic.py] [filelisting.py] +[paths.py] [requestlog.py]
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/paths.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from mozfile import TemporaryDirectory +import mozhttpd +import os +import unittest +import urllib2 + +class PathTest(unittest.TestCase): + def try_get(self, url, expected_contents): + f = urllib2.urlopen(url) + self.assertEqual(f.getcode(), 200) + self.assertEqual(f.read(), expected_contents) + + def try_get_expect_404(self, url): + with self.assertRaises(urllib2.HTTPError) as cm: + urllib2.urlopen(url) + self.assertEqual(404, cm.exception.code) + + def test_basic(self): + """Test that requests to docroot and a path mapping work as expected.""" + with TemporaryDirectory() as d1, TemporaryDirectory() as d2: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents") + httpd = mozhttpd.MozHttpd(port=0, + docroot=d1, + path_mappings={'/files': d2} + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/test1.txt"), "test 1 contents") + self.try_get(httpd.get_url("/files/test2.txt"), "test 2 contents") + self.try_get_expect_404(httpd.get_url("/files/test2_nope.txt")) + httpd.stop() + + def test_substring_mappings(self): + """Test that a path mapping that's a substring of another works.""" + with TemporaryDirectory() as d1, TemporaryDirectory() as d2: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents") + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/abcxyz': d1, + '/abc': d2,} + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/abcxyz/test1.txt"), "test 1 contents") + self.try_get(httpd.get_url("/abc/test2.txt"), "test 2 contents") + httpd.stop() + + def test_multipart_path_mapping(self): + """Test that a path mapping with multiple directories works.""" + with TemporaryDirectory() as d1: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/abc/def/ghi': d1} + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/abc/def/ghi/test1.txt"), "test 1 contents") + self.try_get_expect_404(httpd.get_url("/abc/test1.txt")) + self.try_get_expect_404(httpd.get_url("/abc/def/test1.txt")) + httpd.stop() + + def test_no_docroot(self): + """Test that path mappings with no docroot work.""" + with TemporaryDirectory() as d1: + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/foo': d1}) + httpd.start(block=False) + self.try_get_expect_404(httpd.get_url()) + httpd.stop() + +if __name__ == '__main__': + unittest.main()
--- a/testing/mozbase/mozinfo/mozinfo/mozinfo.py +++ b/testing/mozbase/mozinfo/mozinfo/mozinfo.py @@ -3,26 +3,22 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for # linux) to the information; I certainly wouldn't want anyone parsing this # information and having behaviour depend on it +import json import os import platform import re import sys -try: - import json -except ImportError: - import simplejson as json - import mozfile # keep a copy of the os module since updating globals overrides this _os = os class unknown(object): """marker class for unknown information""" def __nonzero__(self):
--- a/testing/mozbase/mozinfo/setup.py +++ b/testing/mozbase/mozinfo/setup.py @@ -1,22 +1,18 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup -PACKAGE_VERSION = '0.6' +PACKAGE_VERSION = '0.7' # dependencies -deps = ['mozfile >= 0.6'] -try: - import json -except ImportError: - deps = ['simplejson'] +deps = ['mozfile >= 0.12'] setup(name='mozinfo', version=PACKAGE_VERSION, description="Library to get system information for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org", classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Testing Team',
--- a/testing/mozbase/mozinstall/setup.py +++ b/testing/mozbase/mozinstall/setup.py @@ -6,19 +6,19 @@ import os from setuptools import setup try: here = os.path.dirname(os.path.abspath(__file__)) description = file(os.path.join(here, 'README.md')).read() except IOError: description = None -PACKAGE_VERSION = '1.7' +PACKAGE_VERSION = '1.8' -deps = ['mozinfo >= 0.4', +deps = ['mozinfo >= 0.7', 'mozfile' ] setup(name='mozInstall', version=PACKAGE_VERSION, description="package for installing and uninstalling Mozilla applications", long_description="see http://mozbase.readthedocs.org/", # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
--- a/testing/mozbase/moznetwork/moznetwork/moznetwork.py +++ b/testing/mozbase/moznetwork/moznetwork/moznetwork.py @@ -1,15 +1,17 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import socket import array +import re import struct +import subprocess import mozinfo if mozinfo.isLinux: import fcntl class NetworkError(Exception): """Exception thrown when unable to obtain interface or IP.""" @@ -33,39 +35,78 @@ def _get_interface_list(): namestr = names.tostring() return [(namestr[i:i + 32].split('\0', 1)[0], socket.inet_ntoa(namestr[i + 20:i + 24]))\ for i in range(0, outbytes, struct_size)] except IOError: raise NetworkError('Unable to call ioctl with SIOCGIFCONF') +def _proc_matches(args, regex): + """Helper returns the matches of regex in the output of a process created with + the given arguments""" + output = subprocess.Popen(args=args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).stdout.read() + return re.findall(regex, output) + +def _parse_ifconfig(): + """Parse the output of running ifconfig on mac in cases other methods + have failed""" + + # Attempt to determine the default interface in use. + default_iface = _proc_matches(['route', '-n', 'get', 'default'], + 'interface: (\w+)') + if default_iface: + addr_list = _proc_matches(['ifconfig', default_iface[0]], + 'inet (\d+.\d+.\d+.\d+)') + if addr_list and not addr_list[0].startswith('127.'): + return addr_list[0] + + # Iterate over plausible interfaces if we didn't find a suitable default. + for iface in ['en%s' % i for i in range(10)]: + addr_list = _proc_matches(['ifconfig', iface], + 'inet (\d+.\d+.\d+.\d+)') + if addr_list and not addr_list[0].startswith('127.'): + return addr_list[0] + + # Just return any that isn't localhost. If we can't find one, we have + # failed. + addrs = _proc_matches(['ifconfig'], + 'inet (\d+.\d+.\d+.\d+)') + try: + return [addr for addr in addrs if not addr.startswith('127.')][0] + except IndexError: + return None def get_ip(): """Provides an available network interface address, for example "192.168.1.3". A `NetworkError` exception is raised in case of failure.""" try: try: ip = socket.gethostbyname(socket.gethostname()) except socket.gaierror: # for Mac OS X ip = socket.gethostbyname(socket.gethostname() + ".local") except socket.gaierror: # sometimes the hostname doesn't resolve to an ip address, in which # case this will always fail ip = None - if (ip is None or ip.startswith("127.")) and mozinfo.isLinux: - interfaces = _get_interface_list() - for ifconfig in interfaces: - if ifconfig[0] == 'lo': - continue - else: - return ifconfig[1] + if ip is None or ip.startswith("127."): + if mozinfo.isLinux: + interfaces = _get_interface_list() + for ifconfig in interfaces: + if ifconfig[0] == 'lo': + continue + else: + return ifconfig[1] + elif mozinfo.isMac: + ip = _parse_ifconfig() if ip is None: raise NetworkError('Unable to obtain network address') return ip def get_lan_ip():
--- a/testing/mozbase/moznetwork/setup.py +++ b/testing/mozbase/moznetwork/setup.py @@ -1,15 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup -PACKAGE_VERSION = '0.22' +PACKAGE_VERSION = '0.24' deps=[ 'mozinfo' ] setup(name='moznetwork', version=PACKAGE_VERSION, description="Library of network utilities for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org/", classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
--- a/testing/mozbase/moznetwork/tests/manifest.ini +++ b/testing/mozbase/moznetwork/tests/manifest.ini @@ -1,3 +1,1 @@ [test.py] -# Bug 892087 - Doesn't work reliably on osx -skip-if = os == 'mac'
--- a/testing/mozbase/moznetwork/tests/test.py +++ b/testing/mozbase/moznetwork/tests/test.py @@ -57,17 +57,17 @@ class TestGetIP(unittest.TestCase): # Check the IP returned by moznetwork is in the list self.assertTrue(verify_ip_in_list(ip)) def test_get_ip_using_get_interface(self): """ Test that the control flow path for get_ip() using _get_interface_list() is works """ - if mozinfo.isLinux: + if mozinfo.isLinux or mozinfo.isMac: with mock.patch('socket.gethostbyname') as byname: # Force socket.gethostbyname to return None byname.return_value = None ip = moznetwork.get_ip() # Check the IP returned by moznetwork is in the list
--- a/testing/mozbase/mozprofile/mozprofile/__init__.py +++ b/testing/mozbase/mozprofile/mozprofile/__init__.py @@ -7,12 +7,14 @@ To use mozprofile as an API you can impo ``mozprofile.profile`` features a generic ``Profile`` class. In addition, subclasses ``FirefoxProfile`` and ``ThundebirdProfile`` are available with preset preferences for those applications. """ from addons import * from cli import * +from diff import * from permissions import * from prefs import * from profile import * +from view import * from webapps import *
--- a/testing/mozbase/mozprofile/mozprofile/addons.py +++ b/testing/mozbase/mozprofile/mozprofile/addons.py @@ -11,57 +11,58 @@ from distutils import dir_util from manifestparser import ManifestParser from xml.dom import minidom # Needed for the AMO's rest API - https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API AMO_API_VERSION = "1.5" class AddonManager(object): """ - Handles all operations regarding addons in a profile including: installing and cleaning addons + Handles all operations regarding addons in a profile including: + installing and cleaning addons """ - def __init__(self, profile): + def __init__(self, profile, restore=True): """ :param profile: the path to the profile for which we install addons + :param restore: whether to reset to the previous state on instance garbage collection """ self.profile = profile + self.restore = restore # information needed for profile reset: # https://github.com/mozilla/mozbase/blob/270a857328b130860d1b1b512e23899557a3c8f7/mozprofile/mozprofile/profile.py#L93 self.installed_addons = [] self.installed_manifests = [] # addons that we've installed; needed for cleanup - self._addon_dirs = [] + self._addons = [] # backup dir for already existing addons self.backup_dir = None def install_addons(self, addons=None, manifests=None): """ Installs all types of addons :param addons: a list of addon paths to install :param manifest: a list of addon manifests to install """ # install addon paths if addons: if isinstance(addons, basestring): addons = [addons] - self.installed_addons.extend(addons) for addon in addons: self.install_from_path(addon) # install addon manifests if manifests: if isinstance(manifests, basestring): manifests = [manifests] for manifest in manifests: self.install_from_manifest(manifest) - self.installed_manifests.extend(manifests) def install_from_manifest(self, filepath): """ Installs addons from a manifest :param filepath: path to the manifest of addons to install """ manifest = ManifestParser() manifest.read(filepath) @@ -69,25 +70,26 @@ class AddonManager(object): for addon in addons: if '://' in addon['path'] or os.path.exists(addon['path']): self.install_from_path(addon['path']) continue # No path specified, try to grab it off AMO locale = addon.get('amo_locale', 'en_US') - query = 'https://services.addons.mozilla.org/' + locale + '/firefox/api/' + AMO_API_VERSION + '/' if 'amo_id' in addon: query += 'addon/' + addon['amo_id'] # this query grabs information on the addon base on its id else: query += 'search/' + addon['name'] + '/default/1' # this query grabs information on the first addon returned from a search install_path = AddonManager.get_amo_install_path(query) self.install_from_path(install_path) + self.installed_manifests.append(filepath) + @classmethod def get_amo_install_path(self, query): """ Get the addon xpi install path for the specified AMO query. :param query: query-documentation_ .. _query-documentation: https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API @@ -108,17 +110,16 @@ class AddonManager(object): Returns:: {'id': u'rainbow@colors.org', # id of the addon 'version': u'1.4', # version of the addon 'name': u'Rainbow', # name of the addon 'unpack': False } # whether to unpack the addon """ - # TODO: We don't use the unpack variable yet, but we should: bug 662683 details = { 'id': None, 'unpack': False, 'name': None, 'version': None } def get_namespace_id(doc, url): @@ -188,17 +189,17 @@ class AddonManager(object): # if the addon is a directory, install all addons in it addons = [path] if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')): # If the path doesn't exist, then we don't really care, just return if not os.path.isdir(path): return addons = [os.path.join(path, x) for x in os.listdir(path) if - os.path.isdir(os.path.join(path, x))] + os.path.isdir(os.path.join(path, x))] # install each addon for addon in addons: tmpdir = None xpifile = None if addon.endswith('.xpi'): tmpdir = tempfile.mkdtemp(suffix = '.' + os.path.split(addon)[-1]) compressed_file = zipfile.ZipFile(addon, 'r') @@ -222,44 +223,57 @@ class AddonManager(object): # copy the addon to the profile extensions_path = os.path.join(self.profile, 'extensions', 'staged') addon_path = os.path.join(extensions_path, addon_id) if not unpack and not addon_details['unpack'] and xpifile: if not os.path.exists(extensions_path): os.makedirs(extensions_path) # save existing xpi file to restore later - if os.path.exists(addon_path + '.xpi'): + addon_path += '.xpi' + if os.path.exists(addon_path): self.backup_dir = self.backup_dir or tempfile.mkdtemp() - shutil.copy(addon_path + '.xpi', self.backup_dir) - shutil.copy(xpifile, addon_path + '.xpi') + shutil.copy(addon_path, self.backup_dir) + shutil.copy(xpifile, addon_path) else: # save existing dir to restore later if os.path.exists(addon_path): self.backup_dir = self.backup_dir or tempfile.mkdtemp() dir_util.copy_tree(addon_path, self.backup_dir, preserve_symlinks=1) dir_util.copy_tree(addon, addon_path, preserve_symlinks=1) - self._addon_dirs.append(addon_path) + self._addons.append(addon_path) # remove the temporary directory, if any if tmpdir: dir_util.remove_tree(tmpdir) + self.installed_addons.append(addon) + # remove temporary file, if any if tmpfile: os.remove(tmpfile) def clean_addons(self): """Cleans up addons in the profile.""" - for addon in self._addon_dirs: + + # remove addons installed by this instance + for addon in self._addons: if os.path.isdir(addon): dir_util.remove_tree(addon) + elif os.path.isfile(addon): + os.remove(addon) + # restore backups if self.backup_dir and os.path.isdir(self.backup_dir): extensions_path = os.path.join(self.profile, 'extensions', 'staged') for backup in os.listdir(self.backup_dir): backup_path = os.path.join(self.backup_dir, backup) addon_path = os.path.join(extensions_path, backup) shutil.move(backup_path, addon_path) if not os.listdir(self.backup_dir): shutil.rmtree(self.backup_dir, ignore_errors=True) - __del__ = clean_addons + # reset instance variables to defaults via __init__ + self.__init__(self.profile, restore=self.restore) + + def __del__(self): + if self.restore: + self.clean_addons() # reset to pre-instance state
--- a/testing/mozbase/mozprofile/mozprofile/cli.py +++ b/testing/mozbase/mozprofile/mozprofile/cli.py @@ -9,28 +9,32 @@ Creates and/or modifies a Firefox profil The profile can be modified by passing in addons to install or preferences to set. If no profile is specified, a new profile is created and the path of the resulting profile is printed. """ import sys from addons import AddonManager from optparse import OptionParser from prefs import Preferences +from profile import FirefoxProfile from profile import Profile __all__ = ['MozProfileCLI', 'cli'] class MozProfileCLI(object): """The Command Line Interface for ``mozprofile``.""" module = 'mozprofile' + profile_class = Profile - def __init__(self, args=sys.argv[1:]): + def __init__(self, args=sys.argv[1:], add_options=None): self.parser = OptionParser(description=__doc__) self.add_options(self.parser) + if add_options: + add_options(self.parser) (self.options, self.args) = self.parser.parse_args(args) def add_options(self, parser): parser.add_option("-p", "--profile", dest="profile", help="The path to the profile to operate on. If none, creates a new profile in temp directory") parser.add_option("-a", "--addon", dest="addons", action="append", default=[], @@ -76,26 +80,46 @@ class MozProfileCLI(object): return prefs() def profile(self, restore=False): """create the profile""" kwargs = self.profile_args() kwargs['restore'] = restore - return Profile(**kwargs) + return self.profile_class(**kwargs) def cli(args=sys.argv[1:]): """ Handles the command line arguments for ``mozprofile`` via ``sys.argv``""" + # add a view method for this cli method only + def add_options(parser): + parser.add_option('--view', dest='view', + action='store_true', default=False, + help="view summary of profile following invocation") + parser.add_option('--firefox', dest='firefox_profile', + action='store_true', default=False, + help="use FirefoxProfile defaults") + # process the command line - cli = MozProfileCLI(args) + cli = MozProfileCLI(args, add_options) + + if cli.args: + cli.parser.error("Program doesn't support positional arguments.") + + if cli.options.firefox_profile: + cli.profile_class = FirefoxProfile # create the profile profile = cli.profile() + if cli.options.view: + # view the profile, if specified + print profile.summary() + return + # if no profile was passed in print the newly created profile if not cli.options.profile: print profile.profile if __name__ == '__main__': cli()
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/mozprofile/diff.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +""" +diff two profile summaries +""" + +import difflib +import profile +import optparse +import os +import sys + +__all__ = ['diff', 'diff_profiles'] + +def diff(profile1, profile2, diff_function=difflib.unified_diff): + + profiles = (profile1, profile2) + parts = {} + parts_dict = {} + for index in (0, 1): + profile = profiles[index] + + # first part, the path, isn't useful for diffing + parts[index] = profile.summary(return_parts=True)[1:] + + parts_dict[index] = dict(parts[index]) + + # keys the first profile is missing + first_missing = [i for i in parts_dict[1] + if i not in parts_dict[0]] + parts[0].extend([(i, '') for i in first_missing]) + + # diffs + retval = [] + for key, value in parts[0]: + other = parts_dict[1].get(key, '') + value = value.strip(); other = other.strip() + + if key == 'Files': + # first line of files is the path; we don't care to diff that + value = '\n'.join(value.splitlines()[1:]) + if other: + other = '\n'.join(other.splitlines()[1:]) + + value = value.splitlines() + other = other.splitlines() + section_diff = list(diff_function(value, other, profile1.profile, profile2.profile)) + if section_diff: + retval.append((key, '\n'.join(section_diff))) + + return retval + +def diff_profiles(args=sys.argv[1:]): + + # parse command line + usage = '%prog [options] profile1 profile2' + parser = optparse.OptionParser(usage=usage, description=__doc__) + options, args = parser.parse_args(args) + if len(args) != 2: + parser.error("Must give two profile paths") + missing = [arg for arg in args if not os.path.exists(arg)] + if missing: + parser.error("Profile not found: %s" % (', '.join(missing))) + + # get the profile differences + diffs = diff(*([profile.Profile(arg) + for arg in args])) + + # display them + while diffs: + key, value = diffs.pop(0) + print '[%s]:\n' % key + print value + if diffs: + print '-' * 4 + +if __name__ == '__main__': + diff_profiles()
--- a/testing/mozbase/mozprofile/mozprofile/permissions.py +++ b/testing/mozbase/mozprofile/mozprofile/permissions.py @@ -10,20 +10,17 @@ add permissions to the profile __all__ = ['MissingPrimaryLocationError', 'MultiplePrimaryLocationsError', 'DEFAULT_PORTS', 'DuplicateLocationError', 'BadPortLocationError', 'LocationsSyntaxError', 'Location', 'ServerLocations', 'Permissions'] import codecs import itertools import os -try: - import sqlite3 -except ImportError: - from pysqlite2 import dbapi2 as sqlite3 +import sqlite3 import urlparse # http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l28 DEFAULT_PORTS = { 'http': '8888', 'https': '4443', 'ws': '4443', 'wss': '4443' }
--- a/testing/mozbase/mozprofile/mozprofile/prefs.py +++ b/testing/mozbase/mozprofile/mozprofile/prefs.py @@ -3,28 +3,24 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. """ user preferences """ __all__ = ('PreferencesReadError', 'Preferences') +import json import mozfile import os import re import tokenize from ConfigParser import SafeConfigParser as ConfigParser from StringIO import StringIO -try: - import json -except ImportError: - import simplejson as json - class PreferencesReadError(Exception): """read error for prefrences files""" class Preferences(object): """assembly of preferences from various sources""" def __init__(self, prefs=None):
--- a/testing/mozbase/mozprofile/mozprofile/profile.py +++ b/testing/mozbase/mozprofile/mozprofile/profile.py @@ -2,48 +2,47 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. __all__ = ['Profile', 'FirefoxProfile', 'MetroFirefoxProfile', 'ThunderbirdProfile'] +import json import os import time import tempfile import types import uuid + from addons import AddonManager +from mozfile import tree from permissions import Permissions from prefs import Preferences from shutil import copytree, rmtree from webapps import WebappCollection -try: - import json -except ImportError: - import simplejson as json class Profile(object): """Handles all operations regarding profile. Created new profiles, installs extensions, - sets preferences and handles cleanup.""" + sets preferences and handles cleanup. + + :param profile: Path to the profile + :param addons: String of one or list of addons to install + :param addon_manifests: Manifest for addons, see http://ahal.ca/blog/2011/bulk-installing-fx-addons/ + :param apps: Dictionary or class of webapps to install + :param preferences: Dictionary or class of preferences + :param locations: ServerLocations object + :param proxy: setup a proxy + :param restore: If true remove all added addons and preferences when cleaning up + """ def __init__(self, profile=None, addons=None, addon_manifests=None, apps=None, preferences=None, locations=None, proxy=None, restore=True): - """ - :param profile: Path to the profile - :param addons: String of one or list of addons to install - :param addon_manifests: Manifest for addons, see http://ahal.ca/blog/2011/bulk-installing-fx-addons/ - :param apps: Dictionary or class of webapps to install - :param preferences: Dictionary or class of preferences - :param locations: locations to proxy - :param proxy: setup a proxy - dict of server-loc,server-port,ssl-port - :param restore: If true remove all installed addons preferences when cleaning up - """ # if true, remove installed addons/prefs afterwards self.restore = restore # prefs files written to self.written_prefs = set() # our magic markers @@ -81,17 +80,17 @@ class Profile(object): self._locations = locations # store this for reconstruction self._proxy = proxy self.permissions = Permissions(self.profile, locations) prefs_js, user_js = self.permissions.network_prefs(proxy) self.set_preferences(prefs_js, 'prefs.js') self.set_preferences(user_js) # handle addon installation - self.addon_manager = AddonManager(self.profile) + self.addon_manager = AddonManager(self.profile, restore=self.restore) self.addon_manager.install_addons(addons, addon_manifests) # handle webapps self.webapps = WebappCollection(profile=self.profile, apps=apps) self.webapps.update_manifests() def exists(self): """returns whether the profile exists or not""" @@ -168,17 +167,19 @@ class Profile(object): f.close() def pop_preferences(self, filename): """ pop the last set of preferences added returns True if popped """ - lines = file(os.path.join(self.profile, filename)).read().splitlines() + path = os.path.join(self.profile, filename) + with file(path) as f: + lines = f.read().splitlines() def last_index(_list, value): """ returns the last index of an item; this should actually be part of python code but it isn't """ for index in reversed(range(len(_list))): if _list[index] == value: return index @@ -192,19 +193,18 @@ class Profile(object): elif e is None: assert s is None, '%s found without %s' % (self.delimeters[0], self.delimeters[1]) # ensure the markers are in the proper order assert e > s, '%s found at %s, while %s found at %s' % (self.delimeters[1], e, self.delimeters[0], s) # write the prefs cleaned_prefs = '\n'.join(lines[:s] + lines[e+1:]) - f = file(os.path.join(self.profile, 'user.js'), 'w') - f.write(cleaned_prefs) - f.close() + with file(path, 'w') as f: + f.write(cleaned_prefs) return True def clean_preferences(self): """Removed preferences added by mozrunner.""" for filename in self.written_prefs: if not os.path.exists(os.path.join(self.profile, filename)): # file has been deleted break @@ -250,16 +250,90 @@ class Profile(object): else: self.clean_preferences() self.addon_manager.clean_addons() self.permissions.clean_db() self.webapps.clean() __del__ = cleanup + ### methods for introspection + + def summary(self, return_parts=False): + """ + returns string summarizing profile information. + if return_parts is true, return the (Part_name, value) list + of tuples instead of the assembled string + """ + + parts = [('Path', self.profile)] # profile path + + # directory tree + parts.append(('Files', '\n%s' % tree(self.profile))) + + # preferences + for prefs_file in ('user.js', 'prefs.js'): + path = os.path.join(self.profile, prefs_file) + if os.path.exists(path): + + # prefs that get their own section + # This is currently only 'network.proxy.autoconfig_url' + # but could be expanded to include others + section_prefs = ['network.proxy.autoconfig_url'] + line_length = 80 + line_length_buffer = 10 # buffer for 80 character display: length = 80 - len(key) - len(': ') - line_length_buffer + line_length_buffer += len(': ') + def format_value(key, value): + if key not in section_prefs: + return value + max_length = line_length - len(key) - line_length_buffer + if len(value) > max_length: + value = '%s...' % value[:max_length] + return value + + prefs = Preferences.read_prefs(path) + if prefs: + prefs = dict(prefs) + parts.append((prefs_file, + '\n%s' %('\n'.join(['%s: %s' % (key, format_value(key, prefs[key])) + for key in sorted(prefs.keys()) + ])))) + + # Currently hardcorded to 'network.proxy.autoconfig_url' + # but could be generalized, possibly with a generalized (simple) + # JS-parser + network_proxy_autoconfig = prefs.get('network.proxy.autoconfig_url') + if network_proxy_autoconfig and network_proxy_autoconfig.strip(): + network_proxy_autoconfig = network_proxy_autoconfig.strip() + lines = network_proxy_autoconfig.replace(';', ';\n').splitlines() + lines = [line.strip() for line in lines] + origins_string = 'var origins = [' + origins_end = '];' + if origins_string in lines[0]: + start = lines[0].find(origins_string) + end = lines[0].find(origins_end, start); + splitline = [lines[0][:start], + lines[0][start:start+len(origins_string)-1], + ] + splitline.extend(lines[0][start+len(origins_string):end].replace(',', ',\n').splitlines()) + splitline.append(lines[0][end:]) + lines[0:1] = [i.strip() for i in splitline] + parts.append(('Network Proxy Autoconfig, %s' % (prefs_file), + '\n%s' % '\n'.join(lines))) + + if return_parts: + return parts + + retval = '%s\n' % ('\n\n'.join(['[%s]: %s' % (key, value) + for key, value in parts])) + return retval + + __str__ = summary + + class FirefoxProfile(Profile): """Specialized Profile subclass for Firefox""" preferences = {# Don't automatically update the application 'app.update.enabled' : False, # Don't restore the last open set of tabs if the browser has crashed 'browser.sessionstore.resume_from_crash': False, # Don't check for the default web browser during startup @@ -280,16 +354,18 @@ class FirefoxProfile(Profile): # Dont' run the add-on compatibility check during start-up 'extensions.showMismatchUI' : False, # Don't automatically update add-ons 'extensions.update.enabled' : False, # Don't open a dialog to show available add-on updates 'extensions.update.notifyUser' : False, # Enable test mode to run multiple tests in parallel 'focusmanager.testmode' : True, + # Enable test mode to not raise an OS level dialog for location sharing + 'geo.provider.testing' : True, # Suppress delay for main action in popup notifications 'security.notification_enable_delay' : 0, # Suppress automatic safe mode after crashes 'toolkit.startup.max_resumed_crashes' : -1, # Don't report telemetry information 'toolkit.telemetry.enabled' : False, 'toolkit.telemetry.enabledPreRelease' : False, }
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/mozprofile/view.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +""" +script to view mozilla profiles +""" + +import mozprofile +import optparse +import os +import sys + +__all__ = ['view_profile'] + +def view_profile(args=sys.argv[1:]): + + usage = '%prog [options] profile_path <...>' + parser = optparse.OptionParser(usage=usage, description=__doc__) + options, args = parser.parse_args(args) + if not args: + parser.print_usage() + parser.exit() + + # check existence + missing = [i for i in args + if not os.path.exists(i)] + if missing: + if len(missing) > 1: + missing_string = "Profiles do not exist" + else: + missing_string = "Profile does not exist" + parser.error("%s: %s" % (missing_string, ', '.join(missing))) + + # print summary for each profile + while args: + path = args.pop(0) + profile = mozprofile.Profile(path) + print profile.summary() + if args: + print '-' * 4 + +if __name__ == '__main__': + view_profile()
--- a/testing/mozbase/mozprofile/mozprofile/webapps.py +++ b/testing/mozbase/mozprofile/mozprofile/webapps.py @@ -11,24 +11,20 @@ Each webapp has a manifest (https://deve Additionally there is a separate json manifest that keeps track of the installed webapps, their manifestURLs and their permissions. """ __all__ = ["Webapp", "WebappCollection", "WebappFormatException", "APP_STATUS_NOT_INSTALLED", "APP_STATUS_INSTALLED", "APP_STATUS_PRIVILEGED", "APP_STATUS_CERTIFIED"] from string import Template +import json import os import shutil -try: - import json -except ImportError: - import simplejson as json - # from http://hg.mozilla.org/mozilla-central/file/add0b94c2c0b/caps/idl/nsIPrincipal.idl#l163 APP_STATUS_NOT_INSTALLED = 0 APP_STATUS_INSTALLED = 1 APP_STATUS_PRIVILEGED = 2 APP_STATUS_CERTIFIED = 3 class WebappFormatException(Exception): """thrown for invalid webapp objects"""
--- a/testing/mozbase/mozprofile/setup.py +++ b/testing/mozbase/mozprofile/setup.py @@ -1,32 +1,22 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import sys from setuptools import setup -PACKAGE_VERSION = '0.12' +PACKAGE_VERSION = '0.16' # we only support python 2 right now assert sys.version_info[0] == 2 deps = ["ManifestDestiny >= 0.5.4", - "mozfile >= 0.6"] -# version-dependent dependencies -try: - import json -except ImportError: - deps.append('simplejson') -try: - import sqlite3 -except ImportError: - deps.append('pysqlite') - + "mozfile >= 0.12"] setup(name='mozprofile', version=PACKAGE_VERSION, description="Library to create and modify Mozilla application profiles", long_description="see http://mozbase.readthedocs.org/", classifiers=['Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', @@ -44,10 +34,12 @@ setup(name='mozprofile', include_package_data=True, zip_safe=False, install_requires=deps, tests_require=['mozhttpd', 'mozfile'], entry_points=""" # -*- Entry points: -*- [console_scripts] mozprofile = mozprofile:cli + view-profile = mozprofile:view_profile + diff-profiles = mozprofile:diff_profiles """, )
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/addon_stubs.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import tempfile +import mozhttpd +import os +import zipfile + + +here = os.path.dirname(os.path.abspath(__file__)) + +# stubs is a dict of the form {'addon name': 'install manifest content'} +stubs = { + 'empty-0-1.xpi': + open(os.path.join(here, "install_manifests", "empty-0-1.rdf"), 'r').read(), + 'empty-0-2.xpi': + open(os.path.join(here, "install_manifests", "empty-0-2.rdf"), 'r').read(), + 'another-empty-0-1.xpi': + open(os.path.join(here, "install_manifests", "another-empty-0-1.rdf"), 'r').read(), + 'empty-invalid.xpi': + open(os.path.join(here, "install_manifests", "empty-invalid.rdf"), 'r').read()} + +def generate_addon(name, path=None): + """ + Method to generate a single addon. + + :param name: name of an addon to generate from the stubs dictionary + :param path: path where addon and .xpi should be generated + + Returns the file-path of the addon's .xpi file + """ + + if name in stubs.keys(): + addon = name + else: + # If `name` is not in listed stubs, raise exception + raise IOError('Requested addon stub does not exist') + + # Generate directory structure for addon + try: + if path: + tmpdir = path + else: + tmpdir = tempfile.mkdtemp() + addon_dir = os.path.join(tmpdir, addon[:-4]) + os.mkdir(addon_dir) + install_rdf = os.path.join(addon_dir, 'install.rdf') + xpi = os.path.join(tmpdir, addon) + except IOError: + raise IOError('Could not generate directory structure for addon stub.') + + # Write install.rdf for addon + with open(install_rdf, 'w') as f: + f.write(stubs[addon]) + # Generate the .xpi for the addon + with zipfile.ZipFile(xpi, 'w') as x: + x.write(install_rdf, install_rdf[len(addon_dir):]) + + return xpi + +def generate_invalid_addon(path=None): + """ + Method to create an invalid addon + + Returns the file-path to the .xpi of an invalid addon + """ + return generate_addon(name='empty-invalid.xpi', path=path) + +def generate_manifest(path=None): + + if path: + tmpdir = path + else: + tmpdir = tempfile.mkdtemp() + + addon_list = ['empty-0-1.xpi', 'another-empty-0-1.xpi'] + for a in addon_list: + generate_addon(a, tmpdir) + + manifest = os.path.join(tmpdir, 'manifest.ini') + with open(manifest, 'w') as f: + for a in addon_list: + f.write('[' + a + ']\n') + + return manifest
--- a/testing/mozbase/mozprofile/tests/bug785146.py +++ b/testing/mozbase/mozprofile/tests/bug785146.py @@ -2,24 +2,20 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozfile import os import shutil +import sqlite3 import tempfile import unittest from mozprofile.permissions import Permissions -try: - import sqlite3 -except ImportError: - from pysqlite2 import dbapi2 as sqlite3 - class PermissionsTest(unittest.TestCase): locations = """http://mochi.test:8888 primary,privileged http://127.0.0.1:80 noxul http://127.0.0.1:8888 privileged """
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/install_manifests/another-empty-0-1.rdf @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + <Description about="urn:mozilla:install-manifest"> + <em:id>another-test-empty@quality.mozilla.org</em:id> + <em:version>0.1</em:version> + <em:name>Another Test Extension (empty)</em:name> + <em:creator>Mozilla QA</em:creator> + <em:homepageURL>http://quality.mozilla.org</em:homepageURL> + <em:type>2</em:type> + + <!-- Firefox --> + <em:targetApplication> + <Description> + <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> + <em:minVersion>3.5.*</em:minVersion> + <em:maxVersion>*</em:maxVersion> + </Description> + </em:targetApplication> + </Description> +</RDF> +
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/install_manifests/empty-0-1.rdf @@ -0,0 +1,21 @@ +<?xml version="1.0"?> +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + <Description about="urn:mozilla:install-manifest"> + <em:id>test-empty@quality.mozilla.org</em:id> + <em:version>0.1</em:version> + <em:name>Test Extension (empty)</em:name> + <em:creator>Mozilla QA</em:creator> + <em:homepageURL>http://quality.mozilla.org</em:homepageURL> + <em:type>2</em:type> + + <!-- Firefox --> + <em:targetApplication> + <Description> + <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> + <em:minVersion>3.5.*</em:minVersion> + <em:maxVersion>*</em:maxVersion> + </Description> + </em:targetApplication> + </Description> +</RDF>
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/install_manifests/empty-0-2.rdf @@ -0,0 +1,21 @@ +<?xml version="1.0"?> +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + <Description about="urn:mozilla:install-manifest"> + <em:id>test-empty@quality.mozilla.org</em:id> + <em:version>0.2</em:version> + <em:name>Test Extension (empty)</em:name> + <em:creator>Mozilla QA</em:creator> + <em:homepageURL>http://quality.mozilla.org</em:homepageURL> + <em:type>2</em:type> + + <!-- Firefox --> + <em:targetApplication> + <Description> + <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> + <em:minVersion>3.5.*</em:minVersion> + <em:maxVersion>*</em:maxVersion> + </Description> + </em:targetApplication> + </Description> +</RDF>
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/install_manifests/empty-invalid.rdf @@ -0,0 +1,23 @@ +<?xml version="1.0"?> +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + <Description about="urn:mozilla:install-manifest"> + <em:id>test-empty@quality.mozilla.org</em:id> + <!-- Invalid plugin version --> + <em:version>0.NOPE</em:version> + <em:name>Test Extension (empty)</em:name> + <em:creator>Mozilla QA</em:creator> + <em:homepageURL>http://quality.mozilla.org</em:homepageURL> + <em:type>2</em:type> + + <!-- Firefox --> + <em:targetApplication> + <Description> + <!-- Invalid target application string --> + <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> + <em:minVersion>3.5.*</em:minVersion> + <em:maxVersion>*</em:maxVersion> + </Description> + </em:targetApplication> + </Description> +</RDF>
--- a/testing/mozbase/mozprofile/tests/manifest.ini +++ b/testing/mozbase/mozprofile/tests/manifest.ini @@ -2,8 +2,10 @@ [server_locations.py] [test_preferences.py] [permissions.py] [bug758250.py] [test_nonce.py] [bug785146.py] [test_clone_cleanup.py] [test_webapps.py] +[test_profile_view.py] +[test_addons.py]
--- a/testing/mozbase/mozprofile/tests/permissions.py +++ b/testing/mozbase/mozprofile/tests/permissions.py @@ -2,23 +2,20 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozfile import os import shutil +import sqlite3 import tempfile import unittest from mozprofile.permissions import Permissions -try: - import sqlite3 -except ImportError: - from pysqlite2 import dbapi2 as sqlite3 class PermissionsTest(unittest.TestCase): locations = """http://mochi.test:8888 primary,privileged http://127.0.0.1:80 noxul http://127.0.0.1:8888 privileged """
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/test_addons.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import addon_stubs +import mozprofile +import mozfile +import tempfile +import os +import unittest +from manifestparser import ManifestParser + + +class TestAddonsManager(unittest.TestCase): + """ Class to test mozprofile.addons.AddonManager """ + + def setUp(self): + self.profile = mozprofile.profile.Profile() + self.am = mozprofile.addons.AddonManager(profile=self.profile.profile) + + def test_install_from_path(self): + + addons_to_install = [] + addons_installed = [] + + # Generate installer stubs and install them + tmpdir = tempfile.mkdtemp() + for t in ['empty-0-1.xpi', 'another-empty-0-1.xpi']: + temp_addon = addon_stubs.generate_addon(name=t, path=tmpdir) + addons_to_install.append(self.am.addon_details(temp_addon)['id']) + self.am.install_from_path(temp_addon) + # Generate a list of addons installed in the profile + addons_installed = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join( + self.profile.profile, 'extensions', 'staged'))] + self.assertEqual(addons_to_install.sort(), addons_installed.sort()) + # Cleanup the temporary addon directories + mozfile.rmtree(tmpdir) + + @unittest.skip("Feature not implemented as part of AddonManger") + def test_install_from_path_error(self): + """ Check install_from_path raises an error with an invalid addon""" + + temp_addon = addon_stubs.generate_invalid_addon() + # This should raise an error here + self.am.install_from_path(temp_addon) + + def test_install_from_manifest(self): + + temp_manifest = addon_stubs.generate_manifest() + m = ManifestParser() + m.read(temp_manifest) + addons = m.get() + # Obtain details of addons to install from the manifest + addons_to_install = [self.am.addon_details(x['path'])['id'] for x in addons] + + self.am.install_from_manifest(temp_manifest) + # Generate a list of addons installed in the profile + addons_installed = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join( + self.profile.profile, 'extensions', 'staged'))] + self.assertEqual(addons_installed.sort(), addons_to_install.sort()) + # Cleanup the temporary addon and manifest directories + mozfile.rmtree(os.path.dirname(temp_manifest)) + + @unittest.skip("Bug 900154") + def test_clean_addons(self): + + addon_one = addon_stubs.generate_addon('empty-0-1.xpi') + addon_two = addon_stubs.generate_addon('another-empty-0-1.xpi') + + self.am.install_addons(addon_one) + installed_addons = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join( + self.profile.profile, 'extensions', 'staged'))] + + # Create a new profile based on an existing profile + # Install an extra addon in the new profile + # Cleanup addons + duplicate_profile = mozprofile.profile.Profile(profile=self.profile.profile, + addons=addon_two) + duplicate_profile.addon_manager.clean_addons() + + addons_after_cleanup = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join( + duplicate_profile.profile, 'extensions', 'staged'))] + # New addons installed should be removed by clean_addons() + self.assertEqual(installed_addons, addons_after_cleanup) + + def test_noclean(self): + """test `restore=True/False` functionality""" + + profile = tempfile.mkdtemp() + tmpdir = tempfile.mkdtemp() + try: + + # empty initially + self.assertFalse(bool(os.listdir(profile))) + + # make an addon + stub = addon_stubs.generate_addon(name='empty-0-1.xpi', + path=tmpdir) + + # install it with a restore=True AddonManager + addons = mozprofile.addons.AddonManager(profile, restore=True) + addons.install_from_path(stub) + + # now its there + self.assertEqual(os.listdir(profile), ['extensions']) + extensions = os.path.join(profile, 'extensions', 'staged') + self.assertTrue(os.path.exists(extensions)) + contents = os.listdir(extensions) + self.assertEqual(len(contents), 1) + + # del addons; now its gone though the directory tree exists + del addons + self.assertEqual(os.listdir(profile), ['extensions']) + self.assertTrue(os.path.exists(extensions)) + contents = os.listdir(extensions) + self.assertEqual(len(contents), 0) + + finally: + mozfile.rmtree(tmpdir) + mozfile.rmtree(profile) + + +if __name__ == '__main__': + unittest.main()
new file mode 100644 --- /dev/null +++ b/testing/mozbase/mozprofile/tests/test_profile_view.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozfile +import mozprofile +import os +import tempfile +import unittest + +here = os.path.dirname(os.path.abspath(__file__)) + +class TestProfilePrint(unittest.TestCase): + + def test_profileprint(self): + """ + test the summary function + """ + + keys = set(['Files', 'Path', 'user.js']) + ff_prefs = mozprofile.FirefoxProfile.preferences # shorthand + pref_string = '\n'.join(['%s: %s' % (key, ff_prefs[key]) + for key in sorted(ff_prefs.keys())]) + + tempdir = tempfile.mkdtemp() + try: + profile = mozprofile.FirefoxProfile(tempdir) + parts = profile.summary(return_parts=True) + parts = dict(parts) + + self.assertEqual(parts['Path'], tempdir) + self.assertEqual(set(parts.keys()), keys) + self.assertEqual(pref_string, parts['user.js'].strip()) + + except: + raise + finally: + mozfile.rmtree(tempdir) + + def test_strcast(self): + """ + test casting to a string + """ + + profile = mozprofile.Profile() + self.assertEqual(str(profile), profile.summary()) + + def test_profile_diff(self): + profile1 = mozprofile.Profile() + profile2 = mozprofile.Profile(preferences=dict(foo='bar')) + + # diff a profile against itself; no difference + self.assertEqual([], mozprofile.diff(profile1, profile1)) + + # diff two profiles + diff = dict(mozprofile.diff(profile1, profile2)) + self.assertEqual(diff.keys(), ['user.js']) + lines = [line.strip() for line in diff['user.js'].splitlines()] + self.assertTrue('+foo: bar' in lines) + + # diff a blank vs FirefoxProfile + ff_profile = mozprofile.FirefoxProfile() + diff = dict(mozprofile.diff(profile2, ff_profile)) + self.assertEqual(diff.keys(), ['user.js']) + lines = [line.strip() for line in diff['user.js'].splitlines()] + self.assertTrue('-foo: bar' in lines) + ff_pref_lines = ['+%s: %s' % (key, value) + for key, value in mozprofile.FirefoxProfile.preferences.items()] + self.assertTrue(set(ff_pref_lines).issubset(lines)) + +if __name__ == '__main__': + unittest.main()
--- a/testing/mozbase/mozrunner/mozrunner/local.py +++ b/testing/mozbase/mozrunner/mozrunner/local.py @@ -76,18 +76,18 @@ class LocalRunner(Runner): clean_profile=True, process_class=None): profile = cls.profile_class(**(profile_args or {})) return cls(profile, binary=binary, cmdargs=cmdargs, env=env, kp_kwargs=kp_kwargs, clean_profile=clean_profile, process_class=process_class) def __init__(self, profile, binary, cmdargs=None, env=None, kp_kwargs=None, clean_profile=None, process_class=None): - super(LocalRunner, self).__init__(profile, clean_profile=clean_profile, kp_kwargs=None, - process_class=process_class, env=None) + super(LocalRunner, self).__init__(profile, clean_profile=clean_profile, kp_kwargs=kp_kwargs, + process_class=process_class, env=env) # find the binary self.binary = binary if not self.binary: raise Exception("Binary not specified") if not os.path.exists(self.binary): raise OSError("Binary path does not exist: %s" % self.binary) @@ -101,20 +101,23 @@ class LocalRunner(Runner): self.binary = os.path.join(self.binary, "Contents/MacOS/", info['CFBundleExecutable']) self.cmdargs = cmdargs or [] _cmdargs = [i for i in self.cmdargs if i != '-foreground'] if len(_cmdargs) != len(self.cmdargs): # foreground should be last; see - # - https://bugzilla.mozilla.org/show_bug.cgi?id=625614 - # - https://bugzilla.mozilla.org/show_bug.cgi?id=626826 + # https://bugzilla.mozilla.org/show_bug.cgi?id=625614 self.cmdargs = _cmdargs self.cmdargs.append('-foreground') + if mozinfo.isMac and '-foreground' not in self.cmdargs: + # runner should specify '-foreground' on Mac; see + # https://bugzilla.mozilla.org/show_bug.cgi?id=916512 + self.cmdargs.append('-foreground') # process environment if env is None: self.env = os.environ.copy() else: self.env = env.copy() # allows you to run an instance of Firefox separately from any other instances self.env['MOZ_NO_REMOTE'] = '1'
--- a/testing/mozbase/mozrunner/mozrunner/remote.py +++ b/testing/mozbase/mozrunner/mozrunner/remote.py @@ -183,19 +183,25 @@ class B2GRunner(RemoteRunner): def on_output(self, line): print line match = re.findall(r"TEST-START \| ([^\s]*)", line) if match: self.last_test = match[-1] def on_timeout(self): - self.log.testFail("%s | application timed " - "out after %s seconds with no output", - self.last_test, self.timeout) + msg = "%s | application timed out after %s seconds" + + if self.timeout: + timeout = self.timeout + else: + timeout = self.outputTimeout + msg = "%s with no output" % msg + + self.log.testFail(msg % (self.last_test, timeout)) def _reboot_device(self): serial, status = self._get_device_status() self.dm.shellCheckOutput(['/system/bin/reboot']) # The reboot command can return while adb still thinks the device is # connected, so wait a little bit for it to disconnect from adb. time.sleep(10)
--- a/testing/mozbase/mozrunner/setup.py +++ b/testing/mozbase/mozrunner/setup.py @@ -1,26 +1,26 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import sys from setuptools import setup PACKAGE_NAME = "mozrunner" -PACKAGE_VERSION = '5.22' +PACKAGE_VERSION = '5.25' desc = """Reliable start/stop/configuration of Mozilla Applications (Firefox, Thunderbird, etc.)""" -deps = ['mozcrash >= 0.3', +deps = ['mozcrash >= 0.10', 'mozdevice >= 0.28', - 'mozinfo >= 0.4', + 'mozinfo >= 0.7', 'mozlog >= 1.3', 'mozprocess >= 0.8', - 'mozprofile >= 0.11', + 'mozprofile >= 0.16', ] # we only support python 2 right now assert sys.version_info[0] == 2 setup(name=PACKAGE_NAME, version=PACKAGE_VERSION, description=desc,