Bug 1471888 - [mozprofile] Add support for Python 3. r=davehunt
☠☠ backed out by db2ac775e38a ☠ ☠
authorPavel Slepushkin <slepushkin@yandex.ru>
Tue, 11 Sep 2018 11:34:50 +0000
changeset 435658 30ec08a539919dc56d4a2930885f6ea49a80dc70
parent 435657 19e2c611e88918e24f64fc93c4a599432dece8f1
child 435659 ee581fe311ed557cc6dde5f510ff93255cf0a9c9
push id34617
push userbtara@mozilla.com
push dateTue, 11 Sep 2018 22:11:01 +0000
treeherdermozilla-central@ba471b021d0e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1471888
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1471888 - [mozprofile] Add support for Python 3. r=davehunt Bug 1471888 - [mozprofile] Add support for Python 3. Differential Revision: https://phabricator.services.mozilla.com/D5187
testing/mozbase/mozprofile/mozprofile/addons.py
testing/mozbase/mozprofile/mozprofile/permissions.py
testing/mozbase/mozprofile/mozprofile/prefs.py
testing/mozbase/mozprofile/mozprofile/profile.py
testing/mozbase/mozprofile/setup.cfg
testing/mozbase/mozprofile/setup.py
testing/mozbase/mozprofile/tests/manifest.ini
testing/mozbase/mozprofile/tests/test_preferences.py
testing/mozbase/mozprofile/tests/test_profile_view.py
--- a/testing/mozbase/mozprofile/mozprofile/addons.py
+++ b/testing/mozbase/mozprofile/mozprofile/addons.py
@@ -6,24 +6,24 @@ from __future__ import absolute_import
 
 import json
 import os
 import sys
 import shutil
 import tempfile
 import zipfile
 import hashlib
+import binascii
 from xml.dom import minidom
-
 from six import reraise, string_types
 
 import mozfile
 from mozlog.unstructured import getLogger
 
-_SALT = os.urandom(32).encode('hex')
+_SALT = binascii.hexlify(os.urandom(32))
 _TEMPORARY_ADDON_SUFFIX = "@temporary-addon"
 
 # Logger for 'mozprofile.addons' module
 module_logger = getLogger(__name__)
 
 
 class AddonFormatError(Exception):
     """Exception for not well-formed add-on manifest files"""
@@ -198,17 +198,17 @@ class AddonManager(object):
         if isinstance(addons, string_types):
             addons = [addons]
         for addon in set(addons):
             self._install_addon(addon, **kwargs)
 
     @classmethod
     def _gen_iid(cls, addon_path):
         hash = hashlib.sha1(_SALT)
-        hash.update(addon_path)
+        hash.update(addon_path.encode())
         return hash.hexdigest() + _TEMPORARY_ADDON_SUFFIX
 
     @classmethod
     def addon_details(cls, addon_path):
         """
         Returns a dictionary of details about the addon.
 
         :param addon_path: path to the add-on directory or XPI
@@ -257,34 +257,34 @@ class AddonManager(object):
                 # it will cause an exception thrown in Python 2.6.
                 try:
                     compressed_file = zipfile.ZipFile(addon_path, 'r')
                     filenames = [f.filename for f in (compressed_file).filelist]
                     if 'install.rdf' in filenames:
                         manifest = compressed_file.read('install.rdf')
                     elif 'manifest.json' in filenames:
                         is_webext = True
-                        manifest = compressed_file.read('manifest.json')
+                        manifest = compressed_file.read('manifest.json').decode()
                         manifest = json.loads(manifest)
                     else:
                         raise KeyError("No manifest")
                 finally:
                     compressed_file.close()
             elif os.path.isdir(addon_path):
                 try:
                     with open(os.path.join(addon_path, 'install.rdf')) as f:
                         manifest = f.read()
                 except IOError:
                     with open(os.path.join(addon_path, 'manifest.json')) as f:
                         manifest = json.loads(f.read())
                         is_webext = True
             else:
                 raise IOError('Add-on path is neither an XPI nor a directory: %s' % addon_path)
         except (IOError, KeyError) as e:
-            reraise(AddonFormatError(str(e)), None, sys.exc_info()[2])
+            reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2])
 
         if is_webext:
             details['version'] = manifest['version']
             details['name'] = manifest['name']
             try:
                 details['id'] = manifest['applications']['gecko']['id']
             except KeyError:
                 details['id'] = cls._gen_iid(addon_path)
@@ -304,17 +304,17 @@ class AddonManager(object):
                     if entry in details.keys():
                         details.update({entry: value})
                 for node in description.childNodes:
                     # Remove the namespace prefix from the tag for comparison
                     entry = node.nodeName.replace(em, "")
                     if entry in details.keys():
                         details.update({entry: get_text(node)})
             except Exception as e:
-                reraise(AddonFormatError(str(e)), None, sys.exc_info()[2])
+                reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2])
 
         # turn unpack into a true/false value
         if isinstance(details['unpack'], string_types):
             details['unpack'] = details['unpack'].lower() == 'true'
 
         # If no ID is set, the add-on is invalid
         if details.get('id') is None and not is_webext:
             raise AddonFormatError('Add-on id could not be found.')
--- a/testing/mozbase/mozprofile/mozprofile/permissions.py
+++ b/testing/mozbase/mozprofile/mozprofile/permissions.py
@@ -10,16 +10,17 @@ add permissions to the profile
 from __future__ import absolute_import
 
 import codecs
 import os
 import sqlite3
 
 from six import string_types
 from six.moves.urllib import parse
+import six
 
 __all__ = ['MissingPrimaryLocationError', 'MultiplePrimaryLocationsError',
            'DEFAULT_PORTS', 'DuplicateLocationError', 'BadPortLocationError',
            'LocationsSyntaxError', 'Location', 'ServerLocations',
            'Permissions']
 
 # http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l28
 DEFAULT_PORTS = {'http': '8888',
@@ -266,17 +267,17 @@ class Permissions(object):
             cursor.execute("PRAGMA user_version=3;")
         else:
             statement = "INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0)"
             cursor.execute("PRAGMA user_version=2;")
 
         for location in locations:
             # set the permissions
             permissions = {'allowXULXBL': 'noxul' not in location.options}
-            for perm, allow in permissions.iteritems():
+            for perm, allow in six.iteritems(permissions):
                 if allow:
                     permission_type = 1
                 else:
                     permission_type = 2
 
                 if using_origin:
                     # This is a crude approximation of the origin generation
                     # logic from ContentPrincipal and nsStandardURL. It should
--- a/testing/mozbase/mozprofile/mozprofile/prefs.py
+++ b/testing/mozbase/mozprofile/mozprofile/prefs.py
@@ -163,16 +163,19 @@ class Preferences(object):
         :param interpolation: If provided, a dict that will be passed
                               to str.format to interpolate preference values.
         """
 
         marker = '##//'  # magical marker
         lines = [i.strip() for i in mozfile.load(path).readlines()]
         _lines = []
         for line in lines:
+            # decode bytes in case of URL processing
+            if isinstance(line, bytes):
+                line = line.decode()
             if not line.startswith(pref_setter):
                 continue
             if '//' in line:
                 line = line.replace('//', marker)
             _lines.append(line)
         string = '\n'.join(_lines)
 
         # skip trailing comments
--- a/testing/mozbase/mozprofile/mozprofile/profile.py
+++ b/testing/mozbase/mozprofile/mozprofile/profile.py
@@ -9,17 +9,17 @@ import os
 import platform
 import tempfile
 import time
 import uuid
 from abc import ABCMeta, abstractmethod, abstractproperty
 from shutil import copytree
 
 import mozfile
-from six import string_types
+from six import string_types, python_2_unicode_compatible
 
 from .addons import AddonManager
 from .permissions import Permissions
 from .prefs import Preferences
 
 __all__ = ['BaseProfile',
            'ChromeProfile',
            'Profile',
@@ -145,16 +145,17 @@ class BaseProfile(object):
         c.create_new = True  # deletes a cloned profile when restore is True
         return c
 
     def exists(self):
         """returns whether the profile exists or not"""
         return os.path.exists(self.profile)
 
 
+@python_2_unicode_compatible
 class Profile(BaseProfile):
     """Handles all operations regarding profile.
 
     Creating new profiles, installing add-ons, setting preferences and
     handling cleanup.
 
     The files associated with the profile will be removed automatically after
     the object is garbage collected: ::
@@ -416,19 +417,16 @@ class Profile(BaseProfile):
         if return_parts:
             return parts
 
         retval = '%s\n' % ('\n\n'.join(['[%s]: %s' % (key, value)
                                         for key, value in parts]))
         return retval
 
     def __str__(self):
-        return unicode(self).encode('utf-8')
-
-    def __unicode__(self):
         return self.summary()
 
 
 class FirefoxProfile(Profile):
     """Specialized Profile subclass for Firefox"""
 
     preferences = {  # Don't automatically update the application
         'app.update.disabledForTesting': True,
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozprofile/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
--- a/testing/mozbase/mozprofile/setup.py
+++ b/testing/mozbase/mozprofile/setup.py
@@ -2,17 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 from setuptools import setup
 
 PACKAGE_NAME = 'mozprofile'
-PACKAGE_VERSION = '1.1.0'
+PACKAGE_VERSION = '2.0.0'
 
 deps = [
     'mozfile==1.*',
     'mozlog==3.*',
     'six>=1.10.0,<2',
 ]
 
 setup(name=PACKAGE_NAME,
@@ -20,17 +20,17 @@ setup(name=PACKAGE_NAME,
       description="Library to create and modify Mozilla application profiles",
       long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html",
       classifiers=['Environment :: Console',
                    'Intended Audience :: Developers',
                    'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
                    'Natural Language :: English',
                    'Operating System :: OS Independent',
                    'Programming Language :: Python :: 2.7',
-                   'Programming Language :: Python :: 3',
+                   'Programming Language :: Python :: 3.5',
                    'Topic :: Software Development :: Libraries :: Python Modules',
                    ],
       keywords='mozilla',
       author='Mozilla Automation and Tools team',
       author_email='tools@lists.mozilla.org',
       url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase',
       license='MPL 2.0',
       packages=['mozprofile'],
--- a/testing/mozbase/mozprofile/tests/manifest.ini
+++ b/testing/mozbase/mozprofile/tests/manifest.ini
@@ -1,11 +1,10 @@
 [DEFAULT]
 subsuite = mozbase, os == "linux"
-skip-if = python == 3
 [test_addonid.py]
 [test_server_locations.py]
 [test_preferences.py]
 [test_permissions.py]
 [test_bug758250.py]
 [test_nonce.py]
 [test_clone_cleanup.py]
 [test_profile.py]
--- a/testing/mozbase/mozprofile/tests/test_preferences.py
+++ b/testing/mozbase/mozprofile/tests/test_preferences.py
@@ -67,18 +67,17 @@ def compare_generated(run_command):
     return inner
 
 
 def test_basic_prefs(compare_generated):
     """test setting a pref from the command line entry point"""
 
     _prefs = {"browser.startup.homepage": "http://planet.mozilla.org/"}
     commandline = []
-    _prefs = _prefs.items()
-    for pref, value in _prefs:
+    for pref, value in _prefs.items():
         commandline += ["--pref", "%s:%s" % (pref, value)]
     compare_generated(_prefs, commandline)
 
 
 def test_ordered_prefs(compare_generated):
     """ensure the prefs stay in the right order"""
     _prefs = [("browser.startup.homepage", "http://planet.mozilla.org/"),
               ("zoom.minPercent", 30),
@@ -95,18 +94,18 @@ def test_ini(compare_generated):
     # write the .ini file
     _ini = """[DEFAULT]
 browser.startup.homepage = http://planet.mozilla.org/
 
 [foo]
 browser.startup.homepage = http://github.com/
 """
     try:
-        fd, name = tempfile.mkstemp(suffix='.ini')
-        os.write(fd, _ini)
+        fd, name = tempfile.mkstemp(suffix='.ini', text=True)
+        os.write(fd, _ini.encode())
         os.close(fd)
         commandline = ["--preferences", name]
 
         # test the [DEFAULT] section
         _prefs = {'browser.startup.homepage': 'http://planet.mozilla.org/'}
         compare_generated(_prefs, commandline)
 
         # test a specific section
@@ -124,18 +123,18 @@ def test_ini_keep_case(compare_generated
     Read a preferences config file with a preference in camel-case style.
     Check that the read preference name has not been lower-cased
     """
     # write the .ini file
     _ini = """[DEFAULT]
 general.warnOnAboutConfig = False
 """
     try:
-        fd, name = tempfile.mkstemp(suffix='.ini')
-        os.write(fd, _ini)
+        fd, name = tempfile.mkstemp(suffix='.ini', text=True)
+        os.write(fd, _ini.encode())
         os.close(fd)
         commandline = ["--preferences", name]
 
         # test the [DEFAULT] section
         _prefs = {'general.warnOnAboutConfig': 'False'}
         compare_generated(_prefs, commandline)
 
     finally:
@@ -289,44 +288,44 @@ def test_can_read_prefs_with_multiline_c
 * If you make changes to this file while the application is running,
 * the changes will be overwritten when the application exits.
 *
 * To make a manual change to preferences, you can visit the URL about:config
 */
 
 user_pref("webgl.enabled_for_all_sites", true);
 user_pref("webgl.force-enabled", true);
-""")
+""".encode())
         assert Preferences.read_prefs(user_js.name) == [
                 ('webgl.enabled_for_all_sites', True),
                 ('webgl.force-enabled', True)
         ]
     finally:
         mozfile.remove(user_js.name)
 
 
 def test_json(compare_generated):
     _prefs = {"browser.startup.homepage": "http://planet.mozilla.org/"}
     json = '{"browser.startup.homepage": "http://planet.mozilla.org/"}'
 
     # just repr it...could use the json module but we don't need it here
     with mozfile.NamedTemporaryFile(suffix='.json') as f:
-        f.write(json)
+        f.write(json.encode())
         f.flush()
 
         commandline = ["--preferences", f.name]
         compare_generated(_prefs, commandline)
 
 
 def test_json_datatypes():
     # minPercent is at 30.1 to test if non-integer data raises an exception
     json = """{"zoom.minPercent": 30.1, "zoom.maxPercent": 300}"""
 
     with mozfile.NamedTemporaryFile(suffix='.json') as f:
-        f.write(json)
+        f.write(json.encode())
         f.flush()
 
         with pytest.raises(PreferencesReadError):
             Preferences.read_json(f._path)
 
 
 def test_prefs_write():
     """test that the Preferences.write() method correctly serializes preferences"""
@@ -337,17 +336,17 @@ def test_prefs_write():
 
     # make a Preferences manager with the testing preferences
     preferences = Preferences(_prefs)
 
     # write them to a temporary location
     path = None
     read_prefs = None
     try:
-        with mozfile.NamedTemporaryFile(suffix='.js', delete=False) as f:
+        with mozfile.NamedTemporaryFile(suffix='.js', delete=False, mode='w+t') as f:
             path = f.name
             preferences.write(f, _prefs)
 
         # read them back and ensure we get what we put in
         read_prefs = dict(Preferences.read_prefs(path))
 
     finally:
         # cleanup
--- a/testing/mozbase/mozprofile/tests/test_profile_view.py
+++ b/testing/mozbase/mozprofile/tests/test_profile_view.py
@@ -3,16 +3,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import mozprofile
 import os
+import sys
+import pytest
 
 import mozunit
 
 here = os.path.dirname(os.path.abspath(__file__))
 
 
 def test_profileprint(tmpdir):
     """
@@ -32,42 +34,47 @@ def test_profileprint(tmpdir):
     assert parts['Path'] == tmpdir
     assert set(parts.keys()) == keys
     assert pref_string == parts['user.js'].strip()
 
 
 def test_str_cast():
     """Test casting to a string."""
     profile = mozprofile.Profile()
-    assert str(profile) == profile.summary().encode("utf-8")
+    if sys.version_info[0] >= 3:
+        assert str(profile) == profile.summary()
+    else:
+        assert str(profile) == profile.summary().encode("utf-8")
 
 
+@pytest.mark.skipif(sys.version_info[0] >= 3,
+                    reason="no unicode() operator starting from python3")
 def test_unicode_cast():
     """Test casting to a unicode string."""
     profile = mozprofile.Profile()
     assert unicode(profile) == profile.summary()
 
 
 def test_profile_diff():
     profile1 = mozprofile.Profile()
     profile2 = mozprofile.Profile(preferences=dict(foo='bar'))
 
     # diff a profile against itself; no difference
     assert mozprofile.diff(profile1, profile1) == []
 
     # diff two profiles
     diff = dict(mozprofile.diff(profile1, profile2))
-    assert diff.keys() == ['user.js']
+    assert list(diff.keys()) == ['user.js']
     lines = [line.strip() for line in diff['user.js'].splitlines()]
     assert '+foo: bar' in lines
 
     # diff a blank vs FirefoxProfile
     ff_profile = mozprofile.FirefoxProfile()
     diff = dict(mozprofile.diff(profile2, ff_profile))
-    assert diff.keys() == ['user.js']
+    assert list(diff.keys()) == ['user.js']
     lines = [line.strip() for line in diff['user.js'].splitlines()]
     assert '-foo: bar' in lines
     ff_pref_lines = ['+%s: %s' % (key, value)
                      for key, value in mozprofile.FirefoxProfile.preferences.items()]
     assert set(ff_pref_lines).issubset(lines)
 
 
 if __name__ == '__main__':