Bug 1595982 - make mozharness::base::script.py python3 compatible r=aki
authorEdwin Takahashi <egao@mozilla.com>
Fri, 15 Nov 2019 23:52:04 +0000
changeset 502306 ee0fdf5558a849da1045f3fe3c9b6f7a6eb76859
parent 502305 74ac84aa60fd048101ff962b8441334a3770116a
child 502307 9cb264733132c914ab9df9e06c226f967abe8e00
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1595982
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1595982 - make mozharness::base::script.py python3 compatible r=aki Changes: Run `isort` and `autopep8` for automatic fixes of import and code formatting. Replace deprecated imports with updated imports for python3 and wrap the attempt in a `try/except` clause for backwards compatibility. Wherever possible, directly import the object with same name between python2/python3 versions to simplify the main code (eg. HTTPError). Differential Revision: https://phabricator.services.mozilla.com/D52791
testing/mozharness/mozharness/base/script.py
testing/mozharness/tox.ini
--- a/testing/mozharness/mozharness/base/script.py
+++ b/testing/mozharness/mozharness/base/script.py
@@ -7,63 +7,77 @@
 
 script.py, along with config.py and log.py, represents the core of
 mozharness.
 """
 
 from __future__ import print_function
 
 import codecs
-from contextlib import contextmanager
 import datetime
 import errno
 import fnmatch
 import functools
 import gzip
+import hashlib
 import inspect
 import itertools
 import os
 import platform
 import pprint
 import re
 import shutil
 import socket
 import subprocess
 import sys
 import tarfile
 import time
 import traceback
-import urllib2
 import zipfile
-import httplib
-import urlparse
-import hashlib
 import zlib
+from contextlib import contextmanager
+from io import BytesIO
+
+from mozprocess import ProcessHandler
+
+import mozinfo
+from mozharness.base.config import BaseConfig
+from mozharness.base.log import (DEBUG, ERROR, FATAL, INFO, WARNING,
+                                 ConsoleLogger, LogMixin, MultiFileLogger,
+                                 OutputParser, SimpleFileLogger)
+
+try:
+    import httplib
+except ImportError:
+    import http.client as httplib
+try:
+    import simplejson as json
+except ImportError:
+    import json
+try:
+    from urllib2 import quote, urlopen, Request
+except ImportError:
+    from urllib.request import quote, urlopen, Request
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 if os.name == 'nt':
     import locale
     try:
         import win32file
         import win32api
         PYWIN32 = True
     except ImportError:
         PYWIN32 = False
 
 try:
-    import simplejson as json
-    assert json
+    from urllib2 import HTTPError, URLError
 except ImportError:
-    import json
-
-from io import BytesIO
-
-import mozinfo
-from mozprocess import ProcessHandler
-from mozharness.base.config import BaseConfig
-from mozharness.base.log import MultiFileLogger, SimpleFileLogger, ConsoleLogger, \
-    LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL, WARNING
+    from urllib.error import HTTPError, URLError
 
 
 class ContentLengthMismatch(Exception):
     pass
 
 
 def platform_name():
     pm = PlatformMixin()
@@ -365,37 +379,37 @@ class ScriptMixin(PlatformMixin):
 
         parsed = urlparse.urlsplit(url.rstrip('/'))
         if parsed.path != '':
             return parsed.path.rsplit('/', 1)[-1]
         else:
             return parsed.netloc
 
     def _urlopen(self, url, **kwargs):
-        """ open the url `url` using `urllib2`.
+        """ open the url `url` using `urllib2`.`
         This method can be overwritten to extend its complexity
 
         Args:
-            url (str | urllib2.Request): url to open
-            kwargs: Arbitrary keyword arguments passed to the `urllib2.urlopen` function.
+            url (str | urllib.request.Request): url to open
+            kwargs: Arbitrary keyword arguments passed to the `urllib.request.urlopen` function.
 
         Returns:
             file-like: file-like object with additional methods as defined in
-                       `urllib2.urlopen`_.
+                       `urllib.request.urlopen`_.
             None: None may be returned if no handler handles the request.
 
         Raises:
             urllib2.URLError: on errors
 
-        .. _urllib2.urlopen:
+        .. urillib.request.urlopen:
         https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
         """
         # http://bugs.python.org/issue13359 - urllib2 does not automatically quote the URL
-        url_quoted = urllib2.quote(url, safe='%/:=&?~#+!$,;\'@()*[]|')
-        return urllib2.urlopen(url_quoted, **kwargs)
+        url_quoted = quote(url, safe='%/:=&?~#+!$,;\'@()*[]|')
+        return urlopen(url_quoted, **kwargs)
 
     def fetch_url_into_memory(self, url):
         ''' Downloads a file from a url into memory instead of disk.
 
         Args:
             url (str): URL path where the file to be downloaded is located.
 
         Raises:
@@ -417,31 +431,31 @@ class ScriptMixin(PlatformMixin):
 
             content_length = os.stat(path).st_size
 
             # In case we're referrencing a file without file://
             if parsed_url.scheme == '':
                 url = 'file://%s' % os.path.abspath(url)
                 parsed_url = urlparse.urlparse(url)
 
-        request = urllib2.Request(url)
+        request = Request(url)
         # When calling fetch_url_into_memory() you should retry when we raise
         # one of these exceptions:
         # * Bug 1300663 - HTTPError: HTTP Error 404: Not Found
         # * Bug 1300413 - HTTPError: HTTP Error 500: Internal Server Error
         # * Bug 1300943 - HTTPError: HTTP Error 503: Service Unavailable
         # * Bug 1300953 - URLError: <urlopen error [Errno -2] Name or service not known>
         # * Bug 1301594 - URLError: <urlopen error [Errno 10054] An existing connection was ...
         # * Bug 1301597 - URLError: <urlopen error [Errno 8] _ssl.c:504: EOF occurred in ...
         # * Bug 1301855 - URLError: <urlopen error [Errno 60] Operation timed out>
         # * Bug 1302237 - URLError: <urlopen error [Errno 104] Connection reset by peer>
         # * Bug 1301807 - BadStatusLine: ''
         #
         # Bug 1309912 - Adding timeout in hopes to solve blocking on response.read() (bug 1300413)
-        response = urllib2.urlopen(request, timeout=30)
+        response = urlopen(request, timeout=30)
 
         if parsed_url.scheme in ('http', 'https'):
             content_length = int(response.headers.get('Content-Length'))
 
         response_body = response.read()
         response_body_size = len(response_body)
 
         self.info('Content-Length response header: {}'.format(content_length))
@@ -510,17 +524,17 @@ class ScriptMixin(PlatformMixin):
                 # file, and delete the compressed version.
                 local_file = open(file_name + '.gz', 'wb')
             else:
                 local_file = open(file_name, 'wb')
             while True:
                 block = f.read(1024 ** 2)
                 if not block:
                     if f_length is not None and got_length != f_length:
-                        raise urllib2.URLError(
+                        raise URLError(
                             "Download incomplete; content-length was %d, "
                             "but only received %d" % (f_length, got_length))
                     break
                 local_file.write(block)
                 if f_length is not None:
                     got_length += len(block)
             local_file.close()
             if f.info().get('Content-Encoding') == 'gzip':
@@ -531,20 +545,20 @@ class ScriptMixin(PlatformMixin):
                     # So let's do this the python 2.6 way...
                     try:
                         f_in = gzip.open(file_name + '.gz', 'rb')
                         shutil.copyfileobj(f_in, f_out)
                     finally:
                         f_in.close()
                 os.remove(file_name + '.gz')
             return file_name
-        except urllib2.HTTPError as e:
+        except HTTPError as e:
             self.warning("Server returned status %s %s for %s" % (str(e.code), str(e), url))
             raise
-        except urllib2.URLError as e:
+        except URLError as e:
             self.warning("URL Error: %s" % url)
 
             # Failures due to missing local files won't benefit from retry.
             # Raise the original OSError.
             if isinstance(e.args[0], OSError) and e.args[0].errno == errno.ENOENT:
                 raise e.args[0]
 
             raise
@@ -572,17 +586,17 @@ class ScriptMixin(PlatformMixin):
 
         Returns:
             str: `self._download_file` return value is returned
             unknown: `self.retry` `failure_status` is returned on failure, which
                      defaults to -1
         """
         retry_args = dict(
             failure_status=None,
-            retry_exceptions=(urllib2.HTTPError, urllib2.URLError,
+            retry_exceptions=(HTTPError, URLError,
                               httplib.BadStatusLine,
                               socket.timeout, socket.error),
             error_message="Can't download from %s to %s!" % (url, file_name),
             error_level=error_level,
         )
 
         if retry_config:
             retry_args.update(retry_config)
@@ -723,18 +737,18 @@ class ScriptMixin(PlatformMixin):
             extract_to,
             ', '.join(extract_dirs),
             url,
         ))
 
         # 1) Let's fetch the file
         retry_args = dict(
             retry_exceptions=(
-                urllib2.HTTPError,
-                urllib2.URLError,
+                HTTPError,
+                URLError,
                 httplib.BadStatusLine,
                 socket.timeout,
                 socket.error,
                 ContentLengthMismatch,
             ),
             sleeptime=30,
             attempts=5,
             error_message="Can't download from {}".format(url),
--- a/testing/mozharness/tox.ini
+++ b/testing/mozharness/tox.ini
@@ -1,32 +1,36 @@
 [tox]
-envlist = py27-hg4.3
+envlist = py27-hg4.3, py35-hg4.3
 
 [base]
 deps =
     coverage
     distro
     nose
     rednose
     {toxinidir}/../mozbase/mozlog
 mozbase = {toxinidir}/../mozbase/
-    
+
 
 [testenv]
-basepython = python2.7
 setenv =
     HGRCPATH = {toxinidir}/test/hgrc
     PYTHONPATH = $PYTHONPATH:{[base]mozbase}/manifestparser:{[base]mozbase}/mozfile:{[base]mozbase}/mozinfo:{[base]mozbase}/mozprocess
 
 commands =
     coverage run --source configs,mozharness,scripts --branch {envbindir}/nosetests -v --with-xunit --rednose --force-color {posargs}
 
 [testenv:py27-hg4.3]
 deps =
     {[base]deps}
     mercurial==4.3.1
 
+[testenv:py35-hg4.3]
+deps =
+    {[base]deps}
+    mercurial==4.3.1
+
 [testenv:py27-coveralls]
 deps=
     python-coveralls==2.4.3
 commands=
     coveralls