Bug 1231810 - Fix files not following flake8 convention r=ted
authorSimon Garesste <simon@gareste.fr>
Sun, 10 Jan 2016 11:19:37 +0100
changeset 358112 20e72e5cc536b4ad9e041f3ae298101abb0a3c68
parent 358111 ada4d29af82660cf824a66955245e6dc57ac231c
child 358113 66e1f055ca3e54c981bd116f8d56b00081f12d7e
push id31818
push userarchaeopteryx@coole-files.de
push dateSun, 14 May 2017 16:01:21 +0000
treeherdermozilla-central@3a801856dea9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1231810
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1231810 - Fix files not following flake8 convention r=ted flake8 was reporting the following issues: * expected two blank lines * unexpected spaces around keyword / parameter equals * wrongly imported modules or multiple imports on one line * under-indented lines for visual indent * unidentified name because of a bad call MozReview-Commit-ID: 7sjDoH4Z89L
testing/remotecppunittests.py
testing/runcppunittests.py
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -1,34 +1,36 @@
 #!/usr/bin/env python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import os, sys
+import os
+import sys
 import subprocess
-import tempfile
 from zipfile import ZipFile
 import runcppunittests as cppunittests
 import mozcrash
 import mozfile
 import mozinfo
 import mozlog
 import StringIO
 import posixpath
-from mozdevice import devicemanager, devicemanagerADB
+from mozdevice import devicemanagerADB
 
 try:
     from mozbuild.base import MozbuildObject
     build_obj = MozbuildObject.from_environment()
 except ImportError:
     build_obj = None
 
+
 class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
+
     def __init__(self, devmgr, options, progs):
         cppunittests.CPPUnitTests.__init__(self)
         self.options = options
         self.device = devmgr
         self.remote_test_root = self.device.deviceRoot + "/cppunittests"
         self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
         self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
         self.remote_home_dir = posixpath.join(self.remote_test_root, "h")
@@ -54,68 +56,74 @@ class RemoteCPPUnitTests(cppunittests.CP
     def push_libs(self):
         if self.options.local_apk:
             with mozfile.TemporaryDirectory() as tmpdir:
                 apk_contents = ZipFile(self.options.local_apk)
 
                 for info in apk_contents.infolist():
                     if info.filename.endswith(".so"):
                         print >> sys.stderr, "Pushing %s.." % info.filename
-                        remote_file = posixpath.join(self.remote_bin_dir, os.path.basename(info.filename))
+                        remote_file = posixpath.join(
+                            self.remote_bin_dir, os.path.basename(info.filename))
                         apk_contents.extract(info, tmpdir)
                         local_file = os.path.join(tmpdir, info.filename)
                         with open(local_file) as f:
                             # Decompress xz-compressed file.
                             if f.read(5)[1:] == '7zXZ':
-                                cmd = ['xz', '-df', '--suffix', '.so', local_file]
+                                cmd = [
+                                    'xz', '-df', '--suffix', '.so', local_file]
                                 subprocess.check_output(cmd)
                                 # xz strips the ".so" file suffix.
                                 os.rename(local_file[:-3], local_file)
                         self.device.pushFile(local_file, remote_file)
 
         elif self.options.local_lib:
             for file in os.listdir(self.options.local_lib):
                 if file.endswith(".so"):
                     print >> sys.stderr, "Pushing %s.." % file
                     remote_file = posixpath.join(self.remote_bin_dir, file)
                     local_file = os.path.join(self.options.local_lib, file)
                     self.device.pushFile(local_file, remote_file)
-            # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a"
+            # Additional libraries may be found in a sub-directory such as
+            # "lib/armeabi-v7a"
             for subdir in ["assets", "lib"]:
                 local_arm_lib = os.path.join(self.options.local_lib, subdir)
                 if os.path.isdir(local_arm_lib):
                     for root, dirs, files in os.walk(local_arm_lib):
                         for file in files:
                             if (file.endswith(".so")):
                                 print >> sys.stderr, "Pushing %s.." % file
-                                remote_file = posixpath.join(self.remote_bin_dir, file)
+                                remote_file = posixpath.join(
+                                    self.remote_bin_dir, file)
                                 local_file = os.path.join(root, file)
                                 self.device.pushFile(local_file, remote_file)
 
     def push_progs(self, progs):
         for local_file in progs:
-            remote_file = posixpath.join(self.remote_bin_dir, os.path.basename(local_file))
+            remote_file = posixpath.join(
+                self.remote_bin_dir, os.path.basename(local_file))
             self.device.pushFile(local_file, remote_file)
 
     def build_environment(self):
         env = self.build_core_environment()
         env['LD_LIBRARY_PATH'] = self.remote_bin_dir
-        env["TMPDIR"]=self.remote_tmp_dir
-        env["HOME"]=self.remote_home_dir
+        env["TMPDIR"] = self.remote_tmp_dir
+        env["HOME"] = self.remote_home_dir
         env["MOZILLA_FIVE_HOME"] = self.remote_home_dir
         env["MOZ_XRE_DIR"] = self.remote_bin_dir
         if self.options.add_env:
             for envdef in self.options.add_env:
                 envdef_parts = envdef.split("=", 1)
                 if len(envdef_parts) == 2:
                     env[envdef_parts[0]] = envdef_parts[1]
                 elif len(envdef_parts) == 1:
                     env[envdef_parts[0]] = ""
                 else:
-                    self.log.warning("invalid --addEnv option skipped: %s" % envdef)
+                    self.log.warning(
+                        "invalid --addEnv option skipped: %s" % envdef)
 
         return env
 
     def run_one_test(self, prog, env, symbols_path=None, interactive=False,
                      timeout_factor=1):
         """
         Run a single C++ unit test program remotely.
 
@@ -127,19 +135,21 @@ class RemoteCPPUnitTests(cppunittests.CP
         * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
         basename = os.path.basename(prog)
         remote_bin = posixpath.join(self.remote_bin_dir, basename)
         self.log.test_start(basename)
         buf = StringIO.StringIO()
-        test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
-        returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_home_dir,
-                                       timeout=test_timeout)
+        test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * \
+            timeout_factor
+        returncode = self.device.shell(
+            [remote_bin], buf, env=env, cwd=self.remote_home_dir,
+            timeout=test_timeout)
         self.log.process_output(basename, "\n%s" % buf.getvalue(),
                                 command=[remote_bin])
         with mozfile.TemporaryDirectory() as tempdir:
             self.device.getDirectory(self.remote_home_dir, tempdir)
             if mozcrash.check_for_crashes(tempdir, symbols_path,
                                           test_name=basename):
                 self.log.test_end(basename, status='CRASH', expected='PASS')
                 return False
@@ -147,74 +157,79 @@ class RemoteCPPUnitTests(cppunittests.CP
         if not result:
             self.log.test_end(basename, status='FAIL', expected='PASS',
                               message=("test failed with return code %s" %
                                        returncode))
         else:
             self.log.test_end(basename, status='PASS', expected='PASS')
         return result
 
+
 class RemoteCPPUnittestOptions(cppunittests.CPPUnittestOptions):
+
     def __init__(self):
         cppunittests.CPPUnittestOptions.__init__(self)
         defaults = {}
 
         self.add_option("--deviceIP", action="store",
-                        type = "string", dest = "device_ip",
-                        help = "ip address of remote device to test")
+                        type="string", dest="device_ip",
+                        help="ip address of remote device to test")
         defaults["device_ip"] = None
 
         self.add_option("--devicePort", action="store",
-                        type = "string", dest = "device_port",
-                        help = "port of remote device to test")
+                        type="string", dest="device_port",
+                        help="port of remote device to test")
         defaults["device_port"] = 20701
 
         self.add_option("--noSetup", action="store_false",
-                        dest = "setup",
-                        help = "do not copy any files to device (to be used only if device is already setup)")
+                        dest="setup",
+                        help="do not copy any files to device (to be used only if device is already setup)")
         defaults["setup"] = True
 
         self.add_option("--localLib", action="store",
-                        type = "string", dest = "local_lib",
-                        help = "location of libraries to push -- preferably stripped")
+                        type="string", dest="local_lib",
+                        help="location of libraries to push -- preferably stripped")
         defaults["local_lib"] = None
 
         self.add_option("--apk", action="store",
-                        type = "string", dest = "local_apk",
-                        help = "local path to Fennec APK")
+                        type="string", dest="local_apk",
+                        help="local path to Fennec APK")
         defaults["local_apk"] = None
 
         self.add_option("--localBinDir", action="store",
-                        type = "string", dest = "local_bin",
-                        help = "local path to bin directory")
-        defaults["local_bin"] = build_obj.bindir if build_obj is not None else None
+                        type="string", dest="local_bin",
+                        help="local path to bin directory")
+        defaults[
+            "local_bin"] = build_obj.bindir if build_obj is not None else None
 
-        self.add_option("--remoteTestRoot", action = "store",
-                    type = "string", dest = "remote_test_root",
-                    help = "remote directory to use as test root (eg. /data/local/tests)")
+        self.add_option("--remoteTestRoot", action="store",
+                        type="string", dest="remote_test_root",
+                        help="remote directory to use as test root (eg. /data/local/tests)")
         # /data/local/tests is used because it is usually not possible to set +x permissions
         # on binaries on /mnt/sdcard
         defaults["remote_test_root"] = "/data/local/tests"
 
-        self.add_option("--with-b2g-emulator", action = "store",
-                    type = "string", dest = "with_b2g_emulator",
-                    help = "Start B2G Emulator (specify path to b2g home)")
+        self.add_option("--with-b2g-emulator", action="store",
+                        type="string", dest="with_b2g_emulator",
+                        help="Start B2G Emulator (specify path to b2g home)")
         self.add_option("--emulator", default="arm", choices=["x86", "arm"],
-                    help = "Architecture of emulator to use: x86 or arm")
-        self.add_option("--addEnv", action = "append",
-                    type = "string", dest = "add_env",
-                    help = "additional remote environment variable definitions (eg. --addEnv \"somevar=something\")")
+                        help="Architecture of emulator to use: x86 or arm")
+        self.add_option("--addEnv", action="append",
+                        type="string", dest="add_env",
+                        help="additional remote environment variable definitions (eg. --addEnv \"somevar=something\")")
         defaults["add_env"] = None
 
         self.set_defaults(**defaults)
 
+
 def run_test_harness(options, args):
     if options.with_b2g_emulator:
         from mozrunner import B2GEmulatorRunner
-        runner = B2GEmulatorRunner(arch=options.emulator, b2g_home=options.with_b2g_emulator)
+        runner = B2GEmulatorRunner(
+            arch=options.emulator, b2g_home=options.with_b2g_emulator)
         runner.start()
         # because we just started the emulator, we need more than the
         # default number of retries here.
         retryLimit = 50
     else:
         retryLimit = 5
     try:
         dm_args = {'deviceRoot': options.remote_test_root}
@@ -233,23 +248,25 @@ def run_test_harness(options, args):
 
     options.xre_path = os.path.abspath(options.xre_path)
     cppunittests.update_mozinfo()
     progs = cppunittests.extract_unittests_from_args(args,
                                                      mozinfo.info,
                                                      options.manifest_path)
     tester = RemoteCPPUnitTests(dm, options, [item[0] for item in progs])
     try:
-        result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+        result = tester.run_tests(
+            progs, options.xre_path, options.symbols_path)
     finally:
         if options.with_b2g_emulator:
             runner.cleanup()
             runner.wait()
     return result
 
+
 def main():
     parser = RemoteCPPUnittestOptions()
     mozlog.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if options.local_lib is not None and not os.path.isdir(options.local_lib):
@@ -261,15 +278,16 @@ def main():
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
 
     log = mozlog.commandline.setup_logging("remotecppunittests", options,
                                            {"tbpl": sys.stdout})
     try:
         result = run_test_harness(options, args)
-    except Exception, e:
+    except Exception as e:
         log.error(str(e))
         result = False
     sys.exit(0 if result else 1)
 
+
 if __name__ == '__main__':
     main()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -1,28 +1,28 @@
 #!/usr/bin/env python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
-import sys, os, tempfile, shutil
+import sys
+import os
 from optparse import OptionParser
 import manifestparser
 import mozprocess
 import mozinfo
 import mozcrash
 import mozfile
 import mozlog
-from contextlib import contextmanager
-from subprocess import PIPE
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
+
 class CPPUnitTests(object):
     # Time (seconds) to wait for test process to complete
     TEST_PROC_TIMEOUT = 900
     # Time (seconds) in which process will be killed if it produces no output.
     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
 
     def run_one_test(self, prog, env, symbols_path=None, interactive=False,
                      timeout_factor=1):
@@ -48,17 +48,17 @@ class CPPUnitTests(object):
                                                  env=env,
                                                  storeOutput=False)
             else:
                 proc = mozprocess.ProcessHandler([prog],
                                                  cwd=tempdir,
                                                  env=env,
                                                  storeOutput=True,
                                                  processOutputLine=lambda _: None)
-            #TODO: After bug 811320 is fixed, don't let .run() kill the process,
+            # TODO: After bug 811320 is fixed, don't let .run() kill the process,
             # instead use a timeout in .wait() and then kill to get a stack.
             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
             proc.run(timeout=test_timeout,
                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
             proc.wait()
             if proc.output:
                 output = "\n%s" % "\n".join(proc.output)
                 self.log.process_output(proc.pid, output, command=[prog])
@@ -75,23 +75,23 @@ class CPPUnitTests(object):
             if not result:
                 self.log.test_end(basename, status='FAIL', expected='PASS',
                                   message=("test failed with return code %d" %
                                            proc.proc.returncode))
             else:
                 self.log.test_end(basename, status='PASS', expected='PASS')
             return result
 
-    def build_core_environment(self, env = {}):
+    def build_core_environment(self, env={}):
         """
         Add environment variables likely to be used across all platforms, including remote systems.
         """
         env["MOZILLA_FIVE_HOME"] = self.xre_path
         env["MOZ_XRE_DIR"] = self.xre_path
-        #TODO: switch this to just abort once all C++ unit tests have
+        # TODO: switch this to just abort once all C++ unit tests have
         # been fixed to enable crash reporting
         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
         env["MOZ_CRASHREPORTER"] = "1"
         return env
 
     def build_environment(self):
         """
@@ -129,17 +129,17 @@ class CPPUnitTests(object):
             if os.path.isfile(llvmsym):
                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
                 self.log.info("ASan using symbolizer at %s" % llvmsym)
             else:
                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 
             # media/mtransport tests statically link in NSS, which
             # causes ODR violations. See bug 1215679.
-            assert not 'ASAN_OPTIONS' in env
+            assert 'ASAN_OPTIONS' not in env
             env['ASAN_OPTIONS'] = 'detect_leaks=0:detect_odr_violation=0'
 
         return env
 
     def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
         """
         Run a set of C++ unit test programs.
 
@@ -170,31 +170,33 @@ class CPPUnitTests(object):
         self.log.suite_end()
 
         # Mozharness-parseable summary formatting.
         self.log.info("Result summary:")
         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
         return fail_count == 0
 
+
 class CPPUnittestOptions(OptionParser):
     def __init__(self):
         OptionParser.__init__(self)
         self.add_option("--xre-path",
-                        action = "store", type = "string", dest = "xre_path",
-                        default = None,
-                        help = "absolute path to directory containing XRE (probably xulrunner)")
+                        action="store", type="string", dest="xre_path",
+                        default=None,
+                        help="absolute path to directory containing XRE (probably xulrunner)")
         self.add_option("--symbols-path",
-                        action = "store", type = "string", dest = "symbols_path",
-                        default = None,
-                        help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+                        action="store", type="string", dest="symbols_path",
+                        default=None,
+                        help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
         self.add_option("--manifest-path",
-                        action = "store", type = "string", dest = "manifest_path",
-                        default = None,
-                        help = "path to test manifest, if different from the path to test binaries")
+                        action="store", type="string", dest="manifest_path",
+                        default=None,
+                        help="path to test manifest, if different from the path to test binaries")
+
 
 def extract_unittests_from_args(args, environ, manifest_path):
     """Extract unittests from args, expanding directories as needed"""
     mp = manifestparser.TestManifest(strict=True)
     tests = []
     binary_path = None
 
     if manifest_path:
@@ -221,36 +223,39 @@ def extract_unittests_from_args(args, en
     else:
         tests.extend([(test['path'] + suffix, int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
 
     # skip non-existing tests
     tests = [test for test in tests if os.path.isfile(test[0])]
 
     return tests
 
+
 def update_mozinfo():
     """walk up directories to find mozinfo.json update the info"""
     path = SCRIPT_DIR
     dirs = set()
     while path != os.path.expanduser('~'):
         if path in dirs:
             break
         dirs.add(path)
         path = os.path.split(path)[0]
     mozinfo.find_and_update_from_json(*dirs)
 
+
 def run_test_harness(options, args):
     update_mozinfo()
     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
     options.xre_path = os.path.abspath(options.xre_path)
     tester = CPPUnitTests()
     result = tester.run_tests(progs, options.xre_path, options.symbols_path)
 
     return result
 
+
 def main():
     parser = CPPUnittestOptions()
     mozlog.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
@@ -264,10 +269,11 @@ def main():
     try:
         result = run_test_harness(options, args)
     except Exception as e:
         log.error(str(e))
         result = False
 
     sys.exit(0 if result else 1)
 
+
 if __name__ == '__main__':
     main()