☠☠ backed out by 6b65dd49d4f0 ☠ ☠ | |
author | Henrik Skupin <mail@hskupin.info> |
Mon, 18 Jan 2016 19:50:26 +0100 | |
changeset 308168 | 384ff6b4b109bba115a73e1024d9e6d18226a424 |
parent 308167 | d6ee6a38aa43bae545a100d60296508878fde95c |
child 308169 | 4939f5d5c16c957ae0da6d509a26482983fc894a |
push id | 31074 |
push user | hskupin@mozilla.com |
push date | Thu, 04 Aug 2016 19:01:10 +0000 |
treeherder | autoland@4939f5d5c16c [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jlund |
bugs | 1258539 |
milestone | 51.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozharness/mozharness/base/script.py +++ b/testing/mozharness/mozharness/base/script.py @@ -10,29 +10,34 @@ script.py, along with config.py and log.py, represents the core of mozharness. """ import codecs from contextlib import contextmanager import datetime import errno +import fnmatch +import functools import gzip import inspect +import itertools import os import platform import pprint import re import shutil import socket import subprocess import sys +import tarfile import time import traceback import urllib2 +import zipfile import httplib import urlparse import hashlib if os.name == 'nt': try: import win32file import win32api PYWIN32 = True @@ -42,20 +47,20 @@ if os.name == 'nt': try: import simplejson as json assert json except ImportError: import json from mozprocess import ProcessHandler from mozharness.base.config import BaseConfig -from mozharness.base.errors import ZipErrorList from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \ LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL + def platform_name(): pm = PlatformMixin() if pm._is_linux() and pm._is_64_bit(): return 'linux64' elif pm._is_linux() and not pm._is_64_bit(): return 'linux' elif pm._is_darwin(): @@ -448,49 +453,35 @@ class ScriptMixin(PlatformMixin): kwargs = {"url": url, "file_name": file_name} return self.retry( download_func, kwargs=kwargs, **retry_args ) - def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True): + def download_unzip(self, url, parent_dir, target_unzip_dirs=None): """Generic method to download and extract a zip file. The downloaded file will always be saved to the working directory and is not getting deleted after extracting. Args: url (str): URL where the file to be downloaded is located. parent_dir (str): directory where the downloaded file will be extracted to. target_unzip_dirs (list, optional): directories inside the zip file to extract. Defaults to `None`. - halt_on_failure (bool, optional): whether or not to redefine the - log level as `FATAL` on errors. Defaults to True. """ dirs = self.query_abs_dirs() zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'], error_level=FATAL) - command = self.query_exe('unzip', return_type='list') - # Always overwrite to not get an input in a hidden pipe if files already exist - command.extend(['-q', '-o', zipfile, '-d', parent_dir]) - if target_unzip_dirs: - command.extend(target_unzip_dirs) - # TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS - # unzip return code 11 is 'no matching files were found' - self.run_command(command, - error_list=ZipErrorList, - halt_on_failure=halt_on_failure, - fatal_exit_code=3, - success_codes=[0, 11], - ) + self.unpack(zipfile, parent_dir, target_unzip_dirs) def load_json_url(self, url, error_level=None, *args, **kwargs): """ Returns a json object from a url (it retries). """ contents = self._retry_download( url=url, error_level=error_level, *args, **kwargs ) return json.loads(contents.read()) @@ -1100,17 +1091,17 @@ class ScriptMixin(PlatformMixin): throw_exception (bool, optional): whether or not to raise an exception if the return value of the command doesn't match any of the `success_codes`. Defaults to False. output_parser (OutputParser, optional): lets you provide an instance of your own OutputParser subclass. Defaults to `OutputParser`. output_timeout (int): amount of seconds to wait for output before the process is killed. fatal_exit_code (int, optional): call `self.fatal` if the return value - of the command is not on in `success_codes`. Defaults to 2. + of the command is not in `success_codes`. Defaults to 2. error_level (str, optional): log level name to use on error. Defaults to `ERROR`. **kwargs: Arbitrary keyword arguments. Returns: int: -1 on error. Any: `command` return value is returned otherwise. """ @@ -1392,36 +1383,85 @@ class ScriptMixin(PlatformMixin): except OSError: try: open(file_name, 'w').close() except IOError as e: msg = "I/O error(%s): %s" % (e.errno, e.strerror) self.log(msg, error_level=error_level) os.utime(file_name, times) - def unpack(self, filename, extract_to): - ''' + def unpack(self, filename, extract_to, extract_dirs=None, + error_level=ERROR, halt_on_failure=True, fatal_exit_code=2, + verbose=False): + """ This method allows us to extract a file regardless of its extension Args: filename (str): filename of the compressed file. extract_to (str): where to extract the compressed file. - ''' - # XXX: Make sure that filename has a extension of one of our supported file formats - m = re.search('\.tar\.(bz2|gz)$', filename) - if m: - command = self.query_exe('tar', return_type='list') - tar_cmd = "jxfv" - if m.group(1) == "gz": - tar_cmd = "zxfv" - command.extend([tar_cmd, filename, "-C", extract_to]) - self.run_command(command, halt_on_failure=True) + extract_dirs (list, optional): directories inside the archive file to extract. + Defaults to `None`. + halt_on_failure (bool, optional): whether or not to redefine the + log level as `FATAL` on errors. Defaults to True. + fatal_exit_code (int, optional): call `self.fatal` if the return value + of the command is not in `success_codes`. Defaults to 2. + verbose (bool, optional): whether or not extracted content should be displayed. + Defaults to False. + + Raises: + IOError: on `filename` file not found. + + """ + def _filter_entries(namelist): + """Filter entries of the archive based on the specified list of to extract dirs.""" + filter_partial = functools.partial(fnmatch.filter, namelist) + for entry in itertools.chain(*map(filter_partial, extract_dirs or ['*'])): + yield entry + + if not os.path.isfile(filename): + raise IOError('Could not find file to extract: %s' % filename) + + level = FATAL if halt_on_failure else error_level + + if zipfile.is_zipfile(filename): + try: + self.info('Using ZipFile to extract {} to {}'.format(filename, extract_to)) + with zipfile.ZipFile(filename) as bundle: + for entry in _filter_entries(bundle.namelist()): + if verbose: + self.info(' %s' % entry) + bundle.extract(entry, path=extract_to) + + # ZipFile doesn't preserve permissions during extraction: + # http://bugs.python.org/issue15795 + fname = os.path.realpath(os.path.join(extract_to, entry)) + mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF + # Only set permissions if attributes are available. Otherwise all + # permissions will be removed eg. on Windows. + if mode: + os.chmod(fname, mode) + except zipfile.BadZipfile as e: + self.log('%s (%s)' % (e.message, filename), + level=level, exit_code=fatal_exit_code) + + # Bug 1211882 - is_tarfile cannot be trusted for dmg files + elif tarfile.is_tarfile(filename) and not filename.lower().endswith('.dmg'): + try: + self.info('Using TarFile to extract {} to {}'.format(filename, extract_to)) + with tarfile.open(filename) as bundle: + for entry in _filter_entries(bundle.getnames()): + if verbose: + self.info(' %s' % entry) + bundle.extract(entry, path=extract_to) + except tarfile.TarError as e: + self.log('%s (%s)' % (e.message, filename), + level=level, exit_code=fatal_exit_code) else: - # XXX implement - pass + self.log('No extraction method found for: %s' % filename, + level=level, exit_code=fatal_exit_code) def PreScriptRun(func): """Decorator for methods that will be called before script execution. Each method on a BaseScript having this decorator will be called at the beginning of BaseScript.run().
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py +++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py @@ -294,63 +294,16 @@ class FirefoxUITests(TestingMixin, VCSTo def run_tests(self): """Run all the tests""" return self.run_test( binary_path=self.binary_path, env=self.query_env(), ) - def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True): - """Overwritten method from BaseScript until bug 1258539 is fixed. - - The downloaded file will always be saved to the working directory and is not getting - deleted after extracting. - - Args: - url (str): URL where the file to be downloaded is located. - parent_dir (str): directory where the downloaded file will - be extracted to. - target_unzip_dirs (list, optional): directories inside the zip file to extract. - Defaults to `None`. - halt_on_failure (bool, optional): whether or not to redefine the - log level as `FATAL` on errors. Defaults to True. - - """ - import fnmatch - import itertools - import functools - import zipfile - - def _filter_entries(namelist): - """Filter entries of the archive based on the specified list of extract_dirs.""" - filter_partial = functools.partial(fnmatch.filter, namelist) - for entry in itertools.chain(*map(filter_partial, target_unzip_dirs or ['*'])): - yield entry - - dirs = self.query_abs_dirs() - zip = self.download_file(url, parent_dir=dirs['abs_work_dir'], - error_level=FATAL) - - try: - self.info('Using ZipFile to extract {0} to {1}'.format(zip, parent_dir)) - with zipfile.ZipFile(zip) as bundle: - for entry in _filter_entries(bundle.namelist()): - bundle.extract(entry, path=parent_dir) - - # ZipFile doesn't preserve permissions: http://bugs.python.org/issue15795 - fname = os.path.realpath(os.path.join(parent_dir, entry)) - mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF - # Only set permissions if attributes are available. - if mode: - os.chmod(fname, mode) - except zipfile.BadZipfile as e: - self.log('{0} ({1})'.format(e.message, zip), - level=FATAL, exit_code=2) - class FirefoxUIFunctionalTests(FirefoxUITests): cli_script = 'cli_functional.py' default_tests = [ os.path.join('puppeteer', 'manifest.ini'), os.path.join('functional', 'manifest.ini'), ]
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -396,27 +396,16 @@ You can set this by: 1. specifying --test-url URL, or 2. running via buildbot and running the read-buildbot-config action """ if message: self.fatal(message + "Can't run download-and-extract... exiting") - if self.config.get("developer_mode") and self._is_darwin(): - # Bug 1066700 only affects Mac users that try to run mozharness locally - version = self._query_binary_version( - regex=re.compile("UnZip\ (\d+\.\d+)\ .*", re.MULTILINE), - cmd=[self.query_exe('unzip'), '-v'] - ) - if not version >= 6: - self.fatal("We require a more recent version of unzip to unpack our tests.zip files.\n" - "You are currently using version %s. Please update to at least 6.0.\n" - "You can visit http://www.info-zip.org/UnZip.html" % version) - def _read_packages_manifest(self): dirs = self.query_abs_dirs() source = self.download_file(self.test_packages_url, parent_dir=dirs['abs_work_dir'], error_level=FATAL) with self.opened(os.path.realpath(source)) as (fh, err): package_requirements = json.load(fh)
new file mode 100644 index 0000000000000000000000000000000000000000..1dc094198f3198c38e6c04dd9ea98dacecced1e1 GIT binary patch literal 10240 zc%1Fk!D<5`5C&k+`xFy;cXf5vSLmTHu}OrL?#7^o_VK%$QV4;x^k7Qp|6K$X9QcP( zwxT+<aE{kiJ64zL7oodlhaGF{gwl%H3gY^u331A0NJL`vhZJ6K-}~wOLCJQC$<MO9 zmuhdU0r$PVoxibH`FGYRL8@9s7yfzvFKhY~j`=jxJ}$jLITRXZUu5&wwLyKd-G)Qu zKj8QCUz-l||1LiB&$|?kMdq=}8|VD_oBWNt%HQaoG5_!3X>)FX?U2W&@w+k7v!A9& bZ!t~#4ZZ*X000000000000000fJgBGu%BeH
new file mode 100644 index 0000000000000000000000000000000000000000..c393ea4b881cc0ce9bd4d05e2368a162f4da9d78 GIT binary patch literal 256 zc$@(M0ssC&T4*^jL0KkKSvhzRxBviwe}%%30RU(L|9}8QA`yOX-oO9?AOHXuFaY=< zl?fY0lTE3Kpftq505CEPfCCYr$YcfuRMgb+n5NXpra)-TO&+G24LBS5Xko1sIxm7! zi%AltJ5VCYZ8RrlVuP%LMTcgtyXaKpSbXZqntXQt*QZIRZ#^*@VP*~6MzJh$<Hunt z2q16HSt*e@mdr%t@rSF0ajfWI%}vP>7fCu_d39NZ#ieC-%LSVZu{nNy*M?ZVho*Le zBb`PFVx7zdLk+QO$kAv}5ia!Rot+j8MSwM3E#}^#w(M(>gUQ?;_uvp53%MekC`cT< G2wVW@1Z}(k
new file mode 100644 index 0000000000000000000000000000000000000000..0fbfa39b1c76ae868fc8b67d3c76d199fd0c24eb GIT binary patch literal 260 zc$@(Q0sH<RiwFqlQKMG?17UJwXlZt3E_7jX0PWSmY6Bq<24K(o6cc)Pb#>NP=%FvM zNraW|#-N7w@w=N+2!XWpU`pu!T?7>z_=i!pqB^y3j@MN?R+sA+p}S>=9c$}^(u&y% z;`*ftamr;#L}K)Z6kc!N`|12a$##m#&$7IiYHzFo_r1NHzp+>Och)FDs#--C{(1f{ zYx)$9`83l$F1<iG6dGn<Wb@axL4C5_hC}5);P>-in-25;E<W?myA+N^=CR5f=luDb z{EfTH-{_t(|L@^xb8dj`kjJL+yD`$UpQcD}F-`joz5oCK00000000000001hNAUsT K(1A?=C;$KkW`c_V
new file mode 100644 index 0000000000000000000000000000000000000000..aa2fb34c1658128d6a25f0f40b5ae646b198ea7b GIT binary patch literal 517 zc$^FHW@h1H0D*`g7EcBwz``KIkd&FH9~#2Rz?}YTQ8);fR&X;gvb<mhN`r_16x}Af zuFaPRvO$;^s=GM3D6^nMuQ&srG0KYiAhk(}#Ti^&smU4n3LdFBIr$3Z`9(P?id?9+ zoPQV`E(2tPFei}A$uCOH)hnqe!DoXHNRdKjL2+rWLP|bRi$ZZ`i9%v-YKb1uct$2U zW?X?F0c8m={B;D;kRV}&*n|-rs3xHX2*e~tAb#6u2Q>*Pc!)C+)%OS^w=^yRl1O2J S%Q#jxkQQbjd;_9EdKmyw2yPqz
new file mode 100644 index 0000000000000000000000000000000000000000..20bdc5acdf11e3c466c5b59090d1aaa52b721d31 GIT binary patch literal 166 zc$^FHW@h1H00F+TuO19YfP+DX!9LDPMX#iyBs7GRfqC|k3E?1ITEWf0$nt`jfdNbe mcr!A|G2=2r0?yvj2qF<CvO-M6FpZTBq>d2?{eZL+ST6u72^ygQ
new file mode 100755 --- /dev/null +++ b/testing/mozharness/test/helper_files/archives/reference/bin/script.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo Hello world!
new file mode 100644 --- /dev/null +++ b/testing/mozharness/test/helper_files/archives/reference/lorem.txt @@ -0,0 +1,1 @@ +Lorem ipsum dolor sit amet.
new file mode 100755 --- /dev/null +++ b/testing/mozharness/test/helper_files/create_archives.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Script to auto-generate the different archive types under the archives directory. + +cd archives + +rm archive.* + +tar cf archive.tar -C reference . +gzip -fk archive.tar >archive.tar.gz +bzip2 -fk archive.tar >archive.tar.bz2 +cd reference && zip ../archive.zip -r * && cd ..
--- a/testing/mozharness/test/test_base_script.py +++ b/testing/mozharness/test/test_base_script.py @@ -1,12 +1,14 @@ import gc import mock import os import re +import shutil +import tempfile import types import unittest PYWIN32 = False if os.name == 'nt': try: import win32file PYWIN32 = True except: @@ -14,32 +16,37 @@ if os.name == 'nt': import mozharness.base.errors as errors import mozharness.base.log as log from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IGNORE import mozharness.base.script as script from mozharness.base.config import parse_config_file + +here = os.path.dirname(os.path.abspath(__file__)) + test_string = '''foo bar baz''' class CleanupObj(script.ScriptMixin, log.LogMixin): def __init__(self): super(CleanupObj, self).__init__() self.log_obj = None self.config = {'log_level': ERROR} -def cleanup(): +def cleanup(files=None): + files = files or [] + files.extend(('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr')) gc.collect() c = CleanupObj() - for f in ('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'): + for f in files: c.rmtree(f) def get_debug_script_obj(): s = script.BaseScript(config={'log_type': 'multi', 'log_level': DEBUG}, initial_config_file='test/test.json') return s @@ -51,22 +58,23 @@ def _post_fatal(self, **kwargs): fh.close() # TestScript {{{1 class TestScript(unittest.TestCase): def setUp(self): cleanup() self.s = None + self.tmpdir = tempfile.mkdtemp(suffix='.mozharness') def tearDown(self): # Close the logfile handles, or windows can't remove the logs if hasattr(self, 's') and isinstance(self.s, object): del(self.s) - cleanup() + cleanup([self.tmpdir]) # test _dump_config_hierarchy() when --dump-config-hierarchy is passed def test_dump_config_hierarchy_valid_files_len(self): try: self.s = script.BaseScript( initial_config_file='test/test.json', option_args=['--cfg', 'test/test_override.py,test/test_override2.py'], config={'dump_config_hierarchy': True} @@ -246,16 +254,48 @@ class TestScript(unittest.TestCase): 'regex': re.compile(',$'), 'level': IGNORE, }, { 'substr': ']$', 'level': WARNING, }]) error_logsize = os.path.getsize("test_logs/test_error.log") self.assertTrue(error_logsize > 0, msg="error list not working properly") + def test_unpack(self): + self.s = get_debug_script_obj() + + archives_path = os.path.join(here, 'helper_files', 'archives') + + # Test basic decompression + for archive in ('archive.tar', 'archive.tar.bz2', 'archive.tar.gz', 'archive.zip'): + self.s.unpack(os.path.join(archives_path, archive), self.tmpdir) + self.assertIn('script.sh', os.listdir(os.path.join(self.tmpdir, 'bin'))) + self.assertIn('lorem.txt', os.listdir(self.tmpdir)) + shutil.rmtree(self.tmpdir) + + # Test permissions for extracted entries from zip archive + self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir) + file_stats = os.stat(os.path.join(self.tmpdir, 'bin', 'script.sh')) + orig_fstats = os.stat(os.path.join(archives_path, 'reference', 'bin', 'script.sh')) + self.assertEqual(file_stats.st_mode, orig_fstats.st_mode) + shutil.rmtree(self.tmpdir) + + # Test extract specific dirs only + self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir, + extract_dirs=['bin/*']) + self.assertIn('bin', os.listdir(self.tmpdir)) + self.assertNotIn('lorem.txt', os.listdir(self.tmpdir)) + shutil.rmtree(self.tmpdir) + + # Test for invalid filenames (Windows only) + if PYWIN32: + with self.assertRaises(IOError): + self.s.unpack(os.path.join(archives_path, 'archive_invalid_filename.zip'), + self.tmpdir) + # TestHelperFunctions {{{1 class TestHelperFunctions(unittest.TestCase): temp_file = "test_dir/mozilla" def setUp(self): cleanup() self.s = None