Backed out changeset 9323d33afd50 (bug 1397180) for flake8 linting failure attesting/talos/talos/ffsetup.py:102:13 | local variable 'path' is assigned to but never used. r=backout
authorSebastian Hengst <archaeopteryx@coole-files.de>
Thu, 28 Sep 2017 17:16:56 +0200
changeset 383492 6304aff270522a06d4bd5f269c2f549c241e1a13
parent 383491 0e83790a93a65371876838527f7a4e6abaec7e26
child 383493 dce12df1e58de9328c53f9fb54bb51d662d52840
push id32594
push userkwierso@gmail.com
push dateThu, 28 Sep 2017 22:49:33 +0000
treeherdermozilla-central@6dea0ee45b66 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1397180
milestone58.0a1
backs out9323d33afd502a26f0d496aa9be81109c0371fb0
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 9323d33afd50 (bug 1397180) for flake8 linting failure attesting/talos/talos/ffsetup.py:102:13 | local variable 'path' is assigned to but never used. r=backout
testing/talos/requirements.txt
testing/talos/talos/cmdline.py
testing/talos/talos/ffsetup.py
testing/talos/talos/heavy.py
testing/talos/talos/run_tests.py
testing/talos/tests/profile.tgz
testing/talos/tests/test_heavy.py
--- a/testing/talos/requirements.txt
+++ b/testing/talos/requirements.txt
@@ -3,9 +3,8 @@ mozcrash>=0.15
 mozfile>=1.2
 mozhttpd>=0.7
 mozinfo>=0.8
 mozprocess>=0.22
 mozversion>=1.3
 mozprofile>=0.25
 psutil>=3.1.1
 simplejson>=2.1.1
-requests>=2.18.4
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -174,17 +174,16 @@ def create_parser(mach_interface=False):
             help='If given, enable Stylo via Environment variables and '
                  'upload results with Stylo options.')
     add_arg('--disable-stylo', action="store_true",
             dest='disable_stylo',
             help='If given, disable Stylo via Environment variables.')
     add_arg('--stylo-threads', type=int,
             dest='stylothreads',
             help='If given, run Stylo with a certain number of threads')
-    add_arg('--profile', type=str, default=None,
-            help="Downloads a profile from TaskCluster and uses it")
+
     add_logging_group(parser)
     return parser
 
 
 def parse_args(argv=None):
     parser = create_parser()
     return parser.parse_args(argv)
--- a/testing/talos/talos/ffsetup.py
+++ b/testing/talos/talos/ffsetup.py
@@ -14,17 +14,16 @@ import mozfile
 import mozinfo
 import mozrunner
 from mozlog import get_proxy_logger
 from mozprocess import ProcessHandlerMixin
 from mozprofile.profile import Profile
 from talos import utils
 from talos.gecko_profile import GeckoProfile
 from talos.utils import TalosError
-from talos import heavy
 
 LOG = get_proxy_logger()
 
 
 class FFSetup(object):
     """
     Initialize the browser environment before running a test.
 
@@ -88,24 +87,16 @@ class FFSetup(object):
             if type(value) is str:
                 value = utils.interpolate(value, webserver=webserver)
                 preferences[name] = value
 
         extensions = self.browser_config['extensions'][:]
         if self.test_config.get('extensions'):
             extensions.append(self.test_config['extensions'])
 
-        if self.browser_config['develop'] or \
-           'try' in str.lower(self.browser_config['branch_name']):
-            extensions = [os.path.dirname(i) for i in extensions]
-
-        # downloading a profile instead of using the empty one
-        if self.test_config['profile'] is not None:
-            path = heavy.download_profile(self.test_config['profile'])
-
         profile = Profile.clone(
             os.path.normpath(self.test_config['profile_path']),
             self.profile_dir,
             restore=False)
 
         profile.set_preferences(preferences)
 
         # installing addons
deleted file mode 100644
--- a/testing/talos/talos/heavy.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-Downloads Heavy profiles from TaskCluster.
-"""
-from __future__ import absolute_import
-import os
-import tarfile
-import functools
-import datetime
-from email.utils import parsedate
-
-import requests
-from requests.adapters import HTTPAdapter
-from mozlog import get_proxy_logger
-
-
-LOG = get_proxy_logger()
-TC_LINK = ("https://index.taskcluster.net/v1/task/garbage.heavyprofile/"
-           "artifacts/public/today-%s.tgz")
-
-
-class ProgressBar(object):
-    def __init__(self, size, template="\r%d%%"):
-        self.size = size
-        self.current = 0
-        self.tens = 0
-        self.template = template
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        return False
-
-    def incr(self):
-        if self.current == self.size:
-            return
-        percent = float(self.current) / float(self.size) * 100
-        tens, __ = divmod(percent, 10)
-        if tens > self.tens:
-            LOG.info(self.template % percent)
-            self.tens = tens
-
-        self.current += 1
-
-
-def follow_redirects(url, max=3):
-    location = url
-    current = 0
-    page = requests.head(url)
-    while page.status_code == 303 and current < max:
-        current += 1
-        location = page.headers['Location']
-        page = requests.head(location)
-    if page.status_code == 303 and current == max:
-        raise ValueError("Max redirects Reached")
-    last_modified = page.headers['Last-Modified']
-    last_modified = datetime.datetime(*parsedate(last_modified)[:6])
-    return location, last_modified
-
-
-def _recursive_mtime(path):
-    max = os.path.getmtime(path)
-    for root, dirs, files in os.walk(path):
-        for element in dirs + files:
-            age = os.path.getmtime(os.path.join(root, element))
-            if age > max:
-                max = age
-    return max
-
-
-def profile_age(profile_dir, last_modified=None):
-    if last_modified is None:
-        last_modified = datetime.datetime.now()
-
-    profile_ts = _recursive_mtime(profile_dir)
-    profile_ts = datetime.datetime.fromtimestamp(profile_ts)
-    return (last_modified - profile_ts).days
-
-
-def download_profile(name, profiles_dir=None):
-    if profiles_dir is None:
-        profiles_dir = os.path.join(os.path.expanduser('~'), '.mozilla',
-                                    'profiles')
-    profiles_dir = os.path.abspath(profiles_dir)
-    if not os.path.exists(profiles_dir):
-        os.makedirs(profiles_dir)
-
-    target = os.path.join(profiles_dir, name)
-    url = TC_LINK % name
-    cache_dir = os.path.join(profiles_dir, '.cache')
-    if not os.path.exists(cache_dir):
-        os.makedirs(cache_dir)
-
-    archive_file = os.path.join(cache_dir, 'today-%s.tgz' % name)
-
-    url, last_modified = follow_redirects(url)
-    if os.path.exists(target):
-        age = profile_age(target, last_modified)
-        if age < 7:
-            # profile is not older than a week, we're good
-            LOG.info("Local copy of %r is fresh enough" % name)
-            LOG.info("%d days old" % age)
-            return target
-
-    LOG.info("Downloading from %r" % url)
-    session = requests.Session()
-    session.mount('https://', HTTPAdapter(max_retries=5))
-    req = session.get(url, stream=True, timeout=20)
-    req.raise_for_status()
-
-    total_length = int(req.headers.get('content-length'))
-
-    # XXX implement Range to resume download on disconnects
-    template = 'Download progress %d%%'
-    with open(archive_file, 'wb') as f:
-        iter = req.iter_content(chunk_size=1024)
-        size = total_length / 1024 + 1
-        with ProgressBar(size=size, template=template) as bar:
-            for chunk in iter:
-                if chunk:
-                    f.write(chunk)
-                bar.incr()
-
-    LOG.info("Extracting profile in %r" % target)
-    template = 'Extraction progress %d%%'
-
-    with tarfile.open(archive_file, "r:gz") as tar:
-        LOG.info("Checking the tarball content...")
-        size = len(list(tar))
-        with ProgressBar(size=size, template=template) as bar:
-            def _extract(self, *args, **kw):
-                bar.incr()
-                return self.old(*args, **kw)
-            tar.old = tar.extract
-            tar.extract = functools.partial(_extract, tar)
-            tar.extractall(target)
-    LOG.info("Profile downloaded.")
-    return target
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -105,30 +105,23 @@ def run_tests(config, browser_config):
                 os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
                                                '/\\t:\\')))
         if not test.get('url'):
             # build 'url' for tptest
             test['url'] = buildCommandLine(test)
         test['url'] = utils.interpolate(test['url'])
         test['setup'] = utils.interpolate(test['setup'])
         test['cleanup'] = utils.interpolate(test['cleanup'])
-        test['profile'] = config.get('profile')
 
     # pass --no-remote to firefox launch, if --develop is specified
     # we do that to allow locally the user to have another running firefox
     # instance
     if browser_config['develop']:
         browser_config['extra_args'] = '--no-remote'
 
-    # with addon signing for production talos, we want to develop without it
-    if browser_config['develop'] or 'try' in str.lower(browser_config['branch_name']):
-        browser_config['preferences']['xpinstall.signatures.required'] = False
-
-    browser_config['preferences']['extensions.allow-non-mpc-extensions'] = True
-
     # if using firstNonBlankPaint, must turn on pref for it
     if test.get('fnbpaint', False):
         LOG.info("Using firstNonBlankPaint, so turning on pref for it")
         browser_config['preferences']['dom.performance.time_to_non_blank_paint.enabled'] = True
 
     # set defaults
     testdate = config.get('testdate', '')
 
deleted file mode 100644
index 0ca4fe62d3e134a5985bea4ab97d06e5099eb1e3..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
literal 0
Hc$@<O00001
deleted file mode 100644
--- a/testing/talos/tests/test_heavy.py
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env python
-
-"""
-test talos' heavy module:
-
-http://hg.mozilla.org/build/talos/file/tip/talos/heavy.py
-"""
-from __future__ import absolute_import
-import unittest
-import tempfile
-import shutil
-import datetime
-import contextlib
-import os
-import time
-
-import talos.heavy
-
-
-archive = os.path.join(os.path.dirname(__file__), 'profile.tgz')
-archive_size = os.stat(archive).st_size
-
-
-@contextlib.contextmanager
-def mock_requests(**kw):
-    class Session:
-        def mount(self, *args, **kw):
-            pass
-
-    kw['Session'] = Session
-    old = {}
-    for meth, func in kw.items():
-        curr = getattr(talos.heavy.requests, meth)
-        old[meth] = curr
-        setattr(talos.heavy.requests, meth, func)
-        setattr(Session, meth, func)
-    try:
-        yield
-    finally:
-        for meth, func in old.items():
-            setattr(talos.heavy.requests, meth, func)
-
-
-class _Response(object):
-    def __init__(self, code, headers=None, file=None):
-        if headers is None:
-            headers = {}
-        self.headers = headers
-        self.status_code = code
-        self.file = file
-
-    def raise_for_status(self):
-        pass
-
-    def iter_content(self, chunk_size):
-        with open(self.file, 'rb') as f:
-            yield f.read(chunk_size)
-
-
-class Logger:
-    def __init__(self):
-        self.data = []
-
-    def info(self, msg):
-        self.data.append(msg)
-
-
-class TestFilter(unittest.TestCase):
-
-    def setUp(self):
-        self.temp = tempfile.mkdtemp()
-        self.logs = talos.heavy.LOG.logger = Logger()
-
-    def tearDown(self):
-        shutil.rmtree(self.temp)
-
-    def test_profile_age(self):
-        """test profile_age function"""
-        days = talos.heavy.profile_age(self.temp)
-        self.assertEqual(days, 0)
-
-        _8_days = datetime.datetime.now() + datetime.timedelta(days=8)
-        days = talos.heavy.profile_age(self.temp, _8_days)
-        self.assertEqual(days, 8)
-
-    def test_directory_age(self):
-        """make sure it detects changes in files in subdirs"""
-        with open(os.path.join(self.temp, 'file'), 'w') as f:
-            f.write('xxx')
-
-        current_age = talos.heavy._recursive_mtime(self.temp)
-        time.sleep(1.1)
-
-        with open(os.path.join(self.temp, 'file'), 'w') as f:
-            f.write('----')
-
-        self.assertTrue(current_age < talos.heavy._recursive_mtime(self.temp))
-
-    def test_follow_redirect(self):
-        """test follow_redirect function"""
-        _8_days = datetime.datetime.now() + datetime.timedelta(days=8)
-        _8_days = _8_days.strftime('%a, %d %b %Y %H:%M:%S UTC')
-
-        resps = [_Response(303, {'Location': 'blah'}),
-                 _Response(303, {'Location': 'bli'}),
-                 _Response(200, {'Last-Modified': _8_days})]
-
-        class Counter:
-            c = 0
-
-        def _head(url, curr=Counter()):
-            curr.c += 1
-            return resps[curr.c]
-
-        with mock_requests(head=_head):
-            loc, lm = talos.heavy.follow_redirects('https://example.com')
-            days = talos.heavy.profile_age(self.temp, lm)
-            self.assertEqual(days, 8)
-
-    def _test_download(self, age):
-
-        def _days(num):
-            d = datetime.datetime.now() + datetime.timedelta(days=num)
-            return d.strftime('%a, %d %b %Y %H:%M:%S UTC')
-
-        resps = [_Response(303, {'Location': 'blah'}),
-                 _Response(303, {'Location': 'bli'}),
-                 _Response(200, {'Last-Modified': _days(age)})]
-
-        class Counter:
-            c = 0
-
-        def _head(url, curr=Counter()):
-            curr.c += 1
-            return resps[curr.c]
-
-        def _get(url, *args, **kw):
-            return _Response(200, {'Last-Modified': _days(age),
-                                   'content-length': str(archive_size)},
-                             file=archive)
-
-        with mock_requests(head=_head, get=_get):
-            target = talos.heavy.download_profile('simple',
-                                                  profiles_dir=self.temp)
-            profile = os.path.join(self.temp, 'simple')
-            self.assertTrue(os.path.exists(profile))
-            return target
-
-    def test_download_profile(self):
-        """test downloading heavy profile"""
-        # a 12 days old profile gets updated
-        self._test_download(12)
-
-        # a 8 days two
-        self._test_download(8)
-
-        # a 2 days sticks
-        self._test_download(2)
-        self.assertTrue("fresh enough" in self.logs.data[-2])
-
-
-if __name__ == '__main__':
-    unittest.main()