Bug 1437593 - Vendor virtualenv-clone 0.3.0; r=ted
authorDave Hunt <dhunt@mozilla.com>
Tue, 27 Mar 2018 16:19:54 +0100
changeset 472751 c235820f3d76e6cb2ac43c4771af1c859a3da017
parent 472750 961cb8855eb914d3937f03dc22e08bafb03bec00
child 472752 b5b8d6b5923cfdff953d00ff944001df1582f80c
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1437593
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1437593 - Vendor virtualenv-clone 0.3.0; r=ted MozReview-Commit-ID: JkPPc9xcGyU
third_party/python/virtualenv-clone/LICENSE
third_party/python/virtualenv-clone/MANIFEST.in
third_party/python/virtualenv-clone/PKG-INFO
third_party/python/virtualenv-clone/README
third_party/python/virtualenv-clone/clonevirtualenv.py
third_party/python/virtualenv-clone/setup.cfg
third_party/python/virtualenv-clone/setup.py
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2011, Edward George, based on code contained within the
+virtualenv project.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/MANIFEST.in
@@ -0,0 +1,2 @@
+include README
+include LICENSE
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/PKG-INFO
@@ -0,0 +1,21 @@
+Metadata-Version: 1.1
+Name: virtualenv-clone
+Version: 0.3.0
+Summary: script to clone virtualenvs.
+Home-page: http://github.com/edwardgeorge/virtualenv-clone
+Author: Edward George
+Author-email: edwardgeorge@gmail.com
+License: MIT
+Description-Content-Type: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Intended Audience :: Developers
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/README
@@ -0,0 +1,33 @@
+virtualenv cloning script.
+
+A script for cloning a non-relocatable virtualenv.
+
+Virtualenv provides a way to make virtualenv's relocatable which could then be
+copied as we wanted. However making a virtualenv relocatable this way breaks
+the no-site-packages isolation of the virtualenv as well as other aspects that
+come with relative paths and '/usr/bin/env' shebangs that may be undesirable.
+
+Also, the .pth and .egg-link rewriting doesn't seem to work as intended. This
+attempts to overcome these issues and provide a way to easily clone an
+existing virtualenv.
+
+It performs the following:
+
+- copies sys.argv[1] dir to sys.argv[2]
+- updates the hardcoded VIRTUAL_ENV variable in the activate script to the
+  new repo location. (--relocatable doesn't touch this)
+- updates the shebangs of the various scripts in bin to the new python if
+  they pointed to the old python. (version numbering is retained.)
+
+    it can also change '/usr/bin/env python' shebangs to be absolute too,
+    though this functionality is not exposed at present.
+
+- checks sys.path of the cloned virtualenv and if any of the paths are from
+  the old environment it finds any .pth or .egg-link files within sys.path
+  located in the new environment and makes sure any absolute paths to the
+  old environment are updated to the new environment.
+
+- finally it double checks sys.path again and will fail if there are still
+  paths from the old environment present.
+
+NOTE: This script requires Python >= 2.5
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/clonevirtualenv.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+from __future__ import with_statement
+
+import logging
+import optparse
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import sys
+import itertools
+
+version_info = (0, 2, 6)
+__version__ = '.'.join(map(str, version_info))
+
+
+logger = logging.getLogger()
+
+
+if sys.version_info < (2, 6):
+    next = lambda gen: gen.next()
+
+
+env_bin_dir = 'bin'
+if sys.platform == 'win32':
+    env_bin_dir = 'Scripts'
+
+
+class UserError(Exception):
+    pass
+
+
+def _dirmatch(path, matchwith):
+    """Check if path is within matchwith's tree.
+
+    >>> _dirmatch('/home/foo/bar', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar/', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar/etc', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar2', '/home/foo/bar')
+    False
+    >>> _dirmatch('/home/foo/bar2/etc', '/home/foo/bar')
+    False
+    """
+    matchlen = len(matchwith)
+    if (path.startswith(matchwith)
+        and path[matchlen:matchlen + 1] in [os.sep, '']):
+        return True
+    return False
+
+
+def _virtualenv_sys(venv_path):
+    "obtain version and path info from a virtualenv."
+    executable = os.path.join(venv_path, env_bin_dir, 'python')
+    # Must use "executable" as the first argument rather than as the
+    # keyword argument "executable" to get correct value from sys.path
+    p = subprocess.Popen([executable,
+        '-c', 'import sys;'
+              'print (sys.version[:3]);'
+              'print ("\\n".join(sys.path));'],
+        env={},
+        stdout=subprocess.PIPE)
+    stdout, err = p.communicate()
+    assert not p.returncode and stdout
+    lines = stdout.decode('utf-8').splitlines()
+    return lines[0], filter(bool, lines[1:])
+
+
+def clone_virtualenv(src_dir, dst_dir):
+    if not os.path.exists(src_dir):
+        raise UserError('src dir %r does not exist' % src_dir)
+    if os.path.exists(dst_dir):
+        raise UserError('dest dir %r exists' % dst_dir)
+    #sys_path = _virtualenv_syspath(src_dir)
+    logger.info('cloning virtualenv \'%s\' => \'%s\'...' %
+            (src_dir, dst_dir))
+    shutil.copytree(src_dir, dst_dir, symlinks=True,
+            ignore=shutil.ignore_patterns('*.pyc'))
+    version, sys_path = _virtualenv_sys(dst_dir)
+    logger.info('fixing scripts in bin...')
+    fixup_scripts(src_dir, dst_dir, version)
+
+    has_old = lambda s: any(i for i in s if _dirmatch(i, src_dir))
+
+    if has_old(sys_path):
+        # only need to fix stuff in sys.path if we have old
+        # paths in the sys.path of new python env. right?
+        logger.info('fixing paths in sys.path...')
+        fixup_syspath_items(sys_path, src_dir, dst_dir)
+    v_sys = _virtualenv_sys(dst_dir)
+    remaining = has_old(v_sys[1])
+    assert not remaining, v_sys
+    fix_symlink_if_necessary(src_dir, dst_dir)
+
+def fix_symlink_if_necessary(src_dir, dst_dir):
+    #sometimes the source virtual environment has symlinks that point to itself
+    #one example is $OLD_VIRTUAL_ENV/local/lib points to $OLD_VIRTUAL_ENV/lib
+    #this function makes sure
+    #$NEW_VIRTUAL_ENV/local/lib will point to $NEW_VIRTUAL_ENV/lib
+    #usually this goes unnoticed unless one tries to upgrade a package though pip, so this bug is hard to find.
+    logger.info("scanning for internal symlinks that point to the original virtual env")
+    for dirpath, dirnames, filenames in os.walk(dst_dir):
+        for a_file in itertools.chain(filenames, dirnames):
+            full_file_path = os.path.join(dirpath, a_file)
+            if os.path.islink(full_file_path):
+                target = os.path.realpath(full_file_path)
+                if target.startswith(src_dir):
+                    new_target = target.replace(src_dir, dst_dir)
+                    logger.debug('fixing symlink in %s' % (full_file_path,))
+                    os.remove(full_file_path)
+                    os.symlink(new_target, full_file_path)
+
+
+def fixup_scripts(old_dir, new_dir, version, rewrite_env_python=False):
+    bin_dir = os.path.join(new_dir, env_bin_dir)
+    root, dirs, files = next(os.walk(bin_dir))
+    pybinre = re.compile(r'pythonw?([0-9]+(\.[0-9]+(\.[0-9]+)?)?)?$')
+    for file_ in files:
+        filename = os.path.join(root, file_)
+        if file_ in ['python', 'python%s' % version, 'activate_this.py']:
+            continue
+        elif file_.startswith('python') and pybinre.match(file_):
+            # ignore other possible python binaries
+            continue
+        elif file_.endswith('.pyc'):
+            # ignore compiled files
+            continue
+        elif file_ == 'activate' or file_.startswith('activate.'):
+            fixup_activate(os.path.join(root, file_), old_dir, new_dir)
+        elif os.path.islink(filename):
+            fixup_link(filename, old_dir, new_dir)
+        elif os.path.isfile(filename):
+            fixup_script_(root, file_, old_dir, new_dir, version,
+                rewrite_env_python=rewrite_env_python)
+
+
+def fixup_script_(root, file_, old_dir, new_dir, version,
+                  rewrite_env_python=False):
+    old_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(old_dir))
+    new_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(new_dir))
+    env_shebang = '#!/usr/bin/env python'
+
+    filename = os.path.join(root, file_)
+    with open(filename, 'rb') as f:
+        if f.read(2) != b'#!':
+            # no shebang
+            return
+        f.seek(0)
+        lines = f.readlines()
+
+    if not lines:
+        # warn: empty script
+        return
+
+    def rewrite_shebang(version=None):
+        logger.debug('fixing %s' % filename)
+        shebang = new_shebang
+        if version:
+            shebang = shebang + version
+        shebang = (shebang + '\n').encode('utf-8')
+        with open(filename, 'wb') as f:
+            f.write(shebang)
+            f.writelines(lines[1:])
+
+    try:
+        bang = lines[0].decode('utf-8').strip()
+    except UnicodeDecodeError:
+        # binary file
+        return
+
+    # This takes care of the scheme in which shebang is of type
+    # '#!/venv/bin/python3' while the version of system python
+    # is of type 3.x e.g. 3.5.
+    short_version = bang[len(old_shebang):]
+
+    if not bang.startswith('#!'):
+        return
+    elif bang == old_shebang:
+        rewrite_shebang()
+    elif (bang.startswith(old_shebang)
+          and bang[len(old_shebang):] == version):
+        rewrite_shebang(version)
+    elif (bang.startswith(old_shebang)
+          and short_version
+          and bang[len(old_shebang):] == short_version):
+        rewrite_shebang(short_version)
+    elif rewrite_env_python and bang.startswith(env_shebang):
+        if bang == env_shebang:
+            rewrite_shebang()
+        elif bang[len(env_shebang):] == version:
+            rewrite_shebang(version)
+    else:
+        # can't do anything
+        return
+
+
+def fixup_activate(filename, old_dir, new_dir):
+    logger.debug('fixing %s' % filename)
+    with open(filename, 'rb') as f:
+        data = f.read().decode('utf-8')
+
+    data = data.replace(old_dir, new_dir)
+    with open(filename, 'wb') as f:
+        f.write(data.encode('utf-8'))
+
+
+def fixup_link(filename, old_dir, new_dir, target=None):
+    logger.debug('fixing %s' % filename)
+    if target is None:
+        target = os.readlink(filename)
+
+    origdir = os.path.dirname(os.path.abspath(filename)).replace(
+        new_dir, old_dir)
+    if not os.path.isabs(target):
+        target = os.path.abspath(os.path.join(origdir, target))
+        rellink = True
+    else:
+        rellink = False
+
+    if _dirmatch(target, old_dir):
+        if rellink:
+            # keep relative links, but don't keep original in case it
+            # traversed up out of, then back into the venv.
+            # so, recreate a relative link from absolute.
+            target = target[len(origdir):].lstrip(os.sep)
+        else:
+            target = target.replace(old_dir, new_dir, 1)
+
+    # else: links outside the venv, replaced with absolute path to target.
+    _replace_symlink(filename, target)
+
+
+def _replace_symlink(filename, newtarget):
+    tmpfn = "%s.new" % filename
+    os.symlink(newtarget, tmpfn)
+    os.rename(tmpfn, filename)
+
+
+def fixup_syspath_items(syspath, old_dir, new_dir):
+    for path in syspath:
+        if not os.path.isdir(path):
+            continue
+        path = os.path.normcase(os.path.abspath(path))
+        if _dirmatch(path, old_dir):
+            path = path.replace(old_dir, new_dir, 1)
+            if not os.path.exists(path):
+                continue
+        elif not _dirmatch(path, new_dir):
+            continue
+        root, dirs, files = next(os.walk(path))
+        for file_ in files:
+            filename = os.path.join(root, file_)
+            if filename.endswith('.pth'):
+                fixup_pth_file(filename, old_dir, new_dir)
+            elif filename.endswith('.egg-link'):
+                fixup_egglink_file(filename, old_dir, new_dir)
+
+
+def fixup_pth_file(filename, old_dir, new_dir):
+    logger.debug('fixup_pth_file %s' % filename)
+
+    with open(filename, 'r') as f:
+        lines = f.readlines()
+
+    has_change = False
+
+    for num, line in enumerate(lines):
+        line = (line.decode('utf-8') if hasattr(line, 'decode') else line).strip()
+
+        if not line or line.startswith('#') or line.startswith('import '):
+            continue
+        elif _dirmatch(line, old_dir):
+            lines[num] = line.replace(old_dir, new_dir, 1)
+            has_change = True
+
+    if has_change:
+        with open(filename, 'w') as f:
+            payload = os.linesep.join([l.strip() for l in lines]) + os.linesep
+            f.write(payload)
+
+
+def fixup_egglink_file(filename, old_dir, new_dir):
+    logger.debug('fixing %s' % filename)
+    with open(filename, 'rb') as f:
+        link = f.read().decode('utf-8').strip()
+    if _dirmatch(link, old_dir):
+        link = link.replace(old_dir, new_dir, 1)
+        with open(filename, 'wb') as f:
+            link = (link + '\n').encode('utf-8')
+            f.write(link)
+
+
+def main():
+    parser = optparse.OptionParser("usage: %prog [options]"
+        " /path/to/existing/venv /path/to/cloned/venv")
+    parser.add_option('-v',
+            action="count",
+            dest='verbose',
+            default=False,
+            help='verbosity')
+    options, args = parser.parse_args()
+    try:
+        old_dir, new_dir = args
+    except ValueError:
+        print("virtualenv-clone %s" % (__version__,))
+        parser.error("not enough arguments given.")
+    old_dir = os.path.realpath(old_dir)
+    new_dir = os.path.realpath(new_dir)
+    loglevel = (logging.WARNING, logging.INFO, logging.DEBUG)[min(2,
+            options.verbose)]
+    logging.basicConfig(level=loglevel, format='%(message)s')
+    try:
+        clone_virtualenv(old_dir, new_dir)
+    except UserError:
+        e = sys.exc_info()[1]
+        parser.error(str(e))
+
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/setup.cfg
@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+
new file mode 100644
--- /dev/null
+++ b/third_party/python/virtualenv-clone/setup.py
@@ -0,0 +1,53 @@
+import sys
+from setuptools.command.test import test as TestCommand
+from setuptools import setup
+
+
+if __name__ == '__main__' and sys.version_info < (2, 5):
+    raise SystemExit("Python >= 2.5 required for virtualenv-clone")
+
+test_requirements = [
+    'virtualenv',
+    'tox',
+    'pytest'
+]
+
+
+class ToxTest(TestCommand):
+    def finalize_options(self):
+        TestCommand.finalize_options(self)
+        self.test_args = []
+        self.test_suite = True
+
+    def run_tests(self):
+        import tox
+        tox.cmdline()
+
+
+setup(name="virtualenv-clone",
+    version='0.3.0',
+    description='script to clone virtualenvs.',
+    author='Edward George',
+    author_email='edwardgeorge@gmail.com',
+    url='http://github.com/edwardgeorge/virtualenv-clone',
+    license="MIT",
+    py_modules=["clonevirtualenv"],
+    entry_points={
+        'console_scripts': [
+            'virtualenv-clone=clonevirtualenv:main',
+    ]},
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python",
+        "Intended Audience :: Developers",
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+    ],
+    tests_require=test_requirements,
+    cmdclass={'test': ToxTest}
+)