Bug 830996 - implement a way to DRY mozbase packages for m-c;r=gps
authorJeff Hammel <jhammel@mozilla.com>
Fri, 25 Jan 2013 21:51:08 -0800
changeset 119936 818bfecbe8e58f538302e67e0da82041290223fc
parent 119935 ccade0070fb8057dd18bdddfa76ec437956e7937
child 119937 358dd2a3299058e813a16760d41294fb6886bc46
push id24231
push userryanvm@gmail.com
push dateSun, 27 Jan 2013 00:13:14 +0000
treeherdermozilla-central@d802d6faa080 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs830996
milestone21.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 830996 - implement a way to DRY mozbase packages for m-c;r=gps
build/virtualenv/packages.txt
build/virtualenv/populate_virtualenv.py
client.mk
--- a/build/virtualenv/packages.txt
+++ b/build/virtualenv/packages.txt
@@ -1,25 +1,15 @@
 simplejson.pth:python/simplejson-2.1.1
-manifestdestiny.pth:testing/mozbase/manifestdestiny
-mozcrash.pth:testing/mozbase/mozcrash
-mozdevice.pth:testing/mozbase/mozdevice
-mozfile.pth:testing/mozbase/mozfile
-mozhttpd.pth:testing/mozbase/mozhttpd
-mozinfo.pth:testing/mozbase/mozinfo
-mozinstall.pth:testing/mozbase/mozinstall
-mozlog.pth:testing/mozbase/mozlog
-mozprocess.pth:testing/mozbase/mozprocess
-mozprofile.pth:testing/mozbase/mozprofile
-mozrunner.pth:testing/mozbase/mozrunner
 marionette.pth:testing/marionette/client
 blessings.pth:python/blessings
 mach.pth:python/mach
 mozbuild.pth:python/mozbuild
 pymake.pth:build/pymake
 optional:setup.py:python/psutil:build_ext:--inplace
 optional:psutil.pth:python/psutil
 which.pth:python/which
 mock.pth:python/mock-1.0.0
 mozilla.pth:build
 mozilla.pth:config
 mozilla.pth:xpcom/typelib/xpt/tools
 copy:build/buildconfig.py
+packages.txt:testing/mozbase/packages.txt
--- a/build/virtualenv/populate_virtualenv.py
+++ b/build/virtualenv/populate_virtualenv.py
@@ -17,100 +17,120 @@ import sys
 # Minimum version of Python required to build.
 MINIMUM_PYTHON_MAJOR = 2
 MINIMUM_PYTHON_MINOR = 7
 
 
 class VirtualenvManager(object):
     """Contains logic for managing virtualenvs for building the tree."""
 
-    def __init__(self, topsrcdir, virtualenv_path, log_handle):
+    def __init__(self, topsrcdir, virtualenv_path, log_handle, manifest_path):
         """Create a new manager.
 
         Each manager is associated with a source directory, a path where you
         want the virtualenv to be created, and a handle to write output to.
         """
+        assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
         self.topsrcdir = topsrcdir
         self.virtualenv_root = virtualenv_path
         self.log_handle = log_handle
+        self.manifest_path = manifest_path
 
     @property
     def virtualenv_script_path(self):
         """Path to virtualenv's own populator script."""
         return os.path.join(self.topsrcdir, 'python', 'virtualenv',
             'virtualenv.py')
 
     @property
-    def manifest_path(self):
-        return os.path.join(self.topsrcdir, 'build', 'virtualenv',
-            'packages.txt')
-
-    @property
     def python_path(self):
         if sys.platform in ('win32', 'cygwin'):
             return os.path.join(self.virtualenv_root, 'Scripts', 'python.exe')
 
         return os.path.join(self.virtualenv_root, 'bin', 'python')
 
     @property
     def activate_path(self):
         if sys.platform in ('win32', 'cygwin'):
             return os.path.join(self.virtualenv_root, 'Scripts',
                 'activate_this.py')
 
         return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
 
+    def up_to_date(self):
+        """Returns whether the virtualenv is present and up to date."""
+
+        deps = [self.manifest_path, __file__]
+
+        # check if virtualenv exists
+        if not os.path.exists(self.virtualenv_root) or \
+            not os.path.exists(self.activate_path):
+
+            return False
+
+        # check modification times
+        activate_mtime = os.path.getmtime(self.activate_path)
+        dep_mtime = max(os.path.getmtime(p) for p in deps)
+        if dep_mtime > activate_mtime:
+            return False
+
+        # recursively check sub packages.txt files
+        submanifests = [i[1] for i in self.packages()
+                        if i[0] == 'packages.txt']
+        for submanifest in submanifests:
+            submanifest = os.path.join(self.topsrcdir, submanifest)
+            submanager = VirtualenvManager(self.topsrcdir,
+                                           self.virtualenv_root,
+                                           self.log_handle,
+                                           submanifest)
+            if not submanager.up_to_date():
+                return False
+
+        return True
+
     def ensure(self):
         """Ensure the virtualenv is present and up to date.
 
         If the virtualenv is up to date, this does nothing. Otherwise, it
         creates and populates the virtualenv as necessary.
 
         This should be the main API used from this class as it is the
         highest-level.
         """
-        deps = [self.manifest_path, __file__]
-
-        if not os.path.exists(self.virtualenv_root) or \
-            not os.path.exists(self.activate_path):
-
-            return self.build()
-
-        activate_mtime = os.path.getmtime(self.activate_path)
-        dep_mtime = max(os.path.getmtime(p) for p in deps)
-
-        if dep_mtime > activate_mtime:
-            return self.build()
-
-        return self.virtualenv_root
+        if self.up_to_date():
+            return self.virtualenv_root
+        return self.build()
 
     def create(self):
         """Create a new, empty virtualenv.
 
         Receives the path to virtualenv's virtualenv.py script (which will be
         called out to), the path to create the virtualenv in, and a handle to
         write output to.
         """
         env = dict(os.environ)
-        try:
-            del env['PYTHONDONTWRITEBYTECODE']
-        except KeyError:
-            pass
+        env.pop('PYTHONDONTWRITEBYTECODE', None)
 
         args = [sys.executable, self.virtualenv_script_path,
             '--system-site-packages', self.virtualenv_root]
 
         result = subprocess.call(args, stdout=self.log_handle,
             stderr=subprocess.STDOUT, env=env)
 
-        if result != 0:
+        if result:
             raise Exception('Error creating virtualenv.')
 
         return self.virtualenv_root
 
+    def packages(self):
+        with file(self.manifest_path, 'rU') as fh:
+            packages = [line.rstrip().split(':')
+                        for line in fh]
+        return packages
+
     def populate(self):
         """Populate the virtualenv.
 
         The manifest file consists of colon-delimited fields. The first field
         specifies the action. The remaining fields are arguments to that
         action. The following actions are supported:
 
         setup.py -- Invoke setup.py for a package. Expects the arguments:
@@ -131,21 +151,18 @@ class VirtualenvManager(object):
         copy -- Copies the given file in the virtualenv site packages
             directory.
 
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
-        packages = []
-        fh = open(self.manifest_path, 'rU')
-        for line in fh:
-            packages.append(line.rstrip().split(':'))
-        fh.close()
+
+        packages = self.packages()
 
         def handle_package(package):
             python_lib = distutils.sysconfig.get_python_lib()
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
                     package[2:])
@@ -157,16 +174,29 @@ class VirtualenvManager(object):
 
                 src = os.path.join(self.topsrcdir, package[1])
                 dst = os.path.join(python_lib, os.path.basename(package[1]))
 
                 shutil.copy(src, dst)
 
                 return True
 
+            if package[0] == 'packages.txt':
+                assert len(package) == 2
+
+                src = os.path.join(self.topsrcdir, package[1])
+                assert os.path.isfile(src), "'%s' does not exist" % src
+                submanager = VirtualenvManager(self.topsrcdir,
+                                               self.virtualenv_root,
+                                               self.log_handle,
+                                               src)
+                submanager.populate()
+
+                return True
+
             if package[0].endswith('.pth'):
                 assert len(package) == 2
 
                 path = os.path.join(self.topsrcdir, package[1])
 
                 with open(os.path.join(python_lib, package[0]), 'a') as f:
                     f.write("%s\n" % path)
 
@@ -216,27 +246,22 @@ class VirtualenvManager(object):
                     continue
 
                 old_env_variables[k] = os.environ[k]
                 del os.environ[k]
 
             for package in packages:
                 handle_package(package)
         finally:
-            try:
-                del os.environ['MACOSX_DEPLOYMENT_TARGET']
-            except KeyError:
-                pass
+            os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
 
             if old_target is not None:
                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
 
-            for k in old_env_variables:
-                os.environ[k] = old_env_variables[k]
-
+            os.environ.update(old_env_variables)
 
     def call_setup(self, directory, arguments):
         """Calls setup.py in a directory."""
         setup = os.path.join(directory, 'setup.py')
 
         program = [sys.executable, setup]
         program.extend(arguments)
 
@@ -315,15 +340,18 @@ if __name__ == '__main__':
     populate = False
 
     # This should only be called internally.
     if sys.argv[1] == 'populate':
         populate = True
         topsrcdir = sys.argv[2]
         virtualenv_path = sys.argv[3]
 
-    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout)
+    # path to default packages.txt
+    manifest_path = os.path.join(topsrcdir, 'build', 'virtualenv', 'packages.txt')
+
+    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout, manifest_path)
 
     if populate:
         manager.populate()
     else:
         manager.ensure()
 
--- a/client.mk
+++ b/client.mk
@@ -281,16 +281,17 @@ CONFIG_STATUS_DEPS := \
   $(TOPSRCDIR)/allmakefiles.sh \
   $(TOPSRCDIR)/CLOBBER \
   $(TOPSRCDIR)/nsprpub/configure \
   $(TOPSRCDIR)/config/milestone.txt \
   $(TOPSRCDIR)/js/src/config/milestone.txt \
   $(TOPSRCDIR)/browser/config/version.txt \
   $(TOPSRCDIR)/build/virtualenv/packages.txt \
   $(TOPSRCDIR)/build/virtualenv/populate_virtualenv.py \
+  $(TOPSRCDIR)/testing/mozbase/packages.txt \
   $(NULL)
 
 CONFIGURE_ENV_ARGS += \
   MAKE="$(MAKE)" \
   $(NULL)
 
 # configure uses the program name to determine @srcdir@. Calling it without
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full