Bug 924331 - Move config/utils.py into mozbuild; r=mshal
authorGregory Szorc <gps@mozilla.com>
Fri, 11 Oct 2013 08:23:18 -0700
changeset 150481 4a2c4921ff7cabb2ca41bdf30b0f58b0a9e16c67
parent 150480 d7f29909c06f4145645353bed7b2413fc4c4da03
child 150482 342e9d40235aea91cb28a4e3a10e61ffe7b48bc0
child 150519 0f1efa209f534408d2188a55573e269c3a391dce
push id25442
push userryanvm@gmail.com
push dateFri, 11 Oct 2013 19:39:21 +0000
treeherdermozilla-central@342e9d40235a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs924331
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 924331 - Move config/utils.py into mozbuild; r=mshal
config/JarMaker.py
config/MozZipFile.py
config/utils.py
python/mozbuild/mozbuild/action/buildlist.py
python/mozbuild/mozbuild/util.py
--- a/config/JarMaker.py
+++ b/config/JarMaker.py
@@ -12,17 +12,21 @@ import os
 import os.path
 import errno
 import re
 import logging
 from time import localtime
 from MozZipFile import ZipFile
 from cStringIO import StringIO
 
-from utils import pushback_iter, lockFile
+from mozbuild.util import (
+    lock_file,
+    PushbackIter,
+)
+
 from Preprocessor import Preprocessor
 from mozbuild.action.buildlist import addEntriesToListFile
 if sys.platform == "win32":
   from ctypes import windll, WinError
   CreateHardLink = windll.kernel32.CreateHardLinkA
 
 __all__ = ['JarMaker']
 
@@ -167,17 +171,17 @@ class JarMaker(object):
       chromeDir = os.path.basename(os.path.dirname(os.path.normpath(chromeManifest)))
       logging.info("adding '%s' entry to root chrome manifest appid=%s" % (chromeDir, self.rootManifestAppId))
       addEntriesToListFile(rootChromeManifest, ['manifest %s/chrome.manifest application=%s' % (chromeDir, self.rootManifestAppId)])
 
   def updateManifest(self, manifestPath, chromebasepath, register):
     '''updateManifest replaces the % in the chrome registration entries
     with the given chrome base path, and updates the given manifest file.
     '''
-    lock = lockFile(manifestPath + '.lck')
+    lock = lock_file(manifestPath + '.lck')
     try:
       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
                                      register.iterkeys()))
       manifestExists = os.path.isfile(manifestPath)
       mode = (manifestExists and 'r+b') or 'wb'
       mf = open(manifestPath, mode)
       if manifestExists:
         # import previous content into hash, ignoring empty ones and comments
@@ -208,17 +212,17 @@ class JarMaker(object):
     elif self.relativesrcdir:
       self.localedirs = self.generateLocaleDirs(self.relativesrcdir)
     if isinstance(infile, basestring):
       logging.info("processing " + infile)
       self.sourcedirs.append(_normpath(os.path.dirname(infile)))
     pp = self.pp.clone()
     pp.out = StringIO()
     pp.do_include(infile)
-    lines = pushback_iter(pp.out.getvalue().splitlines())
+    lines = PushbackIter(pp.out.getvalue().splitlines())
     try:
       while True:
         l = lines.next()
         m = self.jarline.match(l)
         if not m:
           raise RuntimeError(l)
         if m.group('jarfile') is None:
           # comment
@@ -245,18 +249,18 @@ class JarMaker(object):
       # add en-US if we merge, or if it's not l10n
       locdirs.append(os.path.join(self.topsourcedir, relativesrcdir, 'en-US'))
     return locdirs
 
   def processJarSection(self, jarfile, lines, jardir):
     '''Internal method called by makeJar to actually process a section
     of a jar.mn file.
 
-    jarfile is the basename of the jarfile or the directory name for 
-    flat output, lines is a pushback_iterator of the lines of jar.mn,
+    jarfile is the basename of the jarfile or the directory name for
+    flat output, lines is a PushbackIter of the lines of jar.mn,
     the remaining options are carried over from makeJar.
     '''
 
     # chromebasepath is used for chrome registration manifests
     # {0} is getting replaced with chrome/ for chrome.manifest, and with
     # an empty string for jarfile.manifest
     chromebasepath = '{0}' + os.path.basename(jarfile)
     if self.outputFormat == 'jar':
--- a/config/MozZipFile.py
+++ b/config/MozZipFile.py
@@ -1,30 +1,30 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import zipfile
+import os
 import time
-import binascii, struct
-import zlib
-import os
-from utils import lockFile
+import zipfile
+
+from mozbuild.util import lock_file
+
 
 class ZipFile(zipfile.ZipFile):
   """ Class with methods to open, read, write, close, list zip files.
 
   Subclassing zipfile.ZipFile to allow for overwriting of existing
   entries, though only for writestr, not for write.
   """
   def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED,
                lock = False):
     if lock:
       assert isinstance(file, basestring)
-      self.lockfile = lockFile(file + '.lck')
+      self.lockfile = lock_file(file + '.lck')
     else:
       self.lockfile = None
 
     if mode == 'a' and lock:
       # appending to a file which doesn't exist fails, but we can't check
       # existence util we hold the lock
       if (not os.path.isfile(file)) or os.path.getsize(file) == 0:
         mode = 'w'
deleted file mode 100644
--- a/config/utils.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# >>sys.stderr,  This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-'''Utility methods to be used by python build infrastructure.
-'''
-
-import os
-import errno
-import sys
-import time
-import stat
-
-class LockFile(object):
-  '''LockFile is used by the lockFile method to hold the lock.
-
-  This object should not be used directly, but only through
-  the lockFile method below.
-  '''
-  def __init__(self, lockfile):
-    self.lockfile = lockfile
-  def __del__(self):
-    while True:
-      try:
-        os.remove(self.lockfile)
-        break
-      except OSError as e:
-        if e.errno == errno.EACCES:
-          # another process probably has the file open, we'll retry.
-          # just a short sleep since we want to drop the lock ASAP
-          # (but we need to let some other process close the file first)
-          time.sleep(0.1)
-        else:
-          # re-raise unknown errors
-          raise
-
-def lockFile(lockfile, max_wait = 600):
-  '''Create and hold a lockfile of the given name, with the given timeout.
-
-  To release the lock, delete the returned object.
-  '''
-  while True:
-    try:
-      fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
-      # we created the lockfile, so we're the owner
-      break
-    except OSError as e:
-      if (e.errno == errno.EEXIST or 
-          (sys.platform == "win32" and e.errno == errno.EACCES)):
-        pass
-      else:
-        # should not occur
-        raise
-  
-    try:
-      # the lock file exists, try to stat it to get its age
-      # and read its contents to report the owner PID
-      f = open(lockfile, "r")
-      s = os.stat(lockfile)
-    except EnvironmentError as e:
-      if e.errno == errno.ENOENT or e.errno == errno.EACCES:
-        # we didn't create the lockfile, so it did exist, but it's
-        # gone now. Just try again
-        continue
-      sys.exit("{0} exists but stat() failed: {1}"
-               .format(lockfile, e.strerror))
-  
-    # we didn't create the lockfile and it's still there, check
-    # its age
-    now = int(time.time())
-    if now - s[stat.ST_MTIME] > max_wait:
-      pid = f.readline().rstrip()
-      sys.exit("{0} has been locked for more than "
-               "{1} seconds (PID {2})".format(lockfile, max_wait, pid))
-  
-    # it's not been locked too long, wait a while and retry
-    f.close()
-    time.sleep(1)
-  
-  # if we get here. we have the lockfile. Convert the os.open file
-  # descriptor into a Python file object and record our PID in it
-  
-  f = os.fdopen(fd, "w")
-  f.write("{0}\n".format(os.getpid()))
-  f.close()
-  return LockFile(lockfile)
-
-class pushback_iter(object):
-  '''Utility iterator that can deal with pushed back elements.
-
-  This behaves like a regular iterable, just that you can call
-    iter.pushback(item)
-  to get the given item as next item in the iteration.
-  '''
-  def __init__(self, iterable):
-    self.it = iter(iterable)
-    self.pushed_back = []
-
-  def __iter__(self):
-    return self
-
-  def __nonzero__(self):
-    if self.pushed_back:
-      return True
-
-    try:
-      self.pushed_back.insert(0, self.it.next())
-    except StopIteration:
-      return False
-    else:
-      return True
-
-  def next(self):
-    if self.pushed_back:
-      return self.pushed_back.pop()
-    return self.it.next()
-
-  def pushback(self, item):
-    self.pushed_back.append(item)
--- a/python/mozbuild/mozbuild/action/buildlist.py
+++ b/python/mozbuild/mozbuild/action/buildlist.py
@@ -6,23 +6,24 @@
 if the entry does not already exist.
 
 Usage: buildlist.py <filename> <entry> [<entry> ...]
 '''
 from __future__ import print_function
 
 import sys
 import os
-from utils import lockFile
+
+from mozbuild.util import lock_file
 
 def addEntriesToListFile(listFile, entries):
   """Given a file |listFile| containing one entry per line,
   add each entry in |entries| to the file, unless it is already
   present."""
-  lock = lockFile(listFile + ".lck")
+  lock = lock_file(listFile + ".lck")
   try:
     if os.path.exists(listFile):
       f = open(listFile)
       existing = set(x.strip() for x in f.readlines())
       f.close()
     else:
       existing = set()
     f = open(listFile, 'a')
--- a/python/mozbuild/mozbuild/util.py
+++ b/python/mozbuild/mozbuild/util.py
@@ -6,20 +6,23 @@
 # in particular.
 
 from __future__ import unicode_literals
 
 import copy
 import errno
 import hashlib
 import os
+import stat
 import sys
+import time
 
 from StringIO import StringIO
 
+
 if sys.version_info[0] == 3:
     str_type = str
 else:
     str_type = basestring
 
 def hash_file(path):
     """Hashes a file specified by the path given and returns the hex digest."""
 
@@ -365,8 +368,125 @@ class HierarchicalStringList(object):
 
     def _check_list(self, value):
         if not isinstance(value, list):
             raise ValueError('Expected a list of strings, not %s' % type(value))
         for v in value:
             if not isinstance(v, str_type):
                 raise ValueError(
                     'Expected a list of strings, not an element of %s' % type(v))
+
+
+class LockFile(object):
+    """LockFile is used by the lock_file method to hold the lock.
+
+    This object should not be used directly, but only through
+    the lock_file method below.
+    """
+
+    def __init__(self, lockfile):
+        self.lockfile = lockfile
+
+    def __del__(self):
+        while True:
+            try:
+                os.remove(self.lockfile)
+                break
+            except OSError as e:
+                if e.errno == errno.EACCES:
+                    # Another process probably has the file open, we'll retry.
+                    # Just a short sleep since we want to drop the lock ASAP
+                    # (but we need to let some other process close the file
+                    # first).
+                    time.sleep(0.1)
+            else:
+                # Re-raise unknown errors
+                raise
+
+
+def lock_file(lockfile, max_wait = 600):
+    """Create and hold a lockfile of the given name, with the given timeout.
+
+    To release the lock, delete the returned object.
+    """
+
+    # FUTURE This function and object could be written as a context manager.
+
+    while True:
+        try:
+            fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+            # We created the lockfile, so we're the owner
+            break
+        except OSError as e:
+            if (e.errno == errno.EEXIST or
+                (sys.platform == "win32" and e.errno == errno.EACCES)):
+                pass
+            else:
+                # Should not occur
+                raise
+
+        try:
+            # The lock file exists, try to stat it to get its age
+            # and read its contents to report the owner PID
+            f = open(lockfile, 'r')
+            s = os.stat(lockfile)
+        except EnvironmentError as e:
+            if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+            # We didn't create the lockfile, so it did exist, but it's
+            # gone now. Just try again
+                continue
+
+            raise Exception('{0} exists but stat() failed: {1}'.format(
+                lockfile, e.strerror))
+
+        # We didn't create the lockfile and it's still there, check
+        # its age
+        now = int(time.time())
+        if now - s[stat.ST_MTIME] > max_wait:
+            pid = f.readline().rstrip()
+            raise Exception('{0} has been locked for more than '
+                '{1} seconds (PID {2})'.format(lockfile, max_wait, pid))
+
+        # It's not been locked too long, wait a while and retry
+        f.close()
+        time.sleep(1)
+
+    # if we get here. we have the lockfile. Convert the os.open file
+    # descriptor into a Python file object and record our PID in it
+    f = os.fdopen(fd, 'w')
+    f.write('{0}\n'.format(os.getpid()))
+    f.close()
+
+    return LockFile(lockfile)
+
+
+class PushbackIter(object):
+    '''Utility iterator that can deal with pushed back elements.
+
+    This behaves like a regular iterable, just that you can call
+    iter.pushback(item) to get the given item as next item in the
+    iteration.
+    '''
+    def __init__(self, iterable):
+        self.it = iter(iterable)
+        self.pushed_back = []
+
+    def __iter__(self):
+        return self
+
+    def __nonzero__(self):
+        if self.pushed_back:
+            return True
+
+        try:
+            self.pushed_back.insert(0, self.it.next())
+        except StopIteration:
+            return False
+        else:
+            return True
+
+    def next(self):
+        if self.pushed_back:
+            return self.pushed_back.pop()
+        return self.it.next()
+
+    def pushback(self, item):
+        self.pushed_back.append(item)