bug 439050, JarMaker.py is a rewrite of make-jars.pl to only launch a single process per jar.mn, r=ted
authorAxel Hecht <l10n@mozilla.com>
Fri, 19 Sep 2008 18:19:52 +0200
changeset 19432 9f5e80c5cae293aa5bc334700d02203e4f2302b6
parent 19431 06d7e4e85aa0415881673c9cd9fe8a7115756897
child 19433 5c00820e1f18debf236caf6549f4af4caac62f65
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-beta@bfdb6e623a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs439050
milestone1.9.1b1pre
bug 439050, JarMaker.py is a rewrite of make-jars.pl to only launch a single process per jar.mn, r=ted
config/JarMaker.py
config/MozZipFile.py
config/Preprocessor.py
config/utils.py
new file mode 100644
--- /dev/null
+++ b/config/JarMaker.py
@@ -0,0 +1,436 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla build system.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''jarmaker.py provides a python class to package up chrome content by
+processing jar.mn files.
+
+See the documentation for jar.mn on MDC for further details on the format.
+'''
+
+import sys
+import os
+import os.path
+import re
+import logging
+from time import localtime
+from optparse import OptionParser
+from MozZipFile import ZipFile
+from cStringIO import StringIO
+from datetime import datetime
+
+from utils import pushback_iter
+from Preprocessor import Preprocessor
+
+__all__ = ['JarMaker']
+
+class ZipEntry:
+  '''Helper class for jar output.
+
+  This class defines a simple file-like object for a zipfile.ZipEntry
+  so that we can consecutively write to it and then close it.
+  This methods hooks into ZipFile.writestr on close().
+  '''
+  def __init__(self, name, zipfile):
+    self._zipfile = zipfile
+    self._name = name
+    self._inner = StringIO()
+
+  def write(self, content):
+    'Append the given content to this zip entry'
+    self._inner.write(content)
+    return
+
+  def close(self):
+    'The close method writes the content back to the zip file.'
+    self._zipfile.writestr(self._name, self._inner.getvalue())
+
+def getModTime(aPath):
+  if not os.path.isfile(aPath):
+    return 0
+  mtime = os.stat(aPath).st_mtime
+  return localtime(mtime)
+
+
+class JarMaker(object):
+  '''JarMaker reads jar.mn files and process those into jar files or
+  flat directories, along with chrome.manifest files.
+  '''
+
+  ignore = re.compile('\s*(\#.*)?$')
+  jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
+  regline = re.compile('\%\s+(.*)$')
+  entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
+  entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
+
+  def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
+               useChromeManifest = False):
+    self.outputFormat = outputFormat
+    self.useJarfileManifest = useJarfileManifest
+    self.useChromeManifest = useChromeManifest
+    self.pp = Preprocessor()
+
+  def getCommandLineParser(self):
+    '''Get a optparse.OptionParser for jarmaker.
+
+    This OptionParser has the options for jarmaker as well as
+    the options for the inner PreProcessor.
+    '''
+    # HACK, we need to unescape the string variables we get,
+    # the perl versions didn't grok strings right
+    p = self.pp.getCommandLineParser(unescapeDefines = True)
+    p.add_option('-f', type="choice", default="jar",
+                 choices=('jar', 'flat', 'symlink'),
+                 help="fileformat used for output", metavar="[jar, flat, symlink]")
+    p.add_option('-v', action="store_true", dest="verbose",
+                 help="verbose output")
+    p.add_option('-q', action="store_false", dest="verbose",
+                 help="verbose output")
+    p.add_option('-e', action="store_true",
+                 help="create chrome.manifest instead of jarfile.manifest")
+    p.add_option('-s', type="string", action="append", default=[],
+                 help="source directory")
+    p.add_option('-t', type="string",
+                 help="top source directory")
+    p.add_option('-c', '--l10n-src', type="string", action="append",
+                 help="localization directory")
+    p.add_option('--l10n-base', type="string", action="append", default=[],
+                 help="base directory to be used for localization (multiple)")
+    p.add_option('-j', type="string",
+                 help="jarfile directory")
+    # backwards compat, not needed
+    p.add_option('-a', action="store_false", default=True,
+                 help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
+    p.add_option('-d', type="string",
+                 help="UNUSED, chrome directory")
+    p.add_option('-o', help="cross compile for auto-registration, ignored")
+    p.add_option('-l', action="store_true",
+                 help="ignored (used to switch off locks)")
+    p.add_option('-x', action="store_true",
+                 help="force Unix")
+    p.add_option('-z', help="backwards compat, ignored")
+    p.add_option('-p', help="backwards compat, ignored")
+    return p
+
+  def processIncludes(self, includes):
+    '''Process given includes with the inner PreProcessor.
+
+    Only use this for #defines, the includes shouldn't generate
+    content.
+    '''
+    self.pp.out = StringIO()
+    for inc in includes:
+      self.pp.do_include(inc)
+    includesvalue = self.pp.out.getvalue()
+    if includesvalue:
+      logging.info("WARNING: Includes produce non-empty output")
+    self.pp.out = None
+    pass
+
+  def finalizeJar(self, jarPath, chromebasepath, register,
+                   doZip=True):
+    '''Helper method to write out the chrome registration entries to
+    jarfile.manifest or chrome.manifest, or both.
+
+    The actual file processing is done in updateManifest.
+    '''
+    # rewrite the manifest, if entries given
+    if not register:
+      return
+    if self.useJarfileManifest:
+      self.updateManifest(jarPath + '.manifest', chromebasepath % '',
+                          register)
+    if self.useChromeManifest:
+      manifestPath = os.path.join(os.path.dirname(jarPath),
+                                  '..', 'chrome.manifest')
+      self.updateManifest(manifestPath, chromebasepath % 'chrome/',
+                          register)
+
+  def updateManifest(self, manifestPath, chromebasepath, register):
+    '''updateManifest replaces the % in the chrome registration entries
+    with the given chrome base path, and updates the given manifest file.
+    '''
+    myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
+                                   register.iterkeys()))
+    manifestExists = os.path.isfile(manifestPath)
+    mode = (manifestExists and 'r+b') or 'wb'
+    mf = open(manifestPath, mode)
+    if manifestExists:
+      # import previous content into hash, ignoring empty ones and comments
+      imf = re.compile('(#.*)?$')
+      for l in re.split('[\r\n]+', mf.read()):
+        if imf.match(l):
+          continue
+        myregister[l] = None
+      mf.seek(0)
+    for k in myregister.iterkeys():
+      mf.write(k + os.linesep)
+    mf.close()
+  
+  def makeJar(self, infile=None,
+               jardir='',
+               sourcedirs=[], topsourcedir='', localedirs=None):
+    '''makeJar is the main entry point to JarMaker.
+
+    It takes the input file, the output directory, the source dirs and the
+    top source dir as argument, and optionally the l10n dirs.
+    '''
+    if isinstance(infile, basestring):
+      logging.info("processing " + infile)
+    pp = self.pp.clone()
+    pp.out = StringIO()
+    pp.do_include(infile)
+    lines = pushback_iter(pp.out.getvalue().splitlines())
+    try:
+      while True:
+        l = lines.next()
+        m = self.jarline.match(l)
+        if not m:
+          raise RuntimeError(l)
+        if m.group('jarfile') is None:
+          # comment
+          continue
+        self.processJarSection(m.group('jarfile'), lines,
+                               jardir, sourcedirs, topsourcedir,
+                               localedirs)
+    except StopIteration:
+      # we read the file
+      pass
+    return
+
+  def processJarSection(self, jarfile, lines,
+                        jardir, sourcedirs, topsourcedir, localedirs):
+    '''Internal method called by makeJar to actually process a section
+    of a jar.mn file.
+
+    jarfile is the basename of the jarfile or the directory name for 
+    flat output, lines is a pushback_iterator of the lines of jar.mn,
+    the remaining options are carried over from makeJar.
+    '''
+
+    # chromebasepath is used for chrome registration manifests
+    # %s is getting replaced with chrome/ for chrome.manifest, and with
+    # an empty string for jarfile.manifest
+    chromebasepath = '%s' + jarfile
+    if self.outputFormat == 'jar':
+      chromebasepath = 'jar:' + chromebasepath + '.jar!'
+    chromebasepath += '/'
+
+    jarfile = os.path.join(jardir, jarfile)
+    jf = None
+    if self.outputFormat == 'jar':
+      #jar
+      jarfilepath = jarfile + '.jar'
+      if os.path.isfile(jarfilepath) and \
+            os.path.getsize(jarfilepath) > 0:
+        jf = ZipFile(jarfilepath, 'a', lock = True)
+      else:
+        if not os.path.isdir(os.path.dirname(jarfilepath)):
+          os.makedirs(os.path.dirname(jarfilepath))
+        jf = ZipFile(jarfilepath, 'w', lock = True)
+      outHelper = self.OutputHelper_jar(jf)
+    else:
+      outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
+    register = {}
+    # This loop exits on either
+    # - the end of the jar.mn file
+    # - an line in the jar.mn file that's not part of a jar section
+    while True:
+      try:
+        l = lines.next()
+      except StopIteration:
+        # we're done with this jar.mn, and this jar section
+        self.finalizeJar(jarfile, chromebasepath, register)
+        if jf is not None:
+          jf.close()
+        # reraise the StopIteration for makeJar
+        raise
+      if self.ignore.match(l):
+        continue
+      m = self.regline.match(l)
+      if  m:
+        rline = m.group(1)
+        register[rline] = 1
+        continue
+      m = self.entryline.match(l)
+      if not m:
+        # neither an entry line nor chrome reg, this jar section is done
+        self.finalizeJar(jarfile, chromebasepath, register)
+        if jf is not None:
+          jf.close()
+        lines.pushback(l)
+        return
+      self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
+                            outHelper, jf)
+    return
+
+  def _processEntryLine(self, m, 
+                        sourcedirs, topsourcedir, localedirs,
+                        outHelper, jf):
+      out = m.group('output')
+      src = m.group('source') or os.path.basename(out)
+      # pick the right sourcedir -- l10n, topsrc or src
+      if m.group('locale'):
+        src_base = localedirs
+      elif src.startswith('/'):
+        # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
+        # refers to a path relative to topsourcedir, use that as base
+        # and strip the leading '/'
+        src_base = [topsourcedir]
+        src = src[1:]
+      else:
+        # use srcdirs and the objdir (current working dir) for relative paths
+        src_base = sourcedirs + ['.']
+      # check if the source file exists
+      realsrc = None
+      for _srcdir in src_base:
+        if os.path.isfile(os.path.join(_srcdir, src)):
+          realsrc = os.path.join(_srcdir, src)
+          break
+      if realsrc is None:
+        if jf is not None:
+          jf.close()
+        raise RuntimeError("file not found: " + src)
+      if m.group('optPreprocess'):
+        outf = outHelper.getOutput(out)
+        inf = open(realsrc)
+        pp = self.pp.clone()
+        if src[-4:] == '.css':
+          pp.setMarker('%')
+        pp.out = outf
+        pp.do_include(inf)
+        outf.close()
+        inf.close()
+        return
+      # copy or symlink if newer or overwrite
+      if (m.group('optOverwrite')
+          or (getModTime(realsrc) >
+              outHelper.getDestModTime(m.group('output')))):
+        if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
+          outHelper.symlink(realsrc, out)
+          return
+        outf = outHelper.getOutput(out)
+        # open in binary mode, this can be images etc
+        inf = open(realsrc, 'rb')
+        outf.write(inf.read())
+        outf.close()
+        inf.close()
+    
+
+  class OutputHelper_jar(object):
+    '''Provide getDestModTime and getOutput for a given jarfile.
+    '''
+    def __init__(self, jarfile):
+      self.jarfile = jarfile
+    def getDestModTime(self, aPath):
+      try :
+        info = self.jarfile.getinfo(aPath)
+        return info.date_time
+      except:
+        return 0
+    def getOutput(self, name):
+      return ZipEntry(name, self.jarfile)
+
+  class OutputHelper_flat(object):
+    '''Provide getDestModTime and getOutput for a given flat
+    output directory. The helper method ensureDirFor is used by
+    the symlink subclass.
+    '''
+    def __init__(self, basepath):
+      self.basepath = basepath
+    def getDestModTime(self, aPath):
+      return getModTime(os.path.join(self.basepath, aPath))
+    def getOutput(self, name):
+      out = self.ensureDirFor(name)
+      return open(out, 'wb')
+    def ensureDirFor(self, name):
+      out = os.path.join(self.basepath, name)
+      outdir = os.path.dirname(out)
+      if not os.path.isdir(outdir):
+        os.makedirs(outdir)
+      return out
+
+  class OutputHelper_symlink(OutputHelper_flat):
+    '''Subclass of OutputHelper_flat that provides a helper for
+    creating a symlink including creating the parent directories.
+    '''
+    def symlink(self, src, dest):
+      out = self.ensureDirFor(dest)
+      os.symlink(src, out)
+
+def main():
+  jm = JarMaker()
+  p = jm.getCommandLineParser()
+  (options, args) = p.parse_args()
+  jm.processIncludes(options.I)
+  jm.outputFormat = options.f
+  if options.e:
+    jm.useChromeManifest = True
+    jm.useJarfileManifest = False
+  noise = logging.INFO
+  if options.verbose is not None:
+    noise = (options.verbose and logging.DEBUG) or logging.WARN
+  logging.basicConfig(level = noise, format = "%(message)s")
+  if not args:
+    jm.makeJar(infile=sys.stdin,
+               sourcedirs=options.s, topsourcedir=options.t,
+               localedirs=options.l10n_src,
+               jardir=options.j)
+    return
+  topsrc = options.t
+  topsrc = os.path.normpath(os.path.abspath(topsrc))
+  for infile in args:
+    # guess srcdir and l10n dirs from jar.mn path and topsrcdir
+    # srcdir is the dir of the jar.mn and
+    # the l10n dirs are the relative path of topsrcdir to srcdir
+    # resolved against all l10n base dirs.
+    srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
+    l10ndir = srcdir
+    if os.path.basename(srcdir) == 'locales':
+      l10ndir = os.path.dirname(l10ndir)
+    assert srcdir.startswith(topsrc), "src dir %s not in topsrcdir %s" % (srcdir, topsrc)
+    rell10ndir = l10ndir[len(topsrc):].lstrip(os.sep)
+    l10ndirs = map(lambda d: os.path.join(d, rell10ndir), options.l10n_base)
+    if options.l10n_src is not None:
+      l10ndirs += options.l10n_src
+    srcdirs = options.s + [srcdir]
+    jm.makeJar(infile=infile,
+               sourcedirs=srcdirs, topsourcedir=options.t,
+               localedirs=l10ndirs,
+               jardir=options.j)
+
+if __name__ == "__main__":
+  main()
--- a/config/MozZipFile.py
+++ b/config/MozZipFile.py
@@ -34,29 +34,36 @@
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 import zipfile
 import time
 import binascii, struct
 import zlib
+from utils import lockFile
 
 
 class ZipFile(zipfile.ZipFile):
   """ Class with methods to open, read, write, close, list zip files.
 
   Subclassing zipfile.ZipFile to allow for overwriting of existing
   entries, though only for writestr, not for write.
   """
-  def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED):
+  def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED,
+               lock = False):
     zipfile.ZipFile.__init__(self, file, mode, compression)
     self._remove = []
     self.end = self.fp.tell()
     self.debug = 0
+    if lock:
+      assert isinstance(file, basestring)
+      self.lockfile = lockFile(file + '.lck')
+    else:
+      self.lockfile = None
 
   def writestr(self, zinfo_or_arcname, bytes):
     """Write contents into the archive.
 
     The contents is the argument 'bytes',  'zinfo_or_arcname' is either
     a ZipInfo instance or the name of the file in the archive.
     This method is overloaded to allow overwriting existing entries.
     """
@@ -115,17 +122,19 @@ class ZipFile(zipfile.ZipFile):
   def close(self):
     """Close the file, and for mode "w" and "a" write the ending
     records.
 
     Overwritten to compact overwritten entries.
     """
     if not self._remove:
       # we don't have anything special to do, let's just call base
-      return zipfile.ZipFile.close(self)
+      r = zipfile.ZipFile.close(self)
+      self.lockfile = None
+      return r
 
     if self.fp.mode != 'r+b':
       # adjust file mode if we originally just wrote, now we rewrite
       self.fp.close()
       self.fp = open(self.filename, 'r+b')
     all = map(lambda zi: (zi, True), self.filelist) + \
         map(lambda zi: (zi, False), self._remove)
     all.sort(lambda l, r: cmp(l[0].header_offset, r[0].header_offset))
@@ -143,8 +152,9 @@ class ZipFile(zipfile.ZipFile):
       zi.header_offset = to_pos
       self.fp.seek(oldoff)
       content = self.fp.read(length)
       self.fp.seek(to_pos)
       self.fp.write(content)
       to_pos += length
     self.fp.truncate()
     zipfile.ZipFile.close(self)
+    self.lockfile = None
--- a/config/Preprocessor.py
+++ b/config/Preprocessor.py
@@ -44,18 +44,19 @@ import sys
 import os
 import os.path
 import re
 from optparse import OptionParser
 
 # hack around win32 mangling our line endings
 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
 if sys.platform == "win32":
-  import os, msvcrt
+  import msvcrt
   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+  os.linesep = '\n'
 
 import Expression
 
 __all__ = ['Preprocessor', 'preprocess']
 
 
 class Preprocessor:
   """
@@ -152,60 +153,66 @@ class Preprocessor:
     aLine = re.sub('\n', self.LE, aLine)
     self.out.write(aLine)
   
   def handleCommandLine(self, args, defaultToStdin = False):
     """
     Parse a commandline into this parser.
     Uses OptionParser internally, no args mean sys.argv[1:].
     """
-    includes = []
-    def handleI(option, opt, value, parser):
-      includes.append(value)
+    p = self.getCommandLineParser()
+    (options, args) = p.parse_args(args=args)
+    includes = options.I
+    if defaultToStdin and len(args) == 0:
+      args = [sys.stdin]
+    includes.extend(args)
+    for f in includes:
+      self.do_include(f)
+    pass
+
+  def getCommandLineParser(self, unescapeDefines = False):
+    escapedValue = re.compile('".*"$')
     def handleE(option, opt, value, parser):
       for k,v in os.environ.iteritems():
         self.context[k] = v
     def handleD(option, opt, value, parser):
       vals = value.split('=')
       assert len(vals) < 3
       if len(vals) == 1:
         vals.append(1)
+      elif unescapeDefines and escapedValue.match(vals[1]):
+        # strip escaped string values
+        vals[1] = vals[1][1:-1]
       self.context[vals[0]] = vals[1]
     def handleU(option, opt, value, parser):
       del self.context[value]
     def handleF(option, opt, value, parser):
       self.do_filter(value)
     def handleLE(option, opt, value, parser):
       self.setLineEndings(value)
     def handleMarker(option, opt, value, parser):
       self.setMarker(value)
     p = OptionParser()
-    p.add_option('-I', action='callback', callback=handleI, type="string",
+    p.add_option('-I', action='append', type="string", default = [],
                  metavar="FILENAME", help='Include file')
     p.add_option('-E', action='callback', callback=handleE,
                  help='Import the environment into the defined variables')
     p.add_option('-D', action='callback', callback=handleD, type="string",
                  metavar="VAR[=VAL]", help='Define a variable')
     p.add_option('-U', action='callback', callback=handleU, type="string",
                  metavar="VAR", help='Undefine a variable')
     p.add_option('-F', action='callback', callback=handleF, type="string",
                  metavar="FILTER", help='Enable the specified filter')
     p.add_option('--line-endings', action='callback', callback=handleLE,
                  type="string", metavar="[cr|lr|crlf]",
                  help='Use the specified line endings [Default: OS dependent]')
     p.add_option('--marker', action='callback', callback=handleMarker,
                  type="string",
                  help='Use the specified marker instead of #')
-    (options, args) = p.parse_args(args=args)
-    if defaultToStdin and len(args) == 0:
-      args = [sys.stdin]
-    includes.extend(args)
-    for f in includes:
-      self.do_include(f)
-    pass
+    return p
 
   def handleLine(self, aLine):
     """
     Handle a single line of input (internal).
     """
     m = self.instruction.match(aLine)
     if m:
       args = None
new file mode 100644
--- /dev/null
+++ b/config/utils.py
@@ -0,0 +1,139 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla build system.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Axel Hecht <axel@pike.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''Utility methods to be used by python build infrastructure.
+'''
+
+import os
+import errno
+import sys
+import time
+import stat
+
+class LockFile(object):
+  '''LockFile is used by the lockFile method to hold the lock.
+
+  This object should not be used directly, but only through
+  the lockFile method below.
+  '''
+  def __init__(self, lockfile):
+    self.lockfile = lockfile
+  def __del__(self):
+    os.remove(self.lockfile)
+
+
+def lockFile(lockfile, max_wait = 600):
+  '''Create and hold a lockfile of the given name, with the given timeout.
+
+  To release the lock, delete the returned object.
+  '''
+  while True:
+    try:
+      fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+      # we created the lockfile, so we're the owner
+      break
+    except OSError, e:
+      if e.errno != errno.EEXIST:
+        # should not occur
+        raise
+  
+    try:
+      # the lock file exists, try to stat it to get its age
+      # and read it's contents to report the owner PID
+      f = open(lockfile, "r")
+      s = os.stat(lockfile)
+    except OSError, e:
+      if e.errno != errno.ENOENT:
+        sys.exit("%s exists but stat() failed: %s" %
+                 (lockfile, e.strerror))
+      # we didn't create the lockfile, so it did exist, but it's
+      # gone now. Just try again
+      continue
+  
+    # we didn't create the lockfile and it's still there, check
+    # its age
+    now = int(time.time())
+    if now - s[stat.ST_MTIME] > max_wait:
+      pid = f.readline()
+      sys.exit("%s has been locked for more than " \
+               "%d seconds (PID %s)" % (lockfile, max_wait,
+                                        pid))
+  
+    # it's not been locked too long, wait a while and retry
+    f.close()
+    time.sleep(1)
+  
+  # if we get here. we have the lockfile. Convert the os.open file
+  # descriptor into a Python file object and record our PID in it
+  
+  f = os.fdopen(fd, "w")
+  f.write("%d\n" % os.getpid())
+  f.close()
+  return LockFile(lockfile)
+
+class pushback_iter(object):
+  '''Utility iterator that can deal with pushed back elements.
+
+  This behaves like a regular iterable, just that you can call
+    iter.pushback(item)
+  to get the givem item as next item in the iteration.
+  '''
+  def __init__(self, iterable):
+    self.it = iter(iterable)
+    self.pushed_back = []
+
+  def __iter__(self):
+    return self
+
+  def __nonzero__(self):
+    if self.pushed_back:
+      return True
+
+    try:
+      self.pushed_back.insert(0, self.it.next())
+    except StopIteration:
+      return False
+    else:
+      return True
+
+  def next(self):
+    if self.pushed_back:
+      return self.pushed_back.pop()
+    return self.it.next()
+
+  def pushback(self, item):
+    self.pushed_back.append(item)