Bug 784841 - Part 15: Produce moz.build files for dom imported tests; r=Ms2ger
authorGregory Szorc <gps@mozilla.com>
Mon, 25 Feb 2013 12:46:50 -0800
changeset 123218 c1b0d8399be92a0a541fddec5e580196e7b83e32
parent 122888 06935f2db2679c12ba434c01ddf1d0bab62439c2
child 123219 23673de2aa432551cb25c993238ccc1be2ae9b1f
push id1
push userroot
push dateMon, 20 Oct 2014 17:29:22 +0000
reviewersMs2ger
bugs784841
milestone22.0a1
Bug 784841 - Part 15: Produce moz.build files for dom imported tests; r=Ms2ger
dom/imptests/README
dom/imptests/importTestsuite.py
dom/imptests/parseFailures.py
dom/imptests/updateTestharness.py
dom/imptests/writeBuildFiles.py
dom/imptests/writeMakefile.py
--- a/dom/imptests/README
+++ b/dom/imptests/README
@@ -52,18 +52,18 @@ Source;  Usage and purpose;  License
 * parseFailures.py
   Parses failures out of a mochitest log and writes out JSON files and Makefiles
   into the correct failures/ folder.
   The mochitest log should be produced by setting the 'dumpFailures' flag in
   testharnessreport.js; this will print out the encountered failures, marked
   by @ signs.
   MPL
 
-* writeMakefile.py
-  Helper functions to write out Makefiles.
+* writeBuildFiles.py
+  Helper functions to write out automatically generated build files.
   MPL
 
 * Makefile.in
   Integration with our build system. Installs support files into /resources and
   includes a .mk file for each repository.
   MPL
 
 * failures/
--- a/dom/imptests/importTestsuite.py
+++ b/dom/imptests/importTestsuite.py
@@ -13,17 +13,17 @@ Note: removes both source and destinatio
 from __future__ import print_function, unicode_literals
 
 import os
 import shutil
 import subprocess
 import sys
 
 import parseManifest
-import writeMakefile
+import writeBuildFiles
 
 HEADERS_SUFFIX = "^headers^"
 
 def parseManifestFile(dest, dir):
     subdirs, mochitests, _, __, supportfiles = parseManifest.parseManifestFile("hg-%s/%s/MANIFEST" % (dest, dir))
     return subdirs, mochitests, supportfiles
 
 def getData(confFile):
@@ -31,28 +31,27 @@ def getData(confFile):
     URL of remote repository|Name of the destination directory
     First directory of tests
     ...
     Last directory of tests"""
     repo = ""
     dest = ""
     directories = []
     try:
-        fp = open(confFile, "r")
-        first = True
-        for line in fp:
-            if first:
-                idx = line.index("|")
-                repo = line[:idx].strip()
-                dest = line[idx + 1:].strip()
-                first = False
-            else:
-                directories.append(line.strip())
+        with open(confFile, 'r') as fp:
+            first = True
+            for line in fp:
+                if first:
+                    idx = line.index("|")
+                    repo = line[:idx].strip()
+                    dest = line[idx + 1:].strip()
+                    first = False
+                else:
+                    directories.append(line.strip())
     finally:
-        fp.close()
         return repo, dest, directories
 
 def makePath(a, b):
     if not b:
         # Empty directory, i.e., the repository root.
         return a
     return "%s/%s" % (a, b)
 
@@ -81,78 +80,85 @@ def copy(thissrcdir, dest, directories):
 
         if len(subdirs):
             if d:
                 importDirs(thissrcdir, dest, ["%s/%s" % (d, subdir) for subdir in subdirs])
             else:
                 # Empty directory, i.e., the repository root
                 importDirs(thissrcdir, dest, subdirs)
 
-def printMakefile(dest, directories):
-    """Create a .mk file to be included into the main Makefile.in, which lists the
-    directories with tests.
+def printMozbuildFile(dest, directories):
+    """Create a .mozbuild file to be included into the main moz.build, which
+    lists the directories with tests.
     """
-    print("Creating .mk...")
-    path = dest + ".mk"
-    fp = open(path, "w")
-    fp.write("DIRS += \\\n")
-    fp.write(writeMakefile.makefileString([makePath(dest, d) for d in directories]))
-    fp.write("\n")
-    fp.close()
+    print("Creating mozbuild...")
+    path = dest + ".mozbuild"
+    with open(path, 'w') as fh:
+        normalized = [makePath(dest, d) for d in directories]
+        result = writeBuildFiles.substMozbuild("importTestSuites.py",
+            normalized)
+        fh.write(result)
+
     subprocess.check_call(["hg", "add", path])
 
-def printMakefiles(thissrcdir, dest, directories):
+def printBuildFiles(thissrcdir, dest, directories):
     """Create Makefile.in files for each directory that contains tests we import.
     """
-    print("Creating Makefile.ins...")
+    print("Creating build files...")
     for d in directories:
         path = makePath(dest, d)
         print("Creating Makefile.in in %s..." % path)
 
         subdirs, mochitests, supportfiles = parseManifestFile(dest, d)
 
         files = ["test_%s" % (mochitest, ) for mochitest in mochitests]
         files.extend(supportfiles)
         files.extend(f for f in os.listdir(path) if f.endswith(HEADERS_SUFFIX))
 
-        result = writeMakefile.substMakefile("importTestsuite.py", subdirs, files)
+        with open(path + "/Makefile.in", "w") as fh:
+            result = writeBuildFiles.substMakefile("importTestsuite.py", files)
+            fh.write(result)
 
-        fp = open(path + "/Makefile.in", "w")
-        fp.write(result)
-        fp.close()
+        print("Creating moz.build in %s..." % path)
+        with open(path + "/moz.build", "w") as fh:
+            result = writeBuildFiles.substMozbuild("importTestsuite.py",
+                subdirs)
+            fh.write(result)
+
 
 def hgadd(dest, directories):
     """Inform hg of the files in |directories|."""
     print("hg addremoving...")
     for d in directories:
         subprocess.check_call(["hg", "addremove", "%s/%s" % (dest, d)])
 
 def importDirs(thissrcdir, dest, directories):
     copy(thissrcdir, dest, directories)
-    printMakefiles(thissrcdir, dest, directories)
+    printBuildFiles(thissrcdir, dest, directories)
 
 def importRepo(confFile, thissrcdir):
     try:
         repo, dest, directories = getData(confFile)
         hgdest = "hg-%s" % (dest, )
         print("Going to clone %s to %s..." % (repo, hgdest))
         print("Removing %s..." % dest)
-        subprocess.check_call(["rm", "--recursive", "--force", dest])
+        subprocess.check_call(["rm", "-rf", dest])
         print("Removing %s..." % hgdest)
-        subprocess.check_call(["rm", "--recursive", "--force", hgdest])
+        subprocess.check_call(["rm", "-rf", hgdest])
         print("Cloning %s to %s..." % (repo, hgdest))
         subprocess.check_call(["hg", "clone", repo, hgdest])
         print("Going to import %s..." % directories)
         importDirs(thissrcdir, dest, directories)
-        printMakefile(dest, directories)
+        printMozbuildFile(dest, directories)
         hgadd(dest, directories)
         print("Removing %s again..." % hgdest)
-        subprocess.check_call(["rm", "--recursive", "--force", hgdest])
+        subprocess.check_call(["rm", "-rf", hgdest])
     except subprocess.CalledProcessError as e:
         print(e.returncode)
     finally:
         print("Done")
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
         print("Need one argument.")
     else:
         importRepo(sys.argv[1], "dom/imptests")
+
--- a/dom/imptests/parseFailures.py
+++ b/dom/imptests/parseFailures.py
@@ -4,17 +4,17 @@
 
 from __future__ import print_function, unicode_literals
 
 import collections
 import json
 import os
 import sys
 
-import writeMakefile
+import writeBuildFiles
 
 def extractLines(fp):
     lines = []
     watch = False
     for line in fp:
         line = line.decode('utf-8')
         if line == '@@@ @@@ Failures\n':
             watch = True
@@ -47,34 +47,38 @@ def dumpFailures(lines):
         obj = json.loads(objstr, object_pairs_hook=collections.OrderedDict)
         formattedobjstr = json.dumps(obj, indent=2, separators=(',', sep)) + '\n'
         formattedobj = formattedobjstr.encode('utf-8')
         fp = open(jsonpath, 'wb')
         fp.write(formattedobj)
         fp.close()
     return files
 
-def writeMakefiles(files):
+def writeFiles(files):
     pathmap = {}
     for path in files:
         dirp, leaf = path.rsplit('/', 1)
         pathmap.setdefault(dirp, []).append(leaf)
 
     for k, v in pathmap.items():
-        resultstr = writeMakefile.substMakefile('parseFailures.py', [], v)
-        result = resultstr.encode('utf-8')
+        with open(k + '/Makefile.in', 'wb') as fh:
+            result = writeBuildFiles.substMakefile('parseFailures.py', v)
+            result = result.encode('utf-8')
+            fh.write(result)
 
-        fp = open(k + '/Makefile.in', 'wb')
-        fp.write(result)
-        fp.close()
+        with open(k + '/moz.build', 'wb') as fh:
+            result = writeBuildFiles.substMozbuild('parseFailures.py', [])
+            result = result.encode('utf-8')
+            fh.write(result)
 
 def main(logPath):
     fp = open(logPath, 'rb')
     lines = extractLines(fp)
     fp.close()
 
     files = dumpFailures(lines)
-    writeMakefiles(files)
+    writeFiles(files)
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
         print("Please pass the path to the logfile from which failures should be extracted.")
     main(sys.argv[1])
+
--- a/dom/imptests/updateTestharness.py
+++ b/dom/imptests/updateTestharness.py
@@ -10,9 +10,10 @@ import subprocess
 repo = "https://dvcs.w3.org/hg/resources"
 dest = "resources-upstream"
 files = ["testharness.js", "testharness.css", "idlharness.js", "WebIDLParser.js"]
 
 subprocess.check_call(["hg", "clone", repo, dest])
 for f in files:
     subprocess.check_call(["cp", "%s/%s" % (dest, f), f])
     subprocess.check_call(["hg", "add", f])
-subprocess.check_call(["rm", "--recursive", "--force", dest])
+subprocess.check_call(["rm", "-rf", dest])
+
rename from dom/imptests/writeMakefile.py
rename to dom/imptests/writeBuildFiles.py
--- a/dom/imptests/writeMakefile.py
+++ b/dom/imptests/writeBuildFiles.py
@@ -1,40 +1,63 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import string
 
-makefileTemplate = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
+makefile_template = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
 
 DEPTH := @DEPTH@
 
 topsrcdir := @top_srcdir@
 srcdir := @srcdir@
 VPATH := @srcdir@
 relativesrcdir := @relativesrcdir@
 
 include $$(DEPTH)/config/autoconf.mk
 
-${dirs}
-
 ${files}
 
 include $$(topsrcdir)/config/rules.mk
 """
 
+mozbuild_template = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
+
+${dirs}
+"""
+
+
+
 def makefileString(entries):
     if not len(entries):
         return "  $(NULL)"
     return "\n".join(["  %s \\" % (entry, ) for entry in entries]) + "\n  $(NULL)"
 
+
 def assignList(variable, entries):
     return "%s := \\\n%s" % (variable, makefileString(entries))
 
-def substMakefile(caller, subdirs, files):
-    return string.Template(makefileTemplate).substitute({
-      "caller": caller,
-      "dirs": assignList("DIRS", subdirs),
-      "files": assignList("MOCHITEST_FILES", files) if files else ""
+
+def mozbuildDirs(dirs):
+    """Obtain a DIRS assignment string for mozbuild files."""
+    parts = ['DIRS += [']
+    for d in dirs:
+        parts.append("    '%s'," % d)
+    parts.append(']')
+
+    return '\n'.join(parts)
+
+def substMakefile(caller, files):
+    return string.Template(makefile_template).substitute({
+        "caller": caller,
+        "files": assignList("MOCHITEST_FILES", files) if files else ""
     })
+
+
+def substMozbuild(caller, dirs):
+    return string.Template(mozbuild_template).substitute({
+        "caller": caller,
+        "dirs": mozbuildDirs(dirs),
+    })
+