Bug 784841 - Part 15: Produce moz.build files for dom imported tests; r=Ms2ger
--- a/dom/imptests/README
+++ b/dom/imptests/README
@@ -52,18 +52,18 @@ Source; Usage and purpose; License
* parseFailures.py
Parses failures out of a mochitest log and writes out JSON files and Makefiles
into the correct failures/ folder.
The mochitest log should be produced by setting the 'dumpFailures' flag in
testharnessreport.js; this will print out the encountered failures, marked
by @ signs.
MPL
-* writeMakefile.py
- Helper functions to write out Makefiles.
+* writeBuildFiles.py
+ Helper functions to write out automatically generated build files.
MPL
* Makefile.in
Integration with our build system. Installs support files into /resources and
includes a .mk file for each repository.
MPL
* failures/
--- a/dom/imptests/importTestsuite.py
+++ b/dom/imptests/importTestsuite.py
@@ -13,17 +13,17 @@ Note: removes both source and destinatio
from __future__ import print_function, unicode_literals
import os
import shutil
import subprocess
import sys
import parseManifest
-import writeMakefile
+import writeBuildFiles
HEADERS_SUFFIX = "^headers^"
def parseManifestFile(dest, dir):
subdirs, mochitests, _, __, supportfiles = parseManifest.parseManifestFile("hg-%s/%s/MANIFEST" % (dest, dir))
return subdirs, mochitests, supportfiles
def getData(confFile):
@@ -31,28 +31,27 @@ def getData(confFile):
URL of remote repository|Name of the destination directory
First directory of tests
...
Last directory of tests"""
repo = ""
dest = ""
directories = []
try:
- fp = open(confFile, "r")
- first = True
- for line in fp:
- if first:
- idx = line.index("|")
- repo = line[:idx].strip()
- dest = line[idx + 1:].strip()
- first = False
- else:
- directories.append(line.strip())
+ with open(confFile, 'r') as fp:
+ first = True
+ for line in fp:
+ if first:
+ idx = line.index("|")
+ repo = line[:idx].strip()
+ dest = line[idx + 1:].strip()
+ first = False
+ else:
+ directories.append(line.strip())
finally:
- fp.close()
return repo, dest, directories
def makePath(a, b):
if not b:
# Empty directory, i.e., the repository root.
return a
return "%s/%s" % (a, b)
@@ -81,78 +80,85 @@ def copy(thissrcdir, dest, directories):
if len(subdirs):
if d:
importDirs(thissrcdir, dest, ["%s/%s" % (d, subdir) for subdir in subdirs])
else:
# Empty directory, i.e., the repository root
importDirs(thissrcdir, dest, subdirs)
-def printMakefile(dest, directories):
- """Create a .mk file to be included into the main Makefile.in, which lists the
- directories with tests.
+def printMozbuildFile(dest, directories):
+ """Create a .mozbuild file to be included into the main moz.build, which
+ lists the directories with tests.
"""
- print("Creating .mk...")
- path = dest + ".mk"
- fp = open(path, "w")
- fp.write("DIRS += \\\n")
- fp.write(writeMakefile.makefileString([makePath(dest, d) for d in directories]))
- fp.write("\n")
- fp.close()
+ print("Creating mozbuild...")
+ path = dest + ".mozbuild"
+ with open(path, 'w') as fh:
+ normalized = [makePath(dest, d) for d in directories]
+ result = writeBuildFiles.substMozbuild("importTestSuites.py",
+ normalized)
+ fh.write(result)
+
subprocess.check_call(["hg", "add", path])
-def printMakefiles(thissrcdir, dest, directories):
+def printBuildFiles(thissrcdir, dest, directories):
"""Create Makefile.in files for each directory that contains tests we import.
"""
- print("Creating Makefile.ins...")
+ print("Creating build files...")
for d in directories:
path = makePath(dest, d)
print("Creating Makefile.in in %s..." % path)
subdirs, mochitests, supportfiles = parseManifestFile(dest, d)
files = ["test_%s" % (mochitest, ) for mochitest in mochitests]
files.extend(supportfiles)
files.extend(f for f in os.listdir(path) if f.endswith(HEADERS_SUFFIX))
- result = writeMakefile.substMakefile("importTestsuite.py", subdirs, files)
+ with open(path + "/Makefile.in", "w") as fh:
+ result = writeBuildFiles.substMakefile("importTestsuite.py", files)
+ fh.write(result)
- fp = open(path + "/Makefile.in", "w")
- fp.write(result)
- fp.close()
+ print("Creating moz.build in %s..." % path)
+ with open(path + "/moz.build", "w") as fh:
+ result = writeBuildFiles.substMozbuild("importTestsuite.py",
+ subdirs)
+ fh.write(result)
+
def hgadd(dest, directories):
"""Inform hg of the files in |directories|."""
print("hg addremoving...")
for d in directories:
subprocess.check_call(["hg", "addremove", "%s/%s" % (dest, d)])
def importDirs(thissrcdir, dest, directories):
copy(thissrcdir, dest, directories)
- printMakefiles(thissrcdir, dest, directories)
+ printBuildFiles(thissrcdir, dest, directories)
def importRepo(confFile, thissrcdir):
try:
repo, dest, directories = getData(confFile)
hgdest = "hg-%s" % (dest, )
print("Going to clone %s to %s..." % (repo, hgdest))
print("Removing %s..." % dest)
- subprocess.check_call(["rm", "--recursive", "--force", dest])
+ subprocess.check_call(["rm", "-rf", dest])
print("Removing %s..." % hgdest)
- subprocess.check_call(["rm", "--recursive", "--force", hgdest])
+ subprocess.check_call(["rm", "-rf", hgdest])
print("Cloning %s to %s..." % (repo, hgdest))
subprocess.check_call(["hg", "clone", repo, hgdest])
print("Going to import %s..." % directories)
importDirs(thissrcdir, dest, directories)
- printMakefile(dest, directories)
+ printMozbuildFile(dest, directories)
hgadd(dest, directories)
print("Removing %s again..." % hgdest)
- subprocess.check_call(["rm", "--recursive", "--force", hgdest])
+ subprocess.check_call(["rm", "-rf", hgdest])
except subprocess.CalledProcessError as e:
print(e.returncode)
finally:
print("Done")
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Need one argument.")
else:
importRepo(sys.argv[1], "dom/imptests")
+
--- a/dom/imptests/parseFailures.py
+++ b/dom/imptests/parseFailures.py
@@ -4,17 +4,17 @@
from __future__ import print_function, unicode_literals
import collections
import json
import os
import sys
-import writeMakefile
+import writeBuildFiles
def extractLines(fp):
lines = []
watch = False
for line in fp:
line = line.decode('utf-8')
if line == '@@@ @@@ Failures\n':
watch = True
@@ -47,34 +47,38 @@ def dumpFailures(lines):
obj = json.loads(objstr, object_pairs_hook=collections.OrderedDict)
formattedobjstr = json.dumps(obj, indent=2, separators=(',', sep)) + '\n'
formattedobj = formattedobjstr.encode('utf-8')
fp = open(jsonpath, 'wb')
fp.write(formattedobj)
fp.close()
return files
-def writeMakefiles(files):
+def writeFiles(files):
pathmap = {}
for path in files:
dirp, leaf = path.rsplit('/', 1)
pathmap.setdefault(dirp, []).append(leaf)
for k, v in pathmap.items():
- resultstr = writeMakefile.substMakefile('parseFailures.py', [], v)
- result = resultstr.encode('utf-8')
+ with open(k + '/Makefile.in', 'wb') as fh:
+ result = writeBuildFiles.substMakefile('parseFailures.py', v)
+ result = result.encode('utf-8')
+ fh.write(result)
- fp = open(k + '/Makefile.in', 'wb')
- fp.write(result)
- fp.close()
+ with open(k + '/moz.build', 'wb') as fh:
+ result = writeBuildFiles.substMozbuild('parseFailures.py', [])
+ result = result.encode('utf-8')
+ fh.write(result)
def main(logPath):
fp = open(logPath, 'rb')
lines = extractLines(fp)
fp.close()
files = dumpFailures(lines)
- writeMakefiles(files)
+ writeFiles(files)
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Please pass the path to the logfile from which failures should be extracted.")
main(sys.argv[1])
+
--- a/dom/imptests/updateTestharness.py
+++ b/dom/imptests/updateTestharness.py
@@ -10,9 +10,10 @@ import subprocess
repo = "https://dvcs.w3.org/hg/resources"
dest = "resources-upstream"
files = ["testharness.js", "testharness.css", "idlharness.js", "WebIDLParser.js"]
subprocess.check_call(["hg", "clone", repo, dest])
for f in files:
subprocess.check_call(["cp", "%s/%s" % (dest, f), f])
subprocess.check_call(["hg", "add", f])
-subprocess.check_call(["rm", "--recursive", "--force", dest])
+subprocess.check_call(["rm", "-rf", dest])
+
rename from dom/imptests/writeMakefile.py
rename to dom/imptests/writeBuildFiles.py
--- a/dom/imptests/writeMakefile.py
+++ b/dom/imptests/writeBuildFiles.py
@@ -1,40 +1,63 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import string
-makefileTemplate = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
+makefile_template = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
DEPTH := @DEPTH@
topsrcdir := @top_srcdir@
srcdir := @srcdir@
VPATH := @srcdir@
relativesrcdir := @relativesrcdir@
include $$(DEPTH)/config/autoconf.mk
-${dirs}
-
${files}
include $$(topsrcdir)/config/rules.mk
"""
+mozbuild_template = """# THIS FILE IS AUTOGENERATED BY ${caller} - DO NOT EDIT
+
+${dirs}
+"""
+
+
+
def makefileString(entries):
if not len(entries):
return " $(NULL)"
return "\n".join([" %s \\" % (entry, ) for entry in entries]) + "\n $(NULL)"
+
def assignList(variable, entries):
return "%s := \\\n%s" % (variable, makefileString(entries))
-def substMakefile(caller, subdirs, files):
- return string.Template(makefileTemplate).substitute({
- "caller": caller,
- "dirs": assignList("DIRS", subdirs),
- "files": assignList("MOCHITEST_FILES", files) if files else ""
+
+def mozbuildDirs(dirs):
+ """Obtain a DIRS assignment string for mozbuild files."""
+ parts = ['DIRS += [']
+ for d in dirs:
+ parts.append(" '%s'," % d)
+ parts.append(']')
+
+ return '\n'.join(parts)
+
+def substMakefile(caller, files):
+ return string.Template(makefile_template).substitute({
+ "caller": caller,
+ "files": assignList("MOCHITEST_FILES", files) if files else ""
})
+
+
+def substMozbuild(caller, dirs):
+ return string.Template(mozbuild_template).substitute({
+ "caller": caller,
+ "dirs": mozbuildDirs(dirs),
+ })
+