bug 746244 - Port profileserver.py to Mozbase. r=jhammel
authorTed Mielczarek <ted@mielczarek.org>
Tue, 28 May 2013 15:33:57 -0400
changeset 134096 8b379faea0965b1405e67a59f8cb19ec20602467
parent 134095 008c33fca4a7f21e1e7790758d38fe814415d83d
child 134097 87f0ed19ef8de86b9d762a775dbc89dfacdee493
push id29041
push usertmielczarek@mozilla.com
push dateWed, 05 Jun 2013 13:18:50 +0000
treeherdermozilla-inbound@8b379faea096 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhammel
bugs746244
milestone24.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 746244 - Port profileserver.py to Mozbase. r=jhammel
build/moz.build
build/pgo/Makefile.in
build/pgo/blueprint/Makefile.in
build/pgo/blueprint/moz.build
build/pgo/certs/README
build/pgo/genpgocert.py
build/pgo/genpgocert.py.in
build/pgo/js-input/Makefile.in
build/pgo/js-input/moz.build
build/pgo/moz.build
build/pgo/profileserver.py
testing/testsuite-targets.mk
--- a/build/moz.build
+++ b/build/moz.build
@@ -7,18 +7,16 @@
 if CONFIG['OS_ARCH'] not in ('WINNT', 'OS2'):
     DIRS += ['unix']
 elif CONFIG['OS_ARCH'] == 'WINNT':
     DIRS += ['win32']
 
 if CONFIG['STLPORT_SOURCES']:
     DIRS += ['stlport']
 
-DIRS += ['pgo']
-
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
     TEST_DIRS += [
         'mobile/sutagent/android',
         'mobile/sutagent/android/watcher',
         'mobile/sutagent/android/ffxcp',
         'mobile/sutagent/android/fencp',
         'mobile/robocop',
     ]
deleted file mode 100644
--- a/build/pgo/Makefile.in
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-relativesrcdir = @relativesrcdir@
-
-include $(DEPTH)/config/autoconf.mk
-
-include $(topsrcdir)/config/rules.mk
-
-# We install to _profile/pgo
-TARGET_DEPTH = ../..
-include $(topsrcdir)/build/automation-build.mk
-
-# Need to override the browser_path from binary-location.mk (included via automation-build.mk)
-# since we want to run from e.g. dist/firefox rather than dist/bin
-ifeq ($(OS_ARCH),Darwin)
-browser_path = \"$(TARGET_DIST)/$(MOZ_APP_NAME)/$(MOZ_MACBUNDLE_NAME)/Contents/MacOS/$(PROGRAM)\"
-else
-browser_path = \"$(TARGET_DIST)/$(MOZ_APP_NAME)/$(PROGRAM)\"
-endif
-   
-# Stuff to make a build with a profile
-
-_PGO_FILES = 	\
-  automation.py \
-  $(topsrcdir)/build/automationutils.py \
-  $(topsrcdir)/testing/profiles/prefs_general.js \
-  profileserver.py \
-  genpgocert.py \
-  index.html \
-  server-locations.txt \
-  favicon.ico \
-  $(NULL)
-
-genpgocert.py: genpgocert.py.in
-	$(PYTHON) $(topsrcdir)/config/Preprocessor.py \
-	$(AUTOMATION_PPARGS) $(DEFINES) $(ACDEFINES) $^ > $@
-
-GARBAGE += genpgocert.py
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
-
-genservercert::
-	$(PYTHON) $(DEPTH)/_profile/pgo/genpgocert.py --gen-server
deleted file mode 100644
--- a/build/pgo/blueprint/Makefile.in
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-include $(topsrcdir)/config/rules.mk
-
-_PROFILE_DIR = $(DEPTH)/_profile/pgo/blueprint
-
-_PGO_FILES = 	\
-  sample.html \
-  elements.html \
-  forms.html \
-  grid.html \
-  test.jpg \
-  test-small.jpg \
-  valid.png \
-  screen.css \
-  print.css \
-  grid.png \
-  fancytype-screen.css \
-  $(NULL)
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
\ No newline at end of file
deleted file mode 100644
--- a/build/pgo/blueprint/moz.build
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
new file mode 100644
--- /dev/null
+++ b/build/pgo/certs/README
@@ -0,0 +1,9 @@
+The certificate authority and server certificates here are generated by $topsrcdir/build/pgo/genpgocert.py.
+
+You can generate a new CA cert by running:
+$objdir/_virtualenv/bin/python $topsrcdir/build/pgo/genpgocert.py --gen-ca
+
+You can generate new server certificates by running:
+$objdir/_virtualenv/bin/python $topsrcdir/build/pgo/genpgocert.py --gen-server
+
+These will place the new files in this directory where you can commit them.
rename from build/pgo/genpgocert.py.in
rename to build/pgo/genpgocert.py
--- a/build/pgo/genpgocert.py.in
+++ b/build/pgo/genpgocert.py
@@ -1,27 +1,29 @@
-#
+#!/usr/bin/env python
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from automation import Automation
-from mozfile import NamedTemporaryFile
+# This script exists to generate the Certificate Authority and server
+# certificates used for SSL testing in Mochitest. The already generated
+# certs are located at $topsrcdir/build/pgo/certs/ .
+
+import mozinfo
 import os
 import random
 import re
 import shutil
+import subprocess
 import sys
+import tempfile
 
-#expand DIST_BIN = __XPC_BIN_PATH__
-#expand BIN_SUFFIX = __BIN_SUFFIX__
-#expand PROFILE_DIR = __PROFILE_DIR__
-#expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
-
-automation = Automation()
+from mozbuild.base import MozbuildObject
+from mozfile import NamedTemporaryFile
+from mozprofile.permissions import ServerLocations
 
 dbFiles = [
   re.compile("^cert[0-9]+\.db$"),
   re.compile("^key[0-9]+\.db$"),
   re.compile("^secmod\.db$")
 ]
 
 def unlinkDbFiles(path):
@@ -36,95 +38,103 @@ def dbFilesExist(path):
     for name in files:
       for dbFile in dbFiles:
         if dbFile.match(name) and os.path.exists(os.path.join(root, name)):
           return True
   return False
 
 
 def runUtil(util, args, inputdata = None):
-  if inputdata:
-    proc = automation.Process([util] + args, env = automation.environment(), stdin = automation.PIPE)
-    proc.communicate(inputdata)
-    return proc.returncode
-  return automation.Process([util] + args, env = automation.environment()).wait()
+  env = os.environ.copy()
+  if mozinfo.os == "linux":
+    pathvar = "LD_LIBRARY_PATH"
+    app_path = os.path.dirname(util)
+    if pathvar in env:
+      env[pathvar] = "%s%s%s" % (app_path, os.pathsep, env[pathvar])
+    else:
+      env[pathvar] = app_path
+  proc = subprocess.Popen([util] + args, env=env,
+                          stdin=subprocess.PIPE if inputdata else None)
+  proc.communicate(inputdata)
+  return proc.returncode
 
 
 def createRandomFile(randomFile):
   for count in xrange(0, 2048):
     randomFile.write(chr(random.randint(0, 255)))
 
 
-def createCertificateAuthority(profileDir, srcDir):
-  certutil = DIST_BIN + "/certutil" + BIN_SUFFIX
-  pk12util = DIST_BIN + "/pk12util" + BIN_SUFFIX
+def createCertificateAuthority(build, srcDir):
+  certutil = build.get_binary_path(what="certutil")
+  pk12util = build.get_binary_path(what="pk12util")
 
-  tempDbDir = os.path.join(profileDir, ".temp")
-  if not os.path.exists(tempDbDir):
-    os.mkdir(tempDbDir)
-
+  #TODO: mozfile.TemporaryDirectory
+  tempDbDir = tempfile.mkdtemp()
   with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
     pgoCAModulePathSrc = os.path.join(srcDir, "pgoca.p12")
     pgoCAPathSrc = os.path.join(srcDir, "pgoca.ca")
 
     pwfile.write("\n")
 
-    unlinkDbFiles(tempDbDir)
-
     # Create temporary certification database for CA generation
     status = runUtil(certutil, ["-N", "-d", tempDbDir, "-f", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
     createRandomFile(rndfile)
     status = runUtil(certutil, ["-S", "-d", tempDbDir, "-s", "CN=Temporary Certificate Authority, O=Mozilla Testing, OU=Profile Guided Optimization", "-t", "C,,", "-x", "-m", "1", "-v", "120", "-n", "pgo temporary ca", "-2", "-f", pwfile.name, "-z", rndfile.name], "Y\n0\nN\n")
-    if status != 0:
+    if status:
       return status
 
     status = runUtil(certutil, ["-L", "-d", tempDbDir, "-n", "pgo temporary ca", "-a", "-o", pgoCAPathSrc, "-f", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
     status = runUtil(pk12util, ["-o", pgoCAModulePathSrc, "-n", "pgo temporary ca", "-d", tempDbDir, "-w", pwfile.name, "-k", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
-    unlinkDbFiles(tempDbDir)
-    os.rmdir(tempDbDir)
+  shutil.rmtree(tempDbDir)
   return 0
 
 
-def createSSLServerCertificate(profileDir, srcDir):
-  certutil = DIST_BIN + "/certutil" + BIN_SUFFIX
-  pk12util = DIST_BIN + "/pk12util" + BIN_SUFFIX
+def createSSLServerCertificate(build, srcDir):
+  certutil = build.get_binary_path(what="certutil")
+  pk12util = build.get_binary_path(what="pk12util")
 
   with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
     pgoCAPath = os.path.join(srcDir, "pgoca.p12")
 
     pwfile.write("\n")
 
     if not dbFilesExist(srcDir):
       # Make sure all DB files from src are really deleted
       unlinkDbFiles(srcDir)
 
       # Create certification database for ssltunnel
       status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
       status = runUtil(pk12util, ["-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
     # Generate automatic certificate
-    locations = automation.readLocations(os.path.join(profileDir, "server-locations.txt"))
-    locations.pop(0)
+    locations = ServerLocations(os.path.join(build.topsrcdir,
+                                             "build", "pgo",
+                                             "server-locations.txt"))
+    iterator = iter(locations)
+
+    # Skips the first entry, I don't know why: bug 879740
+    iterator.next()
+
     locationsParam = ""
     firstLocation = ""
-    for loc in locations:
+    for loc in iterator:
       if loc.scheme == "https" and "nocert" not in loc.options:
         customCertOption = False
         customCertRE = re.compile("^cert=(?:\w+)")
         for option in loc.options:
           match = customCertRE.match(option)
           if match:
             customCertOption = True
             break
@@ -132,45 +142,46 @@ def createSSLServerCertificate(profileDi
         if not customCertOption:
           if len(locationsParam) > 0:
             locationsParam += ","
           locationsParam += loc.host
 
           if firstLocation == "":
             firstLocation = loc.host
 
-    if firstLocation == "":
+    if not firstLocation:
       print "Nothing to generate, no automatic secure hosts specified"
     else:
       createRandomFile(rndfile)
 
       runUtil(certutil, ["-D", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name])
       # Ignore the result, the certificate may not be present when new database is being built
 
       status = runUtil(certutil, ["-S", "-s", "CN=%s" % firstLocation, "-t", "Pu,,", "-c", "pgo temporary ca", "-m", "2", "-8", locationsParam, "-v", "120", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
   return 0
 
-
 if len(sys.argv) == 1:
   print "Specify --gen-server or --gen-ca"
   sys.exit(1)
 
+build = MozbuildObject.from_environment()
+certdir = os.path.join(build.topsrcdir, "build", "pgo", "certs")
 if sys.argv[1] == "--gen-server":
-  certificateStatus = createSSLServerCertificate(PROFILE_DIR, CERTS_SRC_DIR)
-  if certificateStatus != 0:
+  certificateStatus = createSSLServerCertificate(build, certdir)
+  if certificateStatus:
     print "TEST-UNEXPECTED-FAIL | SSL Server Certificate generation"
 
   sys.exit(certificateStatus)
 
 if sys.argv[1] == "--gen-ca":
-  certificateStatus = createCertificateAuthority(PROFILE_DIR, CERTS_SRC_DIR)
-  if certificateStatus != 0:
+  certificateStatus = createCertificateAuthority(build, certdir)
+  if certificateStatus:
     print "TEST-UNEXPECTED-FAIL | Certificate Authority generation"
   else:
     print "\n\n"
     print "==================================================="
     print " IMPORTANT:"
     print " To use this new certificate authority in tests"
     print " run 'make' at testing/mochitest"
     print "==================================================="
deleted file mode 100644
--- a/build/pgo/js-input/Makefile.in
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-include $(topsrcdir)/config/rules.mk
-
-_PROFILE_DIR = $(DEPTH)/_profile/pgo/js-input
-
-_PGO_FILES = 	\
-  3d-cube.html \
-  3d-morph.html \
-  3d-raytrace.html \
-  3d-thingy.html \
-  access-binary-trees.html \
-  access-fannkuch.html \
-  access-nbody.html \
-  access-nsieve.html \
-  bitops-3bit-bits-in-byte.html \
-  bitops-bits-in-byte.html \
-  bitops-bitwise-and.html \
-  bitops-nsieve-bits.html \
-  controlflow-recursive.html \
-  crypto-aes.html \
-  crypto-md5.html \
-  crypto-sha1.html \
-  crypto-otp.html \
-  date-format-tofte.html \
-  date-format-xparb.html \
-  math-cordic.html \
-  math-partial-sums.html \
-  math-spectral-norm.html \
-  regexp-dna.html \
-  string-base64.html \
-  string-fasta.html \
-  string-tagcloud.html \
-  string-unpack-code.html \
-  string-validate-input.html \
-  key.gif \
-  valid-xhtml10.png \
-  $(NULL)
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
deleted file mode 100644
--- a/build/pgo/js-input/moz.build
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
deleted file mode 100644
--- a/build/pgo/moz.build
+++ /dev/null
@@ -1,7 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DIRS += ['blueprint', 'js-input']
--- a/build/pgo/profileserver.py
+++ b/build/pgo/profileserver.py
@@ -1,59 +1,71 @@
-#literal #!/usr/bin/python
+#!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import SimpleHTTPServer
-import SocketServer
+from mozprofile import FirefoxProfile, Profile, Preferences
+from mozprofile.permissions import ServerLocations
+from mozrunner import FirefoxRunner, CLI
+from mozhttpd import MozHttpd
+import json
 import socket
 import threading
 import os
 import sys
 import shutil
+import tempfile
 from datetime import datetime
-
-SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
-sys.path.insert(0, SCRIPT_DIR)
-from automation import Automation
-from automationutils import getDebuggerInfo, addCommonOptions
+from mozbuild.base import MozbuildObject
 
 PORT = 8888
-PROFILE_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "./pgoprofile"))
-MOZ_JAR_LOG_FILE = os.path.abspath(os.getenv("JARLOG_FILE"))
-os.chdir(SCRIPT_DIR)
-
-class EasyServer(SocketServer.TCPServer):
-  allow_reuse_address = True
 
 if __name__ == '__main__':
-  from optparse import OptionParser
-  automation = Automation()
+  cli = CLI()
+  debug_args, interactive = cli.debugger_arguments()
 
-  parser = OptionParser()
-  addCommonOptions(parser)
+  build = MozbuildObject.from_environment()
+  httpd = MozHttpd(port=PORT,
+                   docroot=os.path.join(build.topsrcdir, "build", "pgo"))
+  httpd.start(block=False)
 
-  options, args = parser.parse_args()
-
-  debuggerInfo = getDebuggerInfo(".", options.debugger, options.debuggerArgs,
-          options.debuggerInteractive)
+  locations = ServerLocations()
+  locations.add_host(host='127.0.0.1',
+                     port=PORT,
+                     options='primary,privileged')
 
-  httpd = EasyServer(("", PORT), SimpleHTTPServer.SimpleHTTPRequestHandler)
-  t = threading.Thread(target=httpd.serve_forever)
-  t.setDaemon(True) # don't hang on exit
-  t.start()
-  
-  automation.setServerInfo("localhost", PORT)
-  automation.initializeProfile(PROFILE_DIRECTORY)
-  browserEnv = automation.environment()
-  browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
-  browserEnv["MOZ_JAR_LOG_FILE"] = MOZ_JAR_LOG_FILE
+  #TODO: mozfile.TemporaryDirectory
+  profilePath = tempfile.mkdtemp()
+  try:
+    #TODO: refactor this into mozprofile
+    prefpath = os.path.join(build.topsrcdir, "testing", "profiles", "prefs_general.js")
+    prefs = {}
+    prefs.update(Preferences.read_prefs(prefpath))
+    interpolation = { "server": "%s:%d" % httpd.httpd.server_address,
+                      "OOP": "false"}
+    prefs = json.loads(json.dumps(prefs) % interpolation)
+    for pref in prefs:
+      prefs[pref] = Preferences.cast(prefs[pref])
+    profile = FirefoxProfile(profile=profilePath,
+                             preferences=prefs,
+                             #addons=[os.path.join(here, 'extension')],
+                             locations=locations)
 
-  url = "http://localhost:%d/index.html" % PORT
-  appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP)
-  status = automation.runApp(url, browserEnv, appPath, PROFILE_DIRECTORY, {},
-                             debuggerInfo=debuggerInfo,
-                             # the profiling HTML doesn't output anything,
-                             # so let's just run this without a timeout
-                             timeout = None)
-  sys.exit(status)
+    env = os.environ.copy()
+    env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+    env["XPCOM_DEBUG_BREAK"] = "warn"
+    jarlog = os.getenv("JARLOG_FILE")
+    if jarlog:
+      env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
+      print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
+
+    cmdargs = ["http://localhost:%d/index.html" % PORT]
+    runner = FirefoxRunner(profile=profile,
+                           binary=build.get_binary_path(where="staged-package"),
+                           cmdargs=cmdargs,
+                           env=env)
+    runner.start(debug_args=debug_args, interactive=interactive)
+    runner.wait()
+    httpd.stop()
+  finally:
+    shutil.rmtree(profilePath)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -375,17 +375,17 @@ jetpack-tests:
 	$(PYTHON) $(topsrcdir)/addon-sdk/source/bin/cfx -b $(browser_path) --parseable testpkgs
 
 # -- -register
 # -- --trace-malloc malloc.log --shutdown-leaks=sdleak.log
 leaktest:
 	$(PYTHON) _leaktest/leaktest.py $(LEAKTEST_ARGS)
 
 pgo-profile-run:
-	$(PYTHON) $(DEPTH)/_profile/pgo/profileserver.py $(EXTRA_TEST_ARGS)
+	$(PYTHON) $(topsrcdir)/build/pgo/profileserver.py $(EXTRA_TEST_ARGS)
 
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 ifndef UNIVERSAL_BINARY
 PKG_STAGE = $(DIST)/test-package-stage
 package-tests: \
   stage-config \