bug 746244 - Port profileserver.py to Mozbase. r=jhammel
authorTed Mielczarek <ted@mielczarek.org>
Tue, 28 May 2013 15:33:57 -0400
changeset 134111 8b379faea0965b1405e67a59f8cb19ec20602467
parent 134110 008c33fca4a7f21e1e7790758d38fe814415d83d
child 134112 87f0ed19ef8de86b9d762a775dbc89dfacdee493
push idunknown
push userunknown
push dateunknown
reviewersjhammel
bugs746244
milestone24.0a1
bug 746244 - Port profileserver.py to Mozbase. r=jhammel
build/moz.build
build/pgo/Makefile.in
build/pgo/blueprint/Makefile.in
build/pgo/blueprint/moz.build
build/pgo/certs/README
build/pgo/genpgocert.py
build/pgo/genpgocert.py.in
build/pgo/js-input/Makefile.in
build/pgo/js-input/moz.build
build/pgo/moz.build
build/pgo/profileserver.py
testing/testsuite-targets.mk
--- a/build/moz.build
+++ b/build/moz.build
@@ -7,18 +7,16 @@
 if CONFIG['OS_ARCH'] not in ('WINNT', 'OS2'):
     DIRS += ['unix']
 elif CONFIG['OS_ARCH'] == 'WINNT':
     DIRS += ['win32']
 
 if CONFIG['STLPORT_SOURCES']:
     DIRS += ['stlport']
 
-DIRS += ['pgo']
-
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
     TEST_DIRS += [
         'mobile/sutagent/android',
         'mobile/sutagent/android/watcher',
         'mobile/sutagent/android/ffxcp',
         'mobile/sutagent/android/fencp',
         'mobile/robocop',
     ]
deleted file mode 100644
--- a/build/pgo/Makefile.in
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-relativesrcdir = @relativesrcdir@
-
-include $(DEPTH)/config/autoconf.mk
-
-include $(topsrcdir)/config/rules.mk
-
-# We install to _profile/pgo
-TARGET_DEPTH = ../..
-include $(topsrcdir)/build/automation-build.mk
-
-# Need to override the browser_path from binary-location.mk (included via automation-build.mk)
-# since we want to run from e.g. dist/firefox rather than dist/bin
-ifeq ($(OS_ARCH),Darwin)
-browser_path = \"$(TARGET_DIST)/$(MOZ_APP_NAME)/$(MOZ_MACBUNDLE_NAME)/Contents/MacOS/$(PROGRAM)\"
-else
-browser_path = \"$(TARGET_DIST)/$(MOZ_APP_NAME)/$(PROGRAM)\"
-endif
-   
-# Stuff to make a build with a profile
-
-_PGO_FILES = 	\
-  automation.py \
-  $(topsrcdir)/build/automationutils.py \
-  $(topsrcdir)/testing/profiles/prefs_general.js \
-  profileserver.py \
-  genpgocert.py \
-  index.html \
-  server-locations.txt \
-  favicon.ico \
-  $(NULL)
-
-genpgocert.py: genpgocert.py.in
-	$(PYTHON) $(topsrcdir)/config/Preprocessor.py \
-	$(AUTOMATION_PPARGS) $(DEFINES) $(ACDEFINES) $^ > $@
-
-GARBAGE += genpgocert.py
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
-
-genservercert::
-	$(PYTHON) $(DEPTH)/_profile/pgo/genpgocert.py --gen-server
deleted file mode 100644
--- a/build/pgo/blueprint/Makefile.in
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-include $(topsrcdir)/config/rules.mk
-
-_PROFILE_DIR = $(DEPTH)/_profile/pgo/blueprint
-
-_PGO_FILES = 	\
-  sample.html \
-  elements.html \
-  forms.html \
-  grid.html \
-  test.jpg \
-  test-small.jpg \
-  valid.png \
-  screen.css \
-  print.css \
-  grid.png \
-  fancytype-screen.css \
-  $(NULL)
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
\ No newline at end of file
deleted file mode 100644
--- a/build/pgo/blueprint/moz.build
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
new file mode 100644
--- /dev/null
+++ b/build/pgo/certs/README
@@ -0,0 +1,9 @@
+The certificate authority and server certificates here are generated by $topsrcdir/build/pgo/genpgocert.py.
+
+You can generate a new CA cert by running:
+$objdir/_virtualenv/bin/python $topsrcdir/build/pgo/genpgocert.py --gen-ca
+
+You can generate new server certificates by running:
+$objdir/_virtualenv/bin/python $topsrcdir/build/pgo/genpgocert.py --gen-server
+
+These will place the new files in this directory where you can commit them.
rename from build/pgo/genpgocert.py.in
rename to build/pgo/genpgocert.py
--- a/build/pgo/genpgocert.py.in
+++ b/build/pgo/genpgocert.py
@@ -1,27 +1,29 @@
-#
+#!/usr/bin/env python
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from automation import Automation
-from mozfile import NamedTemporaryFile
+# This script exists to generate the Certificate Authority and server
+# certificates used for SSL testing in Mochitest. The already generated
+# certs are located at $topsrcdir/build/pgo/certs/ .
+
+import mozinfo
 import os
 import random
 import re
 import shutil
+import subprocess
 import sys
+import tempfile
 
-#expand DIST_BIN = __XPC_BIN_PATH__
-#expand BIN_SUFFIX = __BIN_SUFFIX__
-#expand PROFILE_DIR = __PROFILE_DIR__
-#expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
-
-automation = Automation()
+from mozbuild.base import MozbuildObject
+from mozfile import NamedTemporaryFile
+from mozprofile.permissions import ServerLocations
 
 dbFiles = [
   re.compile("^cert[0-9]+\.db$"),
   re.compile("^key[0-9]+\.db$"),
   re.compile("^secmod\.db$")
 ]
 
 def unlinkDbFiles(path):
@@ -36,95 +38,103 @@ def dbFilesExist(path):
     for name in files:
       for dbFile in dbFiles:
         if dbFile.match(name) and os.path.exists(os.path.join(root, name)):
           return True
   return False
 
 
 def runUtil(util, args, inputdata = None):
-  if inputdata:
-    proc = automation.Process([util] + args, env = automation.environment(), stdin = automation.PIPE)
-    proc.communicate(inputdata)
-    return proc.returncode
-  return automation.Process([util] + args, env = automation.environment()).wait()
+  env = os.environ.copy()
+  if mozinfo.os == "linux":
+    pathvar = "LD_LIBRARY_PATH"
+    app_path = os.path.dirname(util)
+    if pathvar in env:
+      env[pathvar] = "%s%s%s" % (app_path, os.pathsep, env[pathvar])
+    else:
+      env[pathvar] = app_path
+  proc = subprocess.Popen([util] + args, env=env,
+                          stdin=subprocess.PIPE if inputdata else None)
+  proc.communicate(inputdata)
+  return proc.returncode
 
 
 def createRandomFile(randomFile):
   for count in xrange(0, 2048):
     randomFile.write(chr(random.randint(0, 255)))
 
 
-def createCertificateAuthority(profileDir, srcDir):
-  certutil = DIST_BIN + "/certutil" + BIN_SUFFIX
-  pk12util = DIST_BIN + "/pk12util" + BIN_SUFFIX
+def createCertificateAuthority(build, srcDir):
+  certutil = build.get_binary_path(what="certutil")
+  pk12util = build.get_binary_path(what="pk12util")
 
-  tempDbDir = os.path.join(profileDir, ".temp")
-  if not os.path.exists(tempDbDir):
-    os.mkdir(tempDbDir)
-
+  #TODO: mozfile.TemporaryDirectory
+  tempDbDir = tempfile.mkdtemp()
   with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
     pgoCAModulePathSrc = os.path.join(srcDir, "pgoca.p12")
     pgoCAPathSrc = os.path.join(srcDir, "pgoca.ca")
 
     pwfile.write("\n")
 
-    unlinkDbFiles(tempDbDir)
-
     # Create temporary certification database for CA generation
     status = runUtil(certutil, ["-N", "-d", tempDbDir, "-f", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
     createRandomFile(rndfile)
     status = runUtil(certutil, ["-S", "-d", tempDbDir, "-s", "CN=Temporary Certificate Authority, O=Mozilla Testing, OU=Profile Guided Optimization", "-t", "C,,", "-x", "-m", "1", "-v", "120", "-n", "pgo temporary ca", "-2", "-f", pwfile.name, "-z", rndfile.name], "Y\n0\nN\n")
-    if status != 0:
+    if status:
       return status
 
     status = runUtil(certutil, ["-L", "-d", tempDbDir, "-n", "pgo temporary ca", "-a", "-o", pgoCAPathSrc, "-f", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
     status = runUtil(pk12util, ["-o", pgoCAModulePathSrc, "-n", "pgo temporary ca", "-d", tempDbDir, "-w", pwfile.name, "-k", pwfile.name])
-    if status != 0:
+    if status:
       return status
 
-    unlinkDbFiles(tempDbDir)
-    os.rmdir(tempDbDir)
+  shutil.rmtree(tempDbDir)
   return 0
 
 
-def createSSLServerCertificate(profileDir, srcDir):
-  certutil = DIST_BIN + "/certutil" + BIN_SUFFIX
-  pk12util = DIST_BIN + "/pk12util" + BIN_SUFFIX
+def createSSLServerCertificate(build, srcDir):
+  certutil = build.get_binary_path(what="certutil")
+  pk12util = build.get_binary_path(what="pk12util")
 
   with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
     pgoCAPath = os.path.join(srcDir, "pgoca.p12")
 
     pwfile.write("\n")
 
     if not dbFilesExist(srcDir):
       # Make sure all DB files from src are really deleted
       unlinkDbFiles(srcDir)
 
       # Create certification database for ssltunnel
       status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
       status = runUtil(pk12util, ["-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
     # Generate automatic certificate
-    locations = automation.readLocations(os.path.join(profileDir, "server-locations.txt"))
-    locations.pop(0)
+    locations = ServerLocations(os.path.join(build.topsrcdir,
+                                             "build", "pgo",
+                                             "server-locations.txt"))
+    iterator = iter(locations)
+
+    # Skips the first entry, I don't know why: bug 879740
+    iterator.next()
+
     locationsParam = ""
     firstLocation = ""
-    for loc in locations:
+    for loc in iterator:
       if loc.scheme == "https" and "nocert" not in loc.options:
         customCertOption = False
         customCertRE = re.compile("^cert=(?:\w+)")
         for option in loc.options:
           match = customCertRE.match(option)
           if match:
             customCertOption = True
             break
@@ -132,45 +142,46 @@ def createSSLServerCertificate(profileDi
         if not customCertOption:
           if len(locationsParam) > 0:
             locationsParam += ","
           locationsParam += loc.host
 
           if firstLocation == "":
             firstLocation = loc.host
 
-    if firstLocation == "":
+    if not firstLocation:
       print "Nothing to generate, no automatic secure hosts specified"
     else:
       createRandomFile(rndfile)
 
       runUtil(certutil, ["-D", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name])
       # Ignore the result, the certificate may not be present when new database is being built
 
       status = runUtil(certutil, ["-S", "-s", "CN=%s" % firstLocation, "-t", "Pu,,", "-c", "pgo temporary ca", "-m", "2", "-8", locationsParam, "-v", "120", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name])
-      if status != 0:
+      if status:
         return status
 
   return 0
 
-
 if len(sys.argv) == 1:
   print "Specify --gen-server or --gen-ca"
   sys.exit(1)
 
+build = MozbuildObject.from_environment()
+certdir = os.path.join(build.topsrcdir, "build", "pgo", "certs")
 if sys.argv[1] == "--gen-server":
-  certificateStatus = createSSLServerCertificate(PROFILE_DIR, CERTS_SRC_DIR)
-  if certificateStatus != 0:
+  certificateStatus = createSSLServerCertificate(build, certdir)
+  if certificateStatus:
     print "TEST-UNEXPECTED-FAIL | SSL Server Certificate generation"
 
   sys.exit(certificateStatus)
 
 if sys.argv[1] == "--gen-ca":
-  certificateStatus = createCertificateAuthority(PROFILE_DIR, CERTS_SRC_DIR)
-  if certificateStatus != 0:
+  certificateStatus = createCertificateAuthority(build, certdir)
+  if certificateStatus:
     print "TEST-UNEXPECTED-FAIL | Certificate Authority generation"
   else:
     print "\n\n"
     print "==================================================="
     print " IMPORTANT:"
     print " To use this new certificate authority in tests"
     print " run 'make' at testing/mochitest"
     print "==================================================="
deleted file mode 100644
--- a/build/pgo/js-input/Makefile.in
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH		= @DEPTH@
-topsrcdir	= @top_srcdir@
-srcdir		= @srcdir@
-VPATH		= @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-include $(topsrcdir)/config/rules.mk
-
-_PROFILE_DIR = $(DEPTH)/_profile/pgo/js-input
-
-_PGO_FILES = 	\
-  3d-cube.html \
-  3d-morph.html \
-  3d-raytrace.html \
-  3d-thingy.html \
-  access-binary-trees.html \
-  access-fannkuch.html \
-  access-nbody.html \
-  access-nsieve.html \
-  bitops-3bit-bits-in-byte.html \
-  bitops-bits-in-byte.html \
-  bitops-bitwise-and.html \
-  bitops-nsieve-bits.html \
-  controlflow-recursive.html \
-  crypto-aes.html \
-  crypto-md5.html \
-  crypto-sha1.html \
-  crypto-otp.html \
-  date-format-tofte.html \
-  date-format-xparb.html \
-  math-cordic.html \
-  math-partial-sums.html \
-  math-spectral-norm.html \
-  regexp-dna.html \
-  string-base64.html \
-  string-fasta.html \
-  string-tagcloud.html \
-  string-unpack-code.html \
-  string-validate-input.html \
-  key.gif \
-  valid-xhtml10.png \
-  $(NULL)
-
-libs:: $(_PGO_FILES)
-	$(INSTALL) $^ $(_PROFILE_DIR)
deleted file mode 100644
--- a/build/pgo/js-input/moz.build
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
deleted file mode 100644
--- a/build/pgo/moz.build
+++ /dev/null
@@ -1,7 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DIRS += ['blueprint', 'js-input']
--- a/build/pgo/profileserver.py
+++ b/build/pgo/profileserver.py
@@ -1,59 +1,71 @@
-#literal #!/usr/bin/python
+#!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import SimpleHTTPServer
-import SocketServer
+from mozprofile import FirefoxProfile, Profile, Preferences
+from mozprofile.permissions import ServerLocations
+from mozrunner import FirefoxRunner, CLI
+from mozhttpd import MozHttpd
+import json
 import socket
 import threading
 import os
 import sys
 import shutil
+import tempfile
 from datetime import datetime
-
-SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
-sys.path.insert(0, SCRIPT_DIR)
-from automation import Automation
-from automationutils import getDebuggerInfo, addCommonOptions
+from mozbuild.base import MozbuildObject
 
 PORT = 8888
-PROFILE_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "./pgoprofile"))
-MOZ_JAR_LOG_FILE = os.path.abspath(os.getenv("JARLOG_FILE"))
-os.chdir(SCRIPT_DIR)
-
-class EasyServer(SocketServer.TCPServer):
-  allow_reuse_address = True
 
 if __name__ == '__main__':
-  from optparse import OptionParser
-  automation = Automation()
+  cli = CLI()
+  debug_args, interactive = cli.debugger_arguments()
 
-  parser = OptionParser()
-  addCommonOptions(parser)
+  build = MozbuildObject.from_environment()
+  httpd = MozHttpd(port=PORT,
+                   docroot=os.path.join(build.topsrcdir, "build", "pgo"))
+  httpd.start(block=False)
 
-  options, args = parser.parse_args()
-
-  debuggerInfo = getDebuggerInfo(".", options.debugger, options.debuggerArgs,
-          options.debuggerInteractive)
+  locations = ServerLocations()
+  locations.add_host(host='127.0.0.1',
+                     port=PORT,
+                     options='primary,privileged')
 
-  httpd = EasyServer(("", PORT), SimpleHTTPServer.SimpleHTTPRequestHandler)
-  t = threading.Thread(target=httpd.serve_forever)
-  t.setDaemon(True) # don't hang on exit
-  t.start()
-  
-  automation.setServerInfo("localhost", PORT)
-  automation.initializeProfile(PROFILE_DIRECTORY)
-  browserEnv = automation.environment()
-  browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
-  browserEnv["MOZ_JAR_LOG_FILE"] = MOZ_JAR_LOG_FILE
+  #TODO: mozfile.TemporaryDirectory
+  profilePath = tempfile.mkdtemp()
+  try:
+    #TODO: refactor this into mozprofile
+    prefpath = os.path.join(build.topsrcdir, "testing", "profiles", "prefs_general.js")
+    prefs = {}
+    prefs.update(Preferences.read_prefs(prefpath))
+    interpolation = { "server": "%s:%d" % httpd.httpd.server_address,
+                      "OOP": "false"}
+    prefs = json.loads(json.dumps(prefs) % interpolation)
+    for pref in prefs:
+      prefs[pref] = Preferences.cast(prefs[pref])
+    profile = FirefoxProfile(profile=profilePath,
+                             preferences=prefs,
+                             #addons=[os.path.join(here, 'extension')],
+                             locations=locations)
 
-  url = "http://localhost:%d/index.html" % PORT
-  appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP)
-  status = automation.runApp(url, browserEnv, appPath, PROFILE_DIRECTORY, {},
-                             debuggerInfo=debuggerInfo,
-                             # the profiling HTML doesn't output anything,
-                             # so let's just run this without a timeout
-                             timeout = None)
-  sys.exit(status)
+    env = os.environ.copy()
+    env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+    env["XPCOM_DEBUG_BREAK"] = "warn"
+    jarlog = os.getenv("JARLOG_FILE")
+    if jarlog:
+      env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
+      print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
+
+    cmdargs = ["http://localhost:%d/index.html" % PORT]
+    runner = FirefoxRunner(profile=profile,
+                           binary=build.get_binary_path(where="staged-package"),
+                           cmdargs=cmdargs,
+                           env=env)
+    runner.start(debug_args=debug_args, interactive=interactive)
+    runner.wait()
+    httpd.stop()
+  finally:
+    shutil.rmtree(profilePath)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -375,17 +375,17 @@ jetpack-tests:
 	$(PYTHON) $(topsrcdir)/addon-sdk/source/bin/cfx -b $(browser_path) --parseable testpkgs
 
 # -- -register
 # -- --trace-malloc malloc.log --shutdown-leaks=sdleak.log
 leaktest:
 	$(PYTHON) _leaktest/leaktest.py $(LEAKTEST_ARGS)
 
 pgo-profile-run:
-	$(PYTHON) $(DEPTH)/_profile/pgo/profileserver.py $(EXTRA_TEST_ARGS)
+	$(PYTHON) $(topsrcdir)/build/pgo/profileserver.py $(EXTRA_TEST_ARGS)
 
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 ifndef UNIVERSAL_BINARY
 PKG_STAGE = $(DIST)/test-package-stage
 package-tests: \
   stage-config \