Bug 688667 - refactor automation.py into mozprocess and mozprofile and load via virtualenv. r=jmaher
authorJeff Hammel <jhammel@mozilla.com>
Tue, 30 Jul 2013 08:30:40 -0400
changeset 140506 c22896a275640374218862768e95df771a709956
parent 140505 25e81fe30063a5973a1e7700d1f3e4df09a68c17
child 140507 b8af597df1d023e661abe9ab9cf3b9588fb54a41
push id25030
push userryanvm@gmail.com
push dateTue, 30 Jul 2013 17:07:39 +0000
treeherdermozilla-central@129ce98f4cb2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs688667
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 688667 - refactor automation.py into mozprocess and mozprofile and load via virtualenv. r=jmaher
build/automation.py.in
build/leaktest.py.in
build/mach_bootstrap.py
testing/mochitest/runtests.py
testing/mochitest/runtestsremote.py
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -1,49 +1,55 @@
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
 import codecs
-from datetime import datetime, timedelta
 import itertools
+import json
 import logging
 import os
 import re
 import select
 import shutil
 import signal
 import subprocess
 import sys
 import threading
 import tempfile
 import sqlite3
+from datetime import datetime, timedelta
 from string import Template
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
 # --------------------------------------------------------------
 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
-#
-here = os.path.dirname(__file__)
+# These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR
+here = os.path.dirname(os.path.realpath(__file__))
 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
 
 if os.path.isdir(mozbase):
     for package in os.listdir(mozbase):
-        sys.path.append(os.path.join(mozbase, package))
+        package_path = os.path.join(mozbase, package)
+        if package_path not in sys.path:
+            sys.path.append(package_path)
 
 import mozcrash
+from mozprofile import Profile, Preferences
+from mozprofile.permissions import ServerLocations
 
 # ---------------------------------------------------------------
 
 _DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js')
+_DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json')
 
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
 _DEFAULT_WEBSOCKET_PORT = 9988
 
 # from nsIPrincipal.idl
 _APP_STATUS_NOT_INSTALLED = 0
@@ -288,16 +294,19 @@ class Automation(object):
                                 match.group("port"), options))
 
     if not seenPrimary:
       raise SyntaxError(lineno + 1, "missing primary location")
 
     return locations
 
   def setupPermissionsDatabase(self, profileDir, permissions):
+    # Included for reftest compatibility;
+    # see https://bugzilla.mozilla.org/show_bug.cgi?id=688667
+
     # Open database and create table
     permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite"))
     cursor = permDB.cursor();
 
     cursor.execute("PRAGMA user_version=3");
 
     # SQL copied from nsPermissionManager.cpp
     cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
@@ -315,342 +324,87 @@ class Automation(object):
       for host,allow in permissions[perm]:
         cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)",
                        (host, perm, 1 if allow else 2))
 
     # Commit and close
     permDB.commit()
     cursor.close()
 
-  def setupTestApps(self, profileDir, apps):
-    webappJSONTemplate = Template(""""$id": {
-  "origin": "$origin",
-  "installOrigin": "$origin",
-  "receipt": null,
-  "installTime": 132333986000,
-  "manifestURL": "$manifestURL",
-  "localId": $localId,
-  "id": "$id",
-  "appStatus": $appStatus,
-  "csp": "$csp"
-}""")
-
-    manifestTemplate = Template("""{
-  "name": "$name",
-  "csp": "$csp",
-  "description": "$description",
-  "launch_path": "/",
-  "developer": {
-    "name": "Mozilla",
-    "url": "https://mozilla.org/"
-  },
-  "permissions": [
-  ],
-  "locales": {
-    "en-US": {
-      "name": "$name",
-      "description": "$description"
-    }
-  },
-  "default_locale": "en-US",
-  "icons": {
-  }
-}
-""")
-
-    # Create webapps/webapps.json
-    webappsDir = os.path.join(profileDir, "webapps")
-    if not os.access(webappsDir, os.F_OK):
-      os.mkdir(webappsDir)
-
-    lineRe = re.compile(r'(.*?)"(.*?)": (.*)')
-
-    webappsJSONFilename = os.path.join(webappsDir, "webapps.json")
-    webappsJSON = []
-    if os.access(webappsJSONFilename, os.F_OK):
-      # If there is an existing webapps.json file (which will be the case for
-      # b2g), we parse the data in the existing file before appending test
-      # test apps to it.
-      startId = 1
-      webappsJSONFile = open(webappsJSONFilename, "r")
-      contents = webappsJSONFile.read()
-
-      for app_content in contents.split('},'):
-        app = {}
-        # ghetto json parser needed due to lack of json/simplejson on test slaves
-        for line in app_content.split('\n'):
-          m = lineRe.match(line)
-          if m:
-            value = m.groups()[2]
-            # remove any trailing commas
-            if value[-1:] == ',':
-              value = value[:-1]
-            # set the app name from a line that looks like this:
-            # "name.gaiamobile.org": {
-            if value == '{':
-              app['id'] = m.groups()[1]
-            # parse string, None, bool and int types
-            elif value[0:1] == '"':
-              app[m.groups()[1]] = value[1:-1]
-            elif value == "null":
-              app[m.groups()[1]] = None
-            elif value == "true":
-              app[m.groups()[1]] = True
-            elif value == "false":
-              app[m.groups()[1]] = False
-            else:
-              app[m.groups()[1]] = int(value)
-        if app:
-          webappsJSON.append(app)
-
-      webappsJSONFile.close()
-
-    startId = 1
-    for app in webappsJSON:
-      if app['localId'] >= startId:
-        startId = app['localId'] + 1
-      if not app.get('csp'):
-        app['csp'] = ''
-      if not app.get('appStatus'):
-        app['appStatus'] = 3
-
-    for localId, app in enumerate(apps):
-      app['localId'] = localId + startId # localId must be from 1..N
-      if not app.get('id'):
-        app['id'] = app['name']
-      webappsJSON.append(app)
-
-    contents = []
-    for app in webappsJSON:
-      contents.append(webappJSONTemplate.substitute(app))
-    contents = '{\n' + ',\n'.join(contents) + '\n}\n'
-
-    webappsJSONFile = open(webappsJSONFilename, "w")
-    webappsJSONFile.write(contents)
-    webappsJSONFile.close()
-
-    # Create manifest file for each app.
-    for app in apps:
-      manifest = manifestTemplate.substitute(app)
-
-      manifestDir = os.path.join(webappsDir, app['name'])
-      os.mkdir(manifestDir)
-
-      manifestFile = open(os.path.join(manifestDir, "manifest.webapp"), "a")
-      manifestFile.write(manifest)
-      manifestFile.close()
-
   def initializeProfile(self, profileDir,
                               extraPrefs=None,
                               useServerLocations=False,
-                              initialProfile=None,
-                              prefsPath=_DEFAULT_PREFERENCE_FILE):
+                              prefsPath=_DEFAULT_PREFERENCE_FILE,
+                              appsPath=_DEFAULT_APPS_FILE,
+                              addons=None):
     " Sets up the standard testing profile."
 
     extraPrefs = extraPrefs or []
-    prefs = []
-    # Start with a clean slate.
-    shutil.rmtree(profileDir, True)
-
-    if initialProfile:
-      shutil.copytree(initialProfile, profileDir)
-    else:
-      os.mkdir(profileDir)
-
-    # Set up permissions database
-    locations = self.readLocations()
-    self.setupPermissionsDatabase(profileDir,
-      {'allowXULXBL':[(l.host, 'noxul' not in l.options) for l in locations]});
-
-    f = open(prefsPath, 'r')
-    part = f.read() % {"server" : "%s:%s" % (self.webServer, self.httpPort)}
-    f.close()
-    prefs.append(part)
-
-    if useServerLocations:
-      # We need to proxy every server but the primary one.
-      origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
-                for l in filter(lambda l: "primary" not in l.options, locations)]
-      origins = ", ".join(origins)
 
-      pacURL = """data:text/plain,
-function FindProxyForURL(url, host)
-{
-  var origins = [%(origins)s];
-  var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
-                         '://' +
-                         '(?:[^/@]*@)?' +
-                         '(.*?)' +
-                         '(?::(\\\\\\\\d+))?/');
-  var matches = regex.exec(url);
-  if (!matches)
-    return 'DIRECT';
-  var isHttp = matches[1] == 'http';
-  var isHttps = matches[1] == 'https';
-  var isWebSocket = matches[1] == 'ws';
-  var isWebSocketSSL = matches[1] == 'wss';
-  if (!matches[3])
-  {
-    if (isHttp | isWebSocket) matches[3] = '80';
-    if (isHttps | isWebSocketSSL) matches[3] = '443';
-  }
-  if (isWebSocket)
-    matches[1] = 'http';
-  if (isWebSocketSSL)
-    matches[1] = 'https';
+    # create the profile
+    prefs = {}
+    locations = None
+    if useServerLocations:
+        locations = ServerLocations()
+        locations.read(os.path.abspath('server-locations.txt'), True)
+    else:
+      prefs['network.proxy.type'] = 0
 
-  var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
-  if (origins.indexOf(origin) < 0)
-    return 'DIRECT';
-  if (isHttp)
-    return 'PROXY %(remote)s:%(httpport)s';
-  if (isHttps || isWebSocket || isWebSocketSSL)
-    return 'PROXY %(remote)s:%(sslport)s';
-  return 'DIRECT';
-}""" % { "origins": origins,
-         "remote":  self.webServer,
-         "httpport":self.httpPort,
-         "sslport": self.sslPort }
-      pacURL = "".join(pacURL.splitlines())
-
-      part = """
-user_pref("network.proxy.type", 2);
-user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
-
-user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
-""" % {"pacURL": pacURL}
-    else:
-      part = 'user_pref("network.proxy.type", 0);\n'
-    prefs.append(part)
+    prefs.update(Preferences.read_prefs(prefsPath))
 
     for v in extraPrefs:
       thispref = v.split("=", 1)
       if len(thispref) < 2:
         print "Error: syntax error in --setpref=" + v
         sys.exit(1)
-      part = 'user_pref("%s", %s);\n' % (thispref[0], thispref[1])
-      prefs.append(part)
+      prefs[thispref[0]] = thispref[1]
+
 
-    # write the preferences
-    prefsFile = open(profileDir + "/" + "user.js", "a")
-    prefsFile.write("".join(prefs))
-    prefsFile.close()
+    interpolation = {"server": "%s:%s" % (self.webServer, self.httpPort)}
+    prefs = json.loads(json.dumps(prefs) % interpolation)
+    for pref in prefs:
+        prefs[pref] = Preferences.cast(prefs[pref])
+
+    # load apps
+    apps = None
+    if appsPath and os.path.exists(appsPath):
+        with open(appsPath, 'r') as apps_file:
+            apps = json.load(apps_file)
 
-    apps = [
-      {
-        'name': 'http_example_org',
-        'csp': '',
-        'origin': 'http://example.org',
-        'manifestURL': 'http://example.org/manifest.webapp',
-        'description': 'http://example.org App',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'https_example_com',
-        'csp': '',
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest.webapp',
-        'description': 'https://example.com App',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'http_test1_example_org',
-        'csp': '',
-        'origin': 'http://test1.example.org',
-        'manifestURL': 'http://test1.example.org/manifest.webapp',
-        'description': 'http://test1.example.org App',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'http_test1_example_org_8000',
-        'csp': '',
-        'origin': 'http://test1.example.org:8000',
-        'manifestURL': 'http://test1.example.org:8000/manifest.webapp',
-        'description': 'http://test1.example.org:8000 App',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'http_sub1_test1_example_org',
-        'csp': '',
-        'origin': 'http://sub1.test1.example.org',
-        'manifestURL': 'http://sub1.test1.example.org/manifest.webapp',
-        'description': 'http://sub1.test1.example.org App',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'https_example_com_privileged',
-        'csp': '',
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest_priv.webapp',
-        'description': 'https://example.com Privileged App',
-        'appStatus': _APP_STATUS_PRIVILEGED
-      },
-      {
-        'name': 'https_example_com_certified',
-        'csp': '',
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest_cert.webapp',
-        'description': 'https://example.com Certified App',
-        'appStatus': _APP_STATUS_CERTIFIED
-      },
-      {
-        'name': 'https_example_csp_certified',
-        'csp': "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest_csp_cert.webapp',
-        'description': 'https://example.com Certified App with manifest policy',
-        'appStatus': _APP_STATUS_CERTIFIED
-      },
-      {
-        'name': 'https_example_csp_installed',
-        'csp': "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest_csp_inst.webapp',
-        'description': 'https://example.com Installed App with manifest policy',
-        'appStatus': _APP_STATUS_INSTALLED
-      },
-      {
-        'name': 'https_example_csp_privileged',
-        'csp': "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
-        'origin': 'https://example.com',
-        'manifestURL': 'https://example.com/manifest_csp_priv.webapp',
-        'description': 'https://example.com Privileged App with manifest policy',
-        'appStatus': _APP_STATUS_PRIVILEGED
-      },
-      {
-        'name': 'https_a_domain_certified',
-        'csp': "",
-        'origin': 'https://acertified.com',
-        'manifestURL': 'https://acertified.com/manifest.webapp',
-        'description': 'https://acertified.com Certified App',
-        'appStatus': _APP_STATUS_CERTIFIED
-      },
-      {
-        'name': 'https_a_domain_privileged',
-        'csp': "",
-        'origin': 'https://aprivileged.com',
-        'manifestURL': 'https://aprivileged.com/manifest.webapp',
-        'description': 'https://aprivileged.com Privileged App ',
-        'appStatus': _APP_STATUS_PRIVILEGED
-      },
-    ];
-    self.setupTestApps(profileDir, apps)
+    proxy = {'remote': str(self.webServer),
+             'http': str(self.httpPort),
+             'https': str(self.sslPort),
+    # use SSL port for legacy compatibility; see
+    # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66
+    # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
+    #             'ws': str(self.webSocketPort)
+             'ws': str(self.sslPort)
+             }
+
+    # return profile object
+    profile = Profile(profile=profileDir,
+                      addons=addons,
+                      locations=locations,
+                      preferences=prefs,
+                      restore=False,
+                      apps=apps,
+                      proxy=proxy)
+    return profile
 
   def addCommonOptions(self, parser):
     "Adds command-line options which are common to mochitest and reftest."
 
     parser.add_option("--setpref",
                       action = "append", type = "string",
                       default = [],
                       dest = "extraPrefs", metavar = "PREF=VALUE",
-                      help = "defines an extra user preference")  
+                      help = "defines an extra user preference")
 
   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
     pwfilePath = os.path.join(profileDir, ".crtdbpw")
-  
     pwfile = open(pwfilePath, "w")
     pwfile.write("\n")
     pwfile.close()
 
     # Create head of the ssltunnel configuration file
     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
     sslTunnelConfig = open(sslTunnelConfigPath, "w")
   
--- a/build/leaktest.py.in
+++ b/build/leaktest.py.in
@@ -33,17 +33,20 @@ if __name__ == '__main__':
                                 "ONLY logging to stdout.")
 
     httpd = EasyServer(("", PORT), SimpleHTTPServer.SimpleHTTPRequestHandler)
     t = threading.Thread(target=httpd.serve_forever)
     t.setDaemon(True)
     t.start()
 
     automation.setServerInfo("localhost", PORT)
-    automation.initializeProfile(PROFILE_DIRECTORY)
+
+    # keep a profile reference so that it is not cleaned up immediately via __del__
+    profile = automation.initializeProfile(PROFILE_DIRECTORY)
+
     browserEnv = automation.environment()
 
     if not "XPCOM_DEBUG_BREAK" in browserEnv:
         browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
     url = "http://localhost:%d/bloatcycle.html" % PORT
     appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP)
     status = automation.runApp(url, browserEnv, appPath, PROFILE_DIRECTORY,
                                extraArgs, timeout=1800)
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -46,16 +46,17 @@ SEARCH_PATHS = [
     'testing/mozbase/mozfile',
     'testing/mozbase/mozhttpd',
     'testing/mozbase/mozlog',
     'testing/mozbase/moznetwork',
     'testing/mozbase/mozprocess',
     'testing/mozbase/mozprofile',
     'testing/mozbase/mozrunner',
     'testing/mozbase/mozinfo',
+    'testing/mozbase/manifestdestiny',
     'xpcom/idl-parser',
 ]
 
 # Individual files providing mach commands.
 MACH_MODULES = [
     'addon-sdk/mach_commands.py',
     'layout/tools/reftest/mach_commands.py',
     'python/mach_commands.py',
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -5,17 +5,16 @@
 
 """
 Runs the Mochitest test harness.
 """
 
 from __future__ import with_statement
 import optparse
 import os
-import os.path
 import sys
 import time
 import traceback
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 sys.path.insert(0, SCRIPT_DIR);
 
 import shutil
@@ -427,24 +426,38 @@ class Mochitest(MochitestUtilsMixin):
       self.SERVER_STARTUP_TIMEOUT = 180
     else:
       self.SERVER_STARTUP_TIMEOUT = 90
 
   def buildProfile(self, options):
     """ create the profile and add optional chrome bits and files if requested """
     if options.browserChrome and options.timeout:
       options.extraPrefs.append("testing.browserTestHarness.timeout=%d" % options.timeout)
-    self.automation.initializeProfile(options.profilePath,
-                                      options.extraPrefs,
-                                      useServerLocations=True,
-                                      prefsPath=os.path.join(SCRIPT_DIR,
-                                                        'profile_data', 'prefs_general.js'))
+
+    # get extensions to install
+    extensions = self.getExtensionsToInstall(options)
+
+    # create a profile
+    appsPath = os.path.join(SCRIPT_DIR, 'profile_data', 'webapps_mochitest.json')
+    appsPath = appsPath if os.path.exists(appsPath) else None
+    prefsPath = os.path.join(SCRIPT_DIR, 'profile_data', 'prefs_general.js')
+    profile = self.automation.initializeProfile(options.profilePath,
+                                                options.extraPrefs,
+                                                useServerLocations=True,
+                                                appsPath=appsPath,
+                                                prefsPath=prefsPath,
+                                                addons=extensions)
+
+    #if we don't do this, the profile object is destroyed when we exit this method
+    self.profile = profile
+    options.profilePath = profile.profile
+
+    manifest = self.addChromeToProfile(options)
     self.copyExtraFilesToProfile(options)
-    self.installExtensionsToProfile(options)
-    return self.addChromeToProfile(options)
+    return manifest
 
   def buildBrowserEnv(self, options):
     """ build the environment variables for the specific test and operating system """
     browserEnv = self.automation.environment(xrePath = options.xrePath)
 
     # These variables are necessary for correct application startup; change
     # via the commandline at your own risk.
     browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -598,16 +598,23 @@ def main():
         retVal = None
         for test in robocop_tests:
             if options.testPath and options.testPath != test['name']:
                 continue
 
             if not test['name'] in my_tests:
                 continue
 
+            # When running in a loop, we need to create a fresh profile for each cycle
+            if mochitest.localProfile:
+                options.profilePath = mochitest.localProfile
+                os.system("rm -Rf %s" % options.profilePath)
+                options.profilePath = tempfile.mkdtemp()
+                mochitest.localProfile = options.profilePath
+
             options.app = "am"
             options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"]
             options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name']))
             options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname)
 
             # If the test is for checking the import from bookmarks then make sure there is data to import
             if test['name'] == "testImportFromAndroid":