Bug 1272113 - Add awsy package into m-c. r=erahm, a=test-only
authorPaul Yang <pyang@mozilla.com>
Sat, 11 Mar 2017 02:33:19 +0800
changeset 396105 b2ca3df7fc70bc815e53c71f0bc709c3ba30c584
parent 396104 2bf6db48a260233ba421cd216aeeaea957887123
child 396106 c63ba9243b981063d66bc8e7d8634691ba60119d
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerserahm, test-only
bugs1272113
milestone54.0
Bug 1272113 - Add awsy package into m-c. r=erahm, a=test-only MozReview-Commit-ID: GVtHMiipWBT
testing/awsy/README.md
testing/awsy/awsy/__init__.py
testing/awsy/awsy/parse_about_memory.py
testing/awsy/awsy/process_perf_data.py
testing/awsy/awsy/test_memory_usage.py
testing/awsy/awsy/webservers.py
testing/awsy/conf/prefs.json
testing/awsy/conf/testvars.json
testing/awsy/requirements.txt
testing/awsy/setup.py
testing/awsy/tp5n-pageset.manifest
new file mode 100644
--- /dev/null
+++ b/testing/awsy/README.md
@@ -0,0 +1,2 @@
+# awsy-lite
+Barebones are we slim yet test.
new file mode 100644
--- /dev/null
+++ b/testing/awsy/awsy/__init__.py
@@ -0,0 +1,123 @@
+# Maximum number of tabs to open
+MAX_TABS = 30
+
+# Default amount of seconds to wait in between opening tabs
+PER_TAB_PAUSE = 10
+
+# Default amount of seconds to wait for things to be settled down
+SETTLE_WAIT_TIME = 30
+
+# Amount of times to run through the test suite
+ITERATIONS = 5
+
+# Talos TP5
+TEST_SITES_TEMPLATES = [
+    "http://localhost:{}/tp5n/thesartorialist.blogspot.com/thesartorialist.blogspot.com/index.html",
+    "http://localhost:{}/tp5n/cakewrecks.blogspot.com/cakewrecks.blogspot.com/index.html",
+    "http://localhost:{}/tp5n/baidu.com/www.baidu.com/s@wd=mozilla.html",
+    "http://localhost:{}/tp5n/en.wikipedia.org/en.wikipedia.org/wiki/Rorschach_test.html",
+    "http://localhost:{}/tp5n/twitter.com/twitter.com/ICHCheezburger.html",
+    "http://localhost:{}/tp5n/msn.com/www.msn.com/index.html",
+    "http://localhost:{}/tp5n/yahoo.co.jp/www.yahoo.co.jp/index.html",
+    "http://localhost:{}/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html",
+    "http://localhost:{}/tp5n/linkedin.com/www.linkedin.com/in/christopherblizzard@goback=.nppvan_%252Flemuelf.html",
+    "http://localhost:{}/tp5n/bing.com/www.bing.com/search@q=mozilla&go=&form=QBLH&qs=n&sk=&sc=8-0.html",
+    "http://localhost:{}/tp5n/icanhascheezburger.com/icanhascheezburger.com/index.html",
+    "http://localhost:{}/tp5n/yandex.ru/yandex.ru/yandsearch@text=mozilla&lr=21215.html",
+    "http://localhost:{}/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html",
+    "http://localhost:{}/tp5n/163.com/www.163.com/index.html",
+    "http://localhost:{}/tp5n/mail.ru/mail.ru/index.html",
+    "http://localhost:{}/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html",
+    "http://localhost:{}/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html",
+    "http://localhost:{}/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html",
+    "http://localhost:{}/tp5n/mozilla.com/www.mozilla.com/en-US/firefox/all-older.html",
+    "http://localhost:{}/tp5n/ask.com/www.ask.com/web@q=What%27s+the+difference+between+brown+and+white+eggs%253F&gc=1&qsrc=3045&o=0&l=dir.html",
+    "http://localhost:{}/tp5n/cnn.com/www.cnn.com/index.html",
+    "http://localhost:{}/tp5n/sohu.com/www.sohu.com/index.html",
+    "http://localhost:{}/tp5n/vkontakte.ru/vkontakte.ru/help.php@page=about.html",
+    "http://localhost:{}/tp5n/youku.com/www.youku.com/index.html",
+    "http://localhost:{}/tp5n/myparentswereawesome.tumblr.com/myparentswereawesome.tumblr.com/index.html",
+    "http://localhost:{}/tp5n/ifeng.com/ifeng.com/index.html",
+    "http://localhost:{}/tp5n/ameblo.jp/ameblo.jp/index.html",
+    "http://localhost:{}/tp5n/tudou.com/www.tudou.com/index.html",
+    "http://localhost:{}/tp5n/chemistry.about.com/chemistry.about.com/index.html",
+    "http://localhost:{}/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html",
+    "http://localhost:{}/tp5n/hao123.com/hao123.com/index.html",
+    "http://localhost:{}/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html",
+    "http://localhost:{}/tp5n/alibaba.com/www.alibaba.com/product-tp/101509462/World_s_Cheapest_Laptop.html",
+    "http://localhost:{}/tp5n/uol.com.br/www.uol.com.br/index.html",
+    "http://localhost:{}/tp5n/cnet.com/www.cnet.com/index.html",
+    "http://localhost:{}/tp5n/ehow.com/www.ehow.com/how_4575878_prevent-fire-home.html",
+    "http://localhost:{}/tp5n/thepiratebay.org/thepiratebay.org/top/201.html",
+    "http://localhost:{}/tp5n/page.renren.com/page.renren.com/index.html",
+    "http://localhost:{}/tp5n/chinaz.com/chinaz.com/index.html",
+    "http://localhost:{}/tp5n/globo.com/www.globo.com/index.html",
+    "http://localhost:{}/tp5n/spiegel.de/www.spiegel.de/index.html",
+    "http://localhost:{}/tp5n/dailymotion.com/www.dailymotion.com/us.html",
+    "http://localhost:{}/tp5n/goo.ne.jp/goo.ne.jp/index.html",
+    "http://localhost:{}/tp5n/alipay.com/www.alipay.com/index.html",
+    "http://localhost:{}/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html",
+    "http://localhost:{}/tp5n/nicovideo.jp/www.nicovideo.jp/index.html",
+    "http://localhost:{}/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html",
+    "http://localhost:{}/tp5n/taringa.net/www.taringa.net/index.html",
+    "http://localhost:{}/tp5n/tmall.com/www.tmall.com/index.html@ver=2010s.html",
+    "http://localhost:{}/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html",
+    "http://localhost:{}/tp5n/deviantart.com/www.deviantart.com/index.html",
+    "http://localhost:{}/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html",
+    "http://localhost:{}/tp5n/douban.com/www.douban.com/index.html",
+    "http://localhost:{}/tp5n/imgur.com/imgur.com/gallery/index.html",
+    "http://localhost:{}/tp5n/reddit.com/www.reddit.com/index.html",
+    "http://localhost:{}/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html",
+    "http://localhost:{}/tp5n/filestube.com/www.filestube.com/t/the+vampire+diaries.html",
+    "http://localhost:{}/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html",
+    "http://localhost:{}/tp5n/whois.domaintools.com/whois.domaintools.com/mozilla.com.html",
+    "http://localhost:{}/tp5n/indiatimes.com/www.indiatimes.com/index.html",
+    "http://localhost:{}/tp5n/rambler.ru/www.rambler.ru/index.html",
+    "http://localhost:{}/tp5n/torrentz.eu/torrentz.eu/search@q=movies.html",
+    "http://localhost:{}/tp5n/reuters.com/www.reuters.com/index.html",
+    "http://localhost:{}/tp5n/foxnews.com/www.foxnews.com/index.html",
+    "http://localhost:{}/tp5n/xinhuanet.com/xinhuanet.com/index.html",
+    "http://localhost:{}/tp5n/56.com/www.56.com/index.html",
+    "http://localhost:{}/tp5n/bild.de/www.bild.de/index.html",
+    "http://localhost:{}/tp5n/guardian.co.uk/www.guardian.co.uk/index.html",
+    "http://localhost:{}/tp5n/w3schools.com/www.w3schools.com/html/default.asp.html",
+    "http://localhost:{}/tp5n/naver.com/www.naver.com/index.html",
+    "http://localhost:{}/tp5n/blogfa.com/blogfa.com/index.html",
+    "http://localhost:{}/tp5n/terra.com.br/www.terra.com.br/portal/index.html",
+    "http://localhost:{}/tp5n/ucoz.ru/www.ucoz.ru/index.html",
+    "http://localhost:{}/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html",
+    "http://localhost:{}/tp5n/wsj.com/online.wsj.com/home-page.html",
+    "http://localhost:{}/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html",
+    "http://localhost:{}/tp5n/myspace.com/www.myspace.com/albumart.html",
+    "http://localhost:{}/tp5n/google.com/www.google.com/search@q=mozilla.html",
+    "http://localhost:{}/tp5n/orange.fr/www.orange.fr/index.html",
+    "http://localhost:{}/tp5n/php.net/php.net/index.html",
+    "http://localhost:{}/tp5n/zol.com.cn/www.zol.com.cn/index.html",
+    "http://localhost:{}/tp5n/mashable.com/mashable.com/index.html",
+    "http://localhost:{}/tp5n/etsy.com/www.etsy.com/category/geekery/videogame.html",
+    "http://localhost:{}/tp5n/gmx.net/www.gmx.net/index.html",
+    "http://localhost:{}/tp5n/csdn.net/csdn.net/index.html",
+    "http://localhost:{}/tp5n/xunlei.com/xunlei.com/index.html",
+    "http://localhost:{}/tp5n/hatena.ne.jp/www.hatena.ne.jp/index.html",
+    "http://localhost:{}/tp5n/icious.com/www.delicious.com/index.html",
+    "http://localhost:{}/tp5n/repubblica.it/www.repubblica.it/index.html",
+    "http://localhost:{}/tp5n/web.de/web.de/index.html",
+    "http://localhost:{}/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html",
+    "http://localhost:{}/tp5n/telegraph.co.uk/www.telegraph.co.uk/index.html",
+    "http://localhost:{}/tp5n/seesaa.net/blog.seesaa.jp/index.html",
+    "http://localhost:{}/tp5n/wp.pl/www.wp.pl/index.html",
+    "http://localhost:{}/tp5n/aljazeera.net/aljazeera.net/portal.html",
+    "http://localhost:{}/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html",
+    "http://localhost:{}/tp5n/homeway.com.cn/www.hexun.com/index.html",
+    "http://localhost:{}/tp5n/facebook.com/www.facebook.com/Google.html",
+    "http://localhost:{}/tp5n/youtube.com/www.youtube.com/music.html",
+    "http://localhost:{}/tp5n/people.com.cn/people.com.cn/index.html"
+]
+
+__all__ = ["MAX_TABS",
+           "PER_TAB_PAUSE",
+           "SETTLE_WAIT_TIME",
+           "ITERATIONS",
+           "TEST_SITES_TEMPLATES",
+           "webservers",
+           "process_perf_data"]
new file mode 100644
--- /dev/null
+++ b/testing/awsy/awsy/parse_about_memory.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+# Firefox about:memory log parser.
+
+import argparse
+from collections import defaultdict
+import gzip
+import json
+
+
+def path_total(data, path):
+    totals = defaultdict(int)
+    totals_heap = defaultdict(int)
+    totals_heap_allocated = defaultdict(int)
+    for report in data["reports"]:
+        if report["kind"] == 1 and report["path"].startswith("explicit/"):
+            totals_heap[report["process"]] += report["amount"]
+
+        if report["path"].startswith(path):
+            totals[report["process"]] += report["amount"]
+            if report["kind"] == 1:
+                totals_heap[report["process"]] += report["amount"]
+        elif report["path"] == "heap-allocated":
+            totals_heap_allocated[report["process"]] = report["amount"]
+
+    if path == "explicit/":
+        for k, v in totals_heap.items():
+            if k in totals_heap_allocated:
+                heap_unclassified = totals_heap_allocated[k] - totals_heap[k]
+                totals[k] += heap_unclassified
+    elif path == "explicit/heap-unclassified":
+        for k, v in totals_heap.items():
+            if k in totals_heap_allocated:
+                totals[k] = totals_heap_allocated[k] - totals_heap[k]
+
+    return totals
+
+
+def calculate_memory_report_values(memory_report_path, data_point_path,
+                                   process_name=None):
+    """
+    Opens the given memory report file and calculates the value for the given
+    data point.
+
+    :param memory_report_path: Path to the memory report file to parse.
+    :param data_point_path: Path of the data point to calculate in the memory
+     report, ie: 'explicit/heap-unclassified'.
+    :param process_name: Name of process to limit reports to. ie 'Main'
+    """
+    data = None
+
+    try:
+        with open(memory_report_path) as f:
+            data = json.load(f)
+    except ValueError, e:
+        # Check if the file is gzipped.
+        with gzip.open(memory_report_path, 'rb') as f:
+            data = json.load(f)
+
+    totals = path_total(data, data_point_path)
+
+    # If a process name is provided, restricted output to processes matching
+    # that name.
+    if process_name:
+        for k in totals.keys():
+            if not process_name in k:
+                del totals[k]
+
+    return totals
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+            description='Extract data points from about:memory reports')
+    parser.add_argument('report', action='store',
+                        help='Path to a memory report file.')
+    parser.add_argument('prefix', action='store',
+                        help='Prefix of data point to measure.')
+    parser.add_argument('--proc-filter', action='store', default=None,
+                        help='Process name filter. If not provided all processes will be included.')
+
+    args = parser.parse_args()
+    totals = calculate_memory_report_values(
+                    args.report, args.prefix, args.proc_filter)
+
+    sorted_totals = sorted(totals.iteritems(), key=lambda(k,v): (-v,k))
+    for (k, v) in sorted_totals:
+        if v:
+            print "{0}\t".format(k),
+    print ""
+    for (k, v) in sorted_totals:
+        if v:
+            print "{0}\t".format(v),
+    print ""
new file mode 100644
--- /dev/null
+++ b/testing/awsy/awsy/process_perf_data.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import json
+import math
+import glob
+import parse_about_memory
+
+# A description of each checkpoint and the root path to it.
+CHECKPOINTS = [
+    { 'name': "Fresh start", 'path': "memory-report-Start-0.json.gz" },
+    { 'name': "Fresh start [+30s]", 'path': "memory-report-StartSettled-0.json.gz" },
+    { 'name': "After tabs open", 'path': "memory-report-TabsOpen-4.json.gz" },
+    { 'name': "After tabs open [+30s]", 'path': "memory-report-TabsOpenSettled-4.json.gz" },
+    { 'name': "After tabs open [+30s, forced GC]", 'path': "memory-report-TabsOpenForceGC-4.json.gz" },
+    { 'name': "Tabs closed", 'path': "memory-report-TabsClosed-4.json.gz" },
+    { 'name': "Tabs closed [+30s]", 'path': "memory-report-TabsClosedSettled-4.json.gz" },
+    { 'name': "Tabs closed [+30s, forced GC]", 'path': "memory-report-TabsClosedForceGC-4.json.gz" }
+]
+
+# A description of each perfherder suite and the path to its values.
+PERF_SUITES = [
+    { 'name': "Resident Memory", 'node': "resident" },
+    { 'name': "Explicit Memory", 'node': "explicit/" },
+    { 'name': "Heap Unclassified", 'node': "explicit/heap-unclassified" },
+    { 'name': "JS", 'node': "js-main-runtime" },
+    { 'name': "Images", 'node': "explicit/images" }
+]
+
+def update_checkpoint_paths(checkpoint_files):
+    """
+    Updates CHECKPOINTS with memory report file fetched in data_path
+    :param checkpoint_files: list of files in data_path
+    """
+    target_path = [['Start-', 0],
+                      ['StartSettled-', 0],
+                      ['TabsOpen-', -1],
+                      ['TabsOpenSettled-', -1],
+                      ['TabsOpenForceGC-', -1],
+                      ['TabsClosed-', -1],
+                      ['TabsClosedSettled-', -1],
+                      ['TabsClosedForceGC-', -1]]
+    for i in range(len(target_path)):
+        (name, idx) = target_path[i]
+        paths = sorted([x for x in checkpoint_files if name in x])
+        CHECKPOINTS[i]['path'] = paths[idx]
+
+def create_suite(name, node, data_path):
+    """
+    Creates a suite suitable for adding to a perfherder blob. Calculates the
+    geometric mean of the checkpoint values and adds that to the suite as
+    well.
+
+    :param name: The name of the suite.
+    :param node: The path of the data node to extract data from.
+    :param data_path: The directory to retrieve data from.
+    """
+    suite = {
+        'name': name,
+        'subtests': [],
+        'lowerIsBetter': True,
+        'units': 'bytes'
+    }
+    update_checkpoint_paths(glob.glob(os.path.join(data_path, "memory-report*")))
+
+    total = 0
+    for checkpoint in CHECKPOINTS:
+        memory_report_path = os.path.join(data_path, checkpoint['path'])
+
+        if node != "resident":
+            totals = parse_about_memory.calculate_memory_report_values(
+                                            memory_report_path, node)
+            value = sum(totals.values())
+        else:
+            # For "resident" we really want RSS of the chrome ("Main") process
+            # and USS of the child processes. We'll still call it resident
+            # for simplicity (it's nice to be able to compare RSS of non-e10s
+            # with RSS + USS of e10s).
+            totals_rss = parse_about_memory.calculate_memory_report_values(
+                                            memory_report_path, node, 'Main')
+            totals_uss = parse_about_memory.calculate_memory_report_values(
+                                            memory_report_path, 'resident-unique')
+            value = totals_rss.values()[0] + \
+                    sum([v for k, v in totals_uss.iteritems() if not 'Main' in k])
+
+        subtest = {
+            'name': checkpoint['name'],
+            'value': value,
+            'lowerIsBetter': True,
+            'units': 'bytes'
+        }
+        suite['subtests'].append(subtest);
+        total += math.log(subtest['value'])
+
+    # Add the geometric mean. For more details on the calculation see:
+    #   https://en.wikipedia.org/wiki/Geometric_mean#Relationship_with_arithmetic_mean_of_logarithms
+    suite['value'] = math.exp(total / len(CHECKPOINTS))
+
+    return suite
+
+
+def create_perf_data(data_path):
+    """
+    Builds up a performance data blob suitable for submitting to perfherder.
+    """
+    perf_blob = {
+        'framework': { 'name': 'awsy' },
+        'suites': []
+    }
+
+    for suite in PERF_SUITES:
+        perf_blob['suites'].append(create_suite(suite['name'], suite['node'], data_path))
+
+    return perf_blob
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    if not args:
+        print "Usage: process_perf_data.py data_path"
+        sys.exit(1)
+
+    # Determine which revisions we need to process.
+    data_path = args[0]
+    perf_blob = create_perf_data(data_path)
+    print "PERFHERDER_DATA: %s" % json.dumps(perf_blob)
+
+    sys.exit(0)
new file mode 100644
--- /dev/null
+++ b/testing/awsy/awsy/test_memory_usage.py
@@ -0,0 +1,326 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import sys
+import time
+import shutil
+
+from marionette_harness import MarionetteTestCase
+from marionette_driver import Actions
+from marionette_driver.errors import JavascriptException, ScriptTimeoutException
+import mozlog.structured
+from marionette_driver.keys import Keys
+
+from awsy import TEST_SITES_TEMPLATES, ITERATIONS, PER_TAB_PAUSE, SETTLE_WAIT_TIME, MAX_TABS
+from awsy import process_perf_data, webservers
+
+
+class TestMemoryUsage(MarionetteTestCase):
+    """Provides a test that collects memory usage at various checkpoints:
+      - "Start" - Just after startup
+      - "StartSettled" - After an additional wait time
+      - "TabsOpen" - After opening all provided URLs
+      - "TabsOpenSettled" - After an additional wait time
+      - "TabsOpenForceGC" - After forcibly invoking garbage collection
+      - "TabsClosed" - After closing all tabs
+      - "TabsClosedSettled" - After an additional wait time
+      - "TabsClosedForceGC" - After forcibly invoking garbage collection
+    """
+
+    def setUp(self):
+        MarionetteTestCase.setUp(self)
+        self.logger = mozlog.structured.structuredlog.get_default_logger()
+        self.logger.info("setting up!")
+
+        self.marionette.set_context('chrome')
+
+        self._webroot_dir = os.path.join(os.getcwd(), 'page_load_test')
+        if not os.path.exists(self._webroot_dir):
+            os.mkdir(self._webroot_dir)
+        self._webservers = webservers.WebServers("localhost",
+                                                 8001,
+                                                 os.getcwd(),
+                                                 100)
+        self._webservers.start()
+        test_sites = []
+
+        with open(os.path.join(self._webroot_dir, 'tp5n', 'tp5n.manifest')) as fp:
+            urls = fp.readlines()
+        if urls:
+            urls = map(lambda x:x.replace('localhost', 'localhost:{}'), urls)
+            self._urls = urls
+        else:
+            urls = TEST_SITES_TEMPLATES
+
+        for url, server in zip(urls, self._webservers.servers):
+            test_sites.append(url.format(server.port))
+
+        self._urls = self.testvars.get("urls", test_sites)
+        self._pages_to_load = self.testvars.get("entities", len(self._urls))
+        self._iterations = self.testvars.get("iterations", ITERATIONS)
+        self._perTabPause = self.testvars.get("perTabPause", PER_TAB_PAUSE)
+        self._settleWaitTime = self.testvars.get("settleWaitTime", SETTLE_WAIT_TIME)
+        self._maxTabs = self.testvars.get("maxTabs", MAX_TABS)
+        self._resultsDir = os.path.join(os.getcwd(), "tests", "results")
+
+        self.logger.info("areweslimyet run by %d pages, %d iterations, %d perTabPause,%d settleWaitTime"
+                         % (self._pages_to_load, self._iterations, self._perTabPause, self._settleWaitTime))
+        self.reset_state()
+        self.logger.info("done setting up!")
+
+    def tearDown(self):
+        self.logger.info("tearing down!")
+        MarionetteTestCase.tearDown(self)
+        self.logger.info("tearing down webservers!")
+        self._webservers.stop()
+
+        self.logger.info("processing data in %s!" % self._resultsDir)
+        perf_blob = process_perf_data.create_perf_data(self._resultsDir)
+        self.logger.info("PERFHERDER_DATA: %s" % json.dumps(perf_blob))
+
+        perf_file = os.path.join(self._resultsDir, "perfherder_data.json")
+        with open(perf_file, 'w') as fp:
+            json.dump(perf_blob, fp)
+
+        # copy it to moz upload dir if set
+        if 'MOZ_UPLOAD_DIR' in os.environ:
+            for file in os.listdir(self._resultsDir):
+                file = os.path.join(self._resultsDir, file)
+                if os.path.isfile(file):
+                    shutil.copy2(file, os.environ["MOZ_UPLOAD_DIR"])
+
+        self.logger.info("done tearing down!")
+
+    def reset_state(self):
+        self._pages_loaded = 0
+
+        # Close all tabs except one
+        for x in range(len(self.marionette.window_handles) - 1):
+            self.logger.info("closing window")
+            self.marionette.execute_script("gBrowser.removeCurrentTab();")
+            time.sleep(0.25)
+
+        self._tabs = self.marionette.window_handles
+        self.marionette.switch_to_window(self._tabs[0])
+
+    def do_full_gc(self):
+        """Performs a full garbage collection cycle and returns when it is finished.
+
+        Returns True on success and False on failure.
+        """
+        # NB: we could do this w/ a signal or the fifo queue too
+        self.logger.info("starting gc...")
+        gc_script = """
+            const Cu = Components.utils;
+            const Cc = Components.classes;
+            const Ci = Components.interfaces;
+
+            Cu.import("resource://gre/modules/Services.jsm");
+            Services.obs.notifyObservers(null, "child-mmu-request", null);
+
+            let memMgrSvc = Cc["@mozilla.org/memory-reporter-manager;1"].getService(Ci.nsIMemoryReporterManager);
+            memMgrSvc.minimizeMemoryUsage(() => marionetteScriptFinished("gc done!"));
+            """
+        result = None
+        try:
+            result = self.marionette.execute_async_script(
+                gc_script, script_timeout=180000)
+        except JavascriptException, e:
+            self.logger.error("GC JavaScript error: %s" % e)
+        except ScriptTimeoutException:
+            self.logger.error("GC timed out")
+        except:
+            self.logger.error("Unexpected error: %s" % sys.exc_info()[0])
+        else:
+            self.logger.info(result)
+
+        return result is not None
+
+    def do_memory_report(self, checkpointName, iteration):
+        """Creates a memory report for all processes and and returns the
+        checkpoint.
+
+        This will block until all reports are retrieved or a timeout occurs.
+        Returns the checkpoint or None on error.
+
+        :param checkpointName: The name of the checkpoint.
+        """
+        self.logger.info("starting checkpoint %s..." % checkpointName)
+
+        checkpoint_file = "memory-report-%s-%d.json.gz" % (checkpointName, iteration)
+        checkpoint_path = os.path.join(self._resultsDir, checkpoint_file)
+
+        checkpoint_script = """
+            const Cc = Components.classes;
+            const Ci = Components.interfaces;
+
+            let dumper = Cc["@mozilla.org/memory-info-dumper;1"].getService(Ci.nsIMemoryInfoDumper);
+            dumper.dumpMemoryReportsToNamedFile(
+                "%s",
+                () => marionetteScriptFinished("memory report done!"),
+                null,
+                /* anonymize */ false);
+            """ % checkpoint_path
+
+        checkpoint = None
+        try:
+            finished = self.marionette.execute_async_script(
+                checkpoint_script, script_timeout=60000)
+            if finished:
+              checkpoint = checkpoint_path
+        except JavascriptException, e:
+            self.logger.error("Checkpoint JavaScript error: %s" % e)
+        except ScriptTimeoutException:
+            self.logger.error("Memory report timed out")
+        except:
+            self.logger.error("Unexpected error: %s" % sys.exc_info()[0])
+        else:
+            self.logger.info("checkpoint created, stored in %s" % checkpoint_path)
+
+        return checkpoint
+
+    def open_and_focus(self):
+        """Opens the next URL in the list and focuses on the tab it is opened in.
+
+        A new tab will be opened if |_maxTabs| has not been exceeded, otherwise
+        the URL will be loaded in the next tab.
+        """
+        page_to_load = self._urls[self._pages_loaded % len(self._urls)]
+        tabs_loaded = len(self._tabs)
+        is_new_tab = False
+
+        if tabs_loaded < self._maxTabs and tabs_loaded <= self._pages_loaded:
+            full_tab_list = self.marionette.window_handles
+
+            # Trigger opening a new tab by finding the new tab button and
+            # clicking it
+            newtab_button = (self.marionette.find_element('id', 'tabbrowser-tabs')
+                                            .find_element('anon attribute',
+                                                          {'anonid': 'tabs-newtab-button'}))
+            newtab_button.click()
+
+            self.wait_for_condition(lambda mn: len(
+                mn.window_handles) == tabs_loaded + 1)
+
+            # NB: The tab list isn't sorted, so we do a set diff to determine
+            #     which is the new tab
+            new_tab_list = self.marionette.window_handles
+            new_tabs = list(set(new_tab_list) - set(full_tab_list))
+
+            self._tabs.append(new_tabs[0])
+            tabs_loaded += 1
+
+            is_new_tab = True
+
+        tab_idx = self._pages_loaded % self._maxTabs
+
+        tab = self._tabs[tab_idx]
+
+        # Tell marionette which tab we're on
+        # NB: As a work-around for an e10s marionette bug, only select the tab
+        #     if we're really switching tabs.
+        if tabs_loaded > 1:
+            self.logger.info("switching to tab")
+            self.marionette.switch_to_window(tab)
+            self.logger.info("switched to tab")
+
+        with self.marionette.using_context('content'):
+            self.logger.info("loading %s" % page_to_load)
+            self.marionette.navigate(page_to_load)
+            self.logger.info("loaded!")
+
+        # On e10s the tab handle can change after actually loading content
+        if is_new_tab:
+            # First build a set up w/o the current tab
+            old_tabs = set(self._tabs)
+            old_tabs.remove(tab)
+            # Perform a set diff to get the (possibly) new handle
+            [new_tab] = set(self.marionette.window_handles) - old_tabs
+            # Update the tab list at the current index to preserve the tab
+            # ordering
+            self._tabs[tab_idx] = new_tab
+
+        # give the page time to settle
+        time.sleep(self._perTabPause)
+
+        self._pages_loaded += 1
+
+    def signal_user_active(self):
+        """Signal to the browser that the user is active.
+
+        Normally when being driven by marionette the browser thinks the
+        user is inactive the whole time because user activity is
+        detected by looking at key and mouse events.
+
+        This would be a problem for this test because user inactivity is
+        used to schedule some GCs (in particular shrinking GCs), so it
+        would make this unrepresentative of real use.
+
+        Instead we manually cause some inconsequential activity (a press
+        and release of the shift key) to make the browser think the user
+        is active.  Then when we sleep to allow things to settle the
+        browser will see the user as becoming inactive and trigger
+        appropriate GCs, as would have happened in real use.
+        """
+        action = Actions(self.marionette)
+        action.key_down(Keys.SHIFT)
+        action.key_up(Keys.SHIFT)
+        action.perform()
+
+    def test_open_tabs(self):
+        """Marionette test entry that returns an array of checkoint arrays.
+
+        This will generate a set of checkpoints for each iteration requested.
+        Upon succesful completion the results will be stored in
+        |self.testvars["results"]| and accessible to the test runner via the
+        |testvars| object it passed in.
+        """
+        # setup the results array
+        results = [[] for _ in range(self._iterations)]
+
+        def create_checkpoint(name, iteration):
+            checkpoint = self.do_memory_report(name, iteration)
+            self.assertIsNotNone(checkpoint, "Checkpoint was recorded")
+            results[iteration].append(checkpoint)
+
+        # The first iteration gets Start and StartSettled entries before
+        # opening tabs
+        create_checkpoint("Start", 0)
+        time.sleep(self._settleWaitTime)
+        create_checkpoint("StartSettled", 0)
+
+        for itr in range(self._iterations):
+            for _ in range(self._pages_to_load):
+                self.open_and_focus()
+                self.signal_user_active()
+
+            create_checkpoint("TabsOpen", itr)
+            time.sleep(self._settleWaitTime)
+            create_checkpoint("TabsOpenSettled", itr)
+            self.assertTrue(self.do_full_gc())
+            create_checkpoint("TabsOpenForceGC", itr)
+
+            # Close all tabs
+            self.reset_state()
+
+            self.logger.info("switching to first window")
+            self.marionette.switch_to_window(self._tabs[0])
+            self.logger.info("switched to first window")
+            with self.marionette.using_context('content'):
+                self.logger.info("navigating to about:blank")
+                self.marionette.navigate("about:blank")
+                self.logger.info("navigated to about:blank")
+            self.signal_user_active()
+
+            create_checkpoint("TabsClosed", itr)
+            time.sleep(self._settleWaitTime)
+            create_checkpoint("TabsClosedSettled", itr)
+            self.assertTrue(self.do_full_gc(), "GC ran")
+            create_checkpoint("TabsClosedForceGC", itr)
+
+        # TODO(ER): Temporary hack until bug 1121139 lands
+        self.logger.info("setting results")
+        self.testvars["results"] = results
new file mode 100644
--- /dev/null
+++ b/testing/awsy/awsy/webservers.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+# mozhttpd web server.
+
+import argparse
+import os
+import socket
+
+import mozhttpd
+
+
+# directory of this file
+here = os.path.dirname(os.path.realpath(__file__))
+
+
+class WebServers(object):
+    def __init__(self, host, port, docroot, count):
+        self.host = host
+        self.port = port
+        self.docroot = docroot
+        self.count = count
+        self.servers = []
+
+    def start(self):
+        self.stop()
+        self.servers = []
+        port = self.port
+        while len(self.servers) < self.count:
+            self.servers.append(
+                mozhttpd.MozHttpd(host=self.host,
+                                  port=port,
+                                  docroot=self.docroot))
+            try:
+                self.servers[-1].start()
+            except socket.error, error:
+                if isinstance(error, socket.error):
+                    if error.errno == 98:
+                        print "port %d is in use." % port
+                    else:
+                        print "port %d error %s" % (port, error)
+                elif isinstance(error, str):
+                    print "port %d error %s" % (port, error)
+                self.servers.pop()
+            except Exception, error:
+                print "port %d error %s" % (port, error)
+                self.servers.pop()
+
+            port += 1
+
+    def stop(self):
+        while len(self.servers) > 0:
+            server = self.servers.pop()
+            server.stop()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Start mozhttpd servers for use by areweslimyet.')
+
+    parser.add_argument('--port', type=int, default=8001,
+                        help='Starting port. Defaults to 8001. Web servers will be '
+                        'created for each port from the starting port to starting port '
+                        '+ count - 1.')
+    parser.add_argument('--count', type=int, default=100,
+                        help='Number of web servers to start. Defaults to 100.')
+    parser.add_argument('--host', type=str, default='localhost',
+                        help='Name of webserver host. Defaults to localhost.')
+
+    args = parser.parse_args()
+    web_servers = WebServers(args.host, args.port, "%s/html" % here, argc.count)
+    web_servers.start()
+
+if __name__ == "__main__":
+    main()
new file mode 100644
--- /dev/null
+++ b/testing/awsy/conf/prefs.json
@@ -0,0 +1,12 @@
+{
+    "network.proxy.socks": "localhost",
+    "network.proxy.socks_port": 90000,
+    "network.proxy.socks_remote_dns": true,
+    "network.proxy.type": 1,
+    "startup.homepage_welcome_url": "",
+    "startup.homepage_override_url": "",
+    "browser.newtab.url": "about:blank",
+    "browser.displayedE10SNotice": 1000,
+    "plugin.disable": true,
+    "image.mem.surfacecache.min_expiration_ms": 10000
+}
new file mode 100644
--- /dev/null
+++ b/testing/awsy/conf/testvars.json
@@ -0,0 +1,6 @@
+{
+  "entities": 100,
+  "iterations": 3,
+  "perTabPause": 10,
+  "settleWaitTime": 30
+}
new file mode 100644
--- /dev/null
+++ b/testing/awsy/requirements.txt
@@ -0,0 +1,1 @@
+marionette-harness==4.0
new file mode 100644
--- /dev/null
+++ b/testing/awsy/setup.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup, find_packages
+
+PACKAGE_NAME = 'awsy'
+PACKAGE_VERSION = '0.0.1'
+
+setup(
+    name=PACKAGE_NAME,
+    version=PACKAGE_VERSION,
+    description="AreWeSlimYet",
+    long_description="A memory testing framework for Firefox.",
+    author='Mozilla Automation and Testing Team',
+    author_email='tools@lists.mozilla.org',
+    license='MPL 1.1/GPL 2.0/LGPL 2.1',
+    packages=find_packages(),
+    zip_safe=False,
+    install_requires=["marionette_harness"],
+    classifiers=['Development Status :: 4 - Beta',
+                 'Environment :: Console',
+                 'Intended Audience :: Developers',
+                 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)',
+                 'Operating System :: OS Independent',
+                 'Topic :: Software Development :: Libraries :: Python Modules',
+                 ],
+)
new file mode 100644
--- /dev/null
+++ b/testing/awsy/tp5n-pageset.manifest
@@ -0,0 +1,10 @@
+[
+    {
+        "filename": "tp5n.zip",
+        "size": 81753769,
+        "digest": "7e74bc532d220fa2484f84bd7c2659da7d2ae3aa0bc225ba63e3db70dc0c0697503427209098afa85e235397c4ec58cd488cab7b3435e8079583d3994fff8326",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]
+