Bug 1167080 - HAR export automation. r=jryans
authorJan Odvarko <odvarko@gmail.com>
Wed, 17 Jun 2015 14:34:45 +0200
changeset 280427 c934be66965d921a2d341d04887b9baa82d8c1f7
parent 280426 cfdfff0c69632d142ba815411f4b95aa83679e2d
child 280428 543e22ba9046495300108cf4f452ad86da0db326
push id4932
push userjlund@mozilla.com
push dateMon, 10 Aug 2015 18:23:06 +0000
treeherdermozilla-beta@6dd5a4f5f745 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjryans
bugs1167080
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1167080 - HAR export automation. r=jryans
browser/app/profile/firefox.js
browser/devtools/framework/toolbox.js
browser/devtools/netmonitor/har/har-automation.js
browser/devtools/netmonitor/har/har-builder.js
browser/devtools/netmonitor/har/har-collector.js
browser/devtools/netmonitor/har/har-exporter.js
browser/devtools/netmonitor/har/moz.build
browser/devtools/netmonitor/har/toolbox-overlay.js
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1491,22 +1491,24 @@ pref("devtools.netmonitor.enabled", true
 // The default Network Monitor UI settings
 pref("devtools.netmonitor.panes-network-details-width", 550);
 pref("devtools.netmonitor.panes-network-details-height", 450);
 pref("devtools.netmonitor.statistics", true);
 pref("devtools.netmonitor.filters", "[\"all\"]");
 
 // The default Network monitor HAR export setting
 pref("devtools.netmonitor.har.defaultLogDir", "");
-pref("devtools.netmonitor.har.defaultFileName", "archive");
+pref("devtools.netmonitor.har.defaultFileName", "Archive %y-%m-%d %H-%M-%S");
 pref("devtools.netmonitor.har.jsonp", false);
 pref("devtools.netmonitor.har.jsonpCallback", "");
 pref("devtools.netmonitor.har.includeResponseBodies", true);
 pref("devtools.netmonitor.har.compress", false);
 pref("devtools.netmonitor.har.forceExport", false);
+pref("devtools.netmonitor.har.pageLoadedTimeout", 1500);
+pref("devtools.netmonitor.har.enableAutoExportToFile", false);
 
 // Enable the Tilt inspector
 pref("devtools.tilt.enabled", true);
 pref("devtools.tilt.intro_transition", true);
 pref("devtools.tilt.outro_transition", true);
 
 // Scratchpad settings
 // - recentFileMax: The maximum number of recently-opened files
--- a/browser/devtools/framework/toolbox.js
+++ b/browser/devtools/framework/toolbox.js
@@ -69,16 +69,19 @@ loader.lazyGetter(this, "screenManager",
 });
 loader.lazyGetter(this, "oscpu", () => {
   return Cc["@mozilla.org/network/protocol;1?name=http"]
            .getService(Ci.nsIHttpProtocolHandler).oscpu;
 });
 loader.lazyGetter(this, "is64Bit", () => {
   return Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULAppInfo).is64Bit;
 });
+loader.lazyGetter(this, "registerHarOverlay", () => {
+  return require("devtools/netmonitor/har/toolbox-overlay.js").register;
+});
 
 // White-list buttons that can be toggled to prevent adding prefs for
 // addons that have manually inserted toolbarbuttons into DOM.
 // (By default, supported target is only local tab)
 const ToolboxButtons = exports.ToolboxButtons = [
   { id: "command-button-pick",
     isTargetSupported: target =>
       target.getTrait("highlightable")
@@ -370,16 +373,17 @@ Toolbox.prototype = {
       this._buildDockButtons();
       this._buildOptions();
       this._buildTabs();
       this._applyCacheSettings();
       this._applyServiceWorkersTestingSettings();
       this._addKeysToWindow();
       this._addReloadKeys();
       this._addHostListeners();
+      this._registerOverlays();
       if (this._hostOptions && this._hostOptions.zoom === false) {
         this._disableZoomKeys();
       } else {
         this._addZoomKeys();
         this._loadInitialZoom();
       }
 
       this.webconsolePanel = this.doc.querySelector("#toolbox-panel-webconsole");
@@ -510,16 +514,20 @@ Toolbox.prototype = {
 
     // Split console uses keypress instead of command so the event can be
     // cancelled with stopPropagation on the keypress, and not preventDefault.
     this.doc.addEventListener("keypress", this._splitConsoleOnKeypress, false);
 
     this.doc.addEventListener("focus", this._onFocus, true);
   },
 
+  _registerOverlays: function() {
+    registerHarOverlay(this);
+  },
+
   _saveSplitConsoleHeight: function() {
     Services.prefs.setIntPref(SPLITCONSOLE_HEIGHT_PREF,
       this.webconsolePanel.height);
   },
 
   /**
    * Make sure that the console is showing up properly based on all the
    * possible conditions.
new file mode 100644
--- /dev/null
+++ b/browser/devtools/netmonitor/har/har-automation.js
@@ -0,0 +1,352 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { Cu, Ci, Cc } = require("chrome");
+const { Class } = require("sdk/core/heritage");
+const { devtools } = Cu.import("resource://gre/modules/devtools/Loader.jsm", {});
+const { defer, resolve } = require("sdk/core/promise");
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+
+Cu.import("resource://gre/modules/Task.jsm");
+
+loader.lazyRequireGetter(this, "HarCollector", "devtools/netmonitor/har/har-collector", true);
+loader.lazyRequireGetter(this, "HarExporter", "devtools/netmonitor/har/har-exporter", true);
+loader.lazyRequireGetter(this, "HarUtils", "devtools/netmonitor/har/har-utils", true);
+
+const prefDomain = "devtools.netmonitor.har.";
+
+// Helper tracer. Should be generic sharable by other modules (bug 1171927)
+const trace = {
+  log: function(...args) {
+  }
+}
+
+/**
+ * This object is responsible for automated HAR export. It listens
+ * for Network activity, collects all HTTP data and triggers HAR
+ * export when the page is loaded.
+ *
+ * The user needs to enable the following preference to make the
+ * auto-export work: devtools.netmonitor.har.enableAutoExportToFile
+ *
+ * HAR files are stored within directory that is specified in this
+ * preference: devtools.netmonitor.har.defaultLogDir
+ *
+ * If the default log directory preference isn't set the following
+ * directory is used by default: <profile>/har/logs
+ */
+var HarAutomation = Class({
+  // Initialization
+
+  initialize: function(toolbox) {
+    this.toolbox = toolbox;
+
+    let target = toolbox.target;
+    target.makeRemote().then(() => {
+      this.startMonitoring(target.client, target.form);
+    });
+  },
+
+  destroy: function() {
+    if (this.collector) {
+      this.collector.stop();
+    }
+
+    if (this.tabWatcher) {
+      this.tabWatcher.disconnect();
+    }
+  },
+
+  // Automation
+
+  startMonitoring: function(client, tabGrip, callback) {
+    if (!client) {
+      return;
+    }
+
+    if (!tabGrip) {
+      return;
+    }
+
+    this.debuggerClient = client;
+    this.tabClient = this.toolbox.target.activeTab;
+    this.webConsoleClient = this.toolbox.target.activeConsole;
+
+    let netPrefs = { "NetworkMonitor.saveRequestAndResponseBodies": true };
+    this.webConsoleClient.setPreferences(netPrefs, () => {
+      this.tabWatcher = new TabWatcher(this.toolbox, this);
+      this.tabWatcher.connect();
+    });
+  },
+
+  pageLoadBegin: function(aResponse) {
+    this.resetCollector();
+  },
+
+  resetCollector: function() {
+    if (this.collector) {
+      this.collector.stop();
+    }
+
+    // A page is about to be loaded, start collecting HTTP
+    // data from events sent from the backend.
+    this.collector = new HarCollector({
+      collector: this,
+      webConsoleClient: this.webConsoleClient,
+      debuggerClient: this.debuggerClient
+    });
+
+    this.collector.start();
+  },
+
+  /**
+   * A page is done loading, export collected data. Note that
+   * some requests for additional page resources might be pending,
+   * so export all after all has been properly received from the backend.
+   *
+   * This collector still works and collects any consequent HTTP
+   * traffic (e.g. XHRs) happening after the page is loaded and
+   * The additional traffic can be exported by executing
+   * triggerExport on this object.
+   */
+  pageLoadDone: function(aResponse) {
+    trace.log("HarAutomation.pageLoadDone; ", aResponse);
+
+    if (this.collector) {
+      this.collector.waitForHarLoad().then(collector => {
+        return this.autoExport();
+      });
+    }
+  },
+
+  autoExport: function() {
+    let autoExport = Services.prefs.getBoolPref(prefDomain +
+      "enableAutoExportToFile");
+
+    if (!autoExport) {
+      return resolve();
+    }
+
+    // Auto export to file is enabled, so save collected data
+    // into a file and use all the default options.
+    let data = {
+      fileName: Services.prefs.getCharPref(prefDomain + "defaultFileName"),
+    }
+
+    return this.executeExport(data);
+  },
+
+  // Public API
+
+  /**
+   * Export all what is currently collected.
+   */
+  triggerExport: function(data) {
+    if (!data.fileName) {
+      data.fileName = Services.prefs.getCharPref(prefDomain +
+        "defaultFileName");
+    }
+
+    this.executeExport(data);
+  },
+
+  /**
+   * Clear currently collected data.
+   */
+  clear: function() {
+    this.resetCollector();
+  },
+
+  // HAR Export
+
+  /**
+   * Execute HAR export. This method fetches all data from the
+   * Network panel (asynchronously) and saves it into a file.
+   */
+  executeExport: function(data) {
+    let items = this.collector.getItems();
+    let form = this.toolbox.target.form;
+    let title = form.title || form.url;
+
+    let options = {
+      getString: this.getString.bind(this),
+      view: this,
+      items: items,
+    }
+
+    options.defaultFileName = data.fileName;
+    options.compress = data.compress;
+    options.title = data.title || title;
+    options.id = data.id;
+    options.jsonp = data.jsonp;
+    options.includeResponseBodies = data.includeResponseBodies;
+    options.jsonpCallback = data.jsonpCallback;
+    options.forceExport = data.forceExport;
+
+    trace.log("HarAutomation.executeExport; " + data.fileName, options);
+
+    return HarExporter.fetchHarData(options).then(jsonString => {
+      // Save the HAR file if the file name is provided.
+      if (jsonString && options.defaultFileName) {
+        let file = getDefaultTargetFile(options);
+        if (file) {
+          HarUtils.saveToFile(file, jsonString, options.compress);
+        }
+      }
+
+      return jsonString;
+    });
+  },
+
+  // Use WebConsoleClient.getString as soon as Bug 1171408 is fixed
+
+  /**
+   * Fetches the full text of a LongString.
+   *
+   * @param object | string aStringGrip
+   *        The long string grip containing the corresponding actor.
+   *        If you pass in a plain string (by accident or because you're lazy),
+   *        then a promise of the same string is simply returned.
+   * @return object Promise
+   *         A promise that is resolved when the full string contents
+   *         are available, or rejected if something goes wrong.
+   */
+  getString: function(aStringGrip) {
+    // Make sure this is a long string.
+    if (typeof aStringGrip != "object" || aStringGrip.type != "longString") {
+      return resolve(aStringGrip); // Go home string, you're drunk.
+    }
+    // Fetch the long string only once.
+    if (aStringGrip._fullText) {
+      return aStringGrip._fullText.promise;
+    }
+
+    let deferred = aStringGrip._fullText = defer();
+    let { actor, initial, length } = aStringGrip;
+    let longStringClient = this.webConsoleClient.longString(aStringGrip);
+
+    longStringClient.substring(initial.length, length, aResponse => {
+      if (aResponse.error) {
+        Cu.reportError(aResponse.error + ": " + aResponse.message);
+        deferred.reject(aResponse);
+        return;
+      }
+      deferred.resolve(initial + aResponse.substring);
+    });
+
+    return deferred.promise;
+  },
+
+  /**
+   * Extracts any urlencoded form data sections (e.g. "?foo=bar&baz=42") from a
+   * POST request.
+   *
+   * @param object aHeaders
+   *        The "requestHeaders".
+   * @param object aUploadHeaders
+   *        The "requestHeadersFromUploadStream".
+   * @param object aPostData
+   *        The "requestPostData".
+   * @return array
+   *        A promise that is resolved with the extracted form data.
+   */
+  _getFormDataSections: Task.async(function*(aHeaders, aUploadHeaders, aPostData) {
+    let formDataSections = [];
+
+    let { headers: requestHeaders } = aHeaders;
+    let { headers: payloadHeaders } = aUploadHeaders;
+    let allHeaders = [...payloadHeaders, ...requestHeaders];
+
+    let contentTypeHeader = allHeaders.find(e => e.name.toLowerCase() == "content-type");
+    let contentTypeLongString = contentTypeHeader ? contentTypeHeader.value : "";
+    let contentType = yield this.getString(contentTypeLongString);
+
+    if (contentType.includes("x-www-form-urlencoded")) {
+      let postDataLongString = aPostData.postData.text;
+      let postData = yield this.getString(postDataLongString);
+
+      for (let section of postData.split(/\r\n|\r|\n/)) {
+        // Before displaying it, make sure this section of the POST data
+        // isn't a line containing upload stream headers.
+        if (payloadHeaders.every(header => !section.startsWith(header.name))) {
+          formDataSections.push(section);
+        }
+      }
+    }
+
+    return formDataSections;
+  }),
+});
+
+// Helpers
+
+function TabWatcher(toolbox, listener) {
+  this.target = toolbox.target;
+  this.listener = listener;
+
+  this.onTabNavigated = this.onTabNavigated.bind(this);
+}
+
+TabWatcher.prototype = {
+  // Connection
+
+  connect: function() {
+    this.target.on("navigate", this.onTabNavigated);
+    this.target.on("will-navigate", this.onTabNavigated);
+  },
+
+  disconnect: function() {
+    if (!this.target) {
+      return;
+    }
+
+    this.target.off("navigate", this.onTabNavigated);
+    this.target.off("will-navigate", this.onTabNavigated);
+  },
+
+  // Event Handlers
+
+  /**
+   * Called for each location change in the monitored tab.
+   *
+   * @param string aType
+   *        Packet type.
+   * @param object aPacket
+   *        Packet received from the server.
+   */
+  onTabNavigated: function(aType, aPacket) {
+    switch (aType) {
+      case "will-navigate": {
+        this.listener.pageLoadBegin(aPacket);
+        break;
+      }
+      case "navigate": {
+        this.listener.pageLoadDone(aPacket);
+        break;
+      }
+    }
+  },
+};
+
+// Protocol Helpers
+
+/**
+ * Returns target file for exported HAR data.
+ */
+function getDefaultTargetFile(options) {
+  let path = options.defaultLogDir ||
+    Services.prefs.getCharPref("devtools.netmonitor.har.defaultLogDir");
+  let folder = HarUtils.getLocalDirectory(path);
+  let fileName = HarUtils.getHarFileName(options.defaultFileName,
+    options.jsonp, options.compress);
+
+  folder.append(fileName);
+  folder.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0666", 8));
+
+  return folder;
+}
+
+// Exports from this module
+exports.HarAutomation = HarAutomation;
--- a/browser/devtools/netmonitor/har/har-builder.js
+++ b/browser/devtools/netmonitor/har/har-builder.js
@@ -2,33 +2,28 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 
 const { Cu, Ci, Cc } = require("chrome");
 const { defer, all, resolve } = require("sdk/core/promise");
 const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 const { devtools } = Cu.import("resource://gre/modules/devtools/Loader.jsm", {});
-const { XPCOMUtils } = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
 
-XPCOMUtils.defineLazyGetter(this, "appInfo", function() {
+loader.lazyImporter(this, "ViewHelpers", "resource:///modules/devtools/ViewHelpers.jsm");
+loader.lazyRequireGetter(this, "NetworkHelper", "devtools/toolkit/webconsole/network-helper");
+
+loader.lazyGetter(this, "appInfo", () => {
   return Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULAppInfo);
 });
 
-XPCOMUtils.defineLazyModuleGetter(this, "ViewHelpers",
-  "resource:///modules/devtools/ViewHelpers.jsm");
-
-XPCOMUtils.defineLazyGetter(this, "L10N", function() {
+loader.lazyGetter(this, "L10N", () => {
   return new ViewHelpers.L10N("chrome://browser/locale/devtools/har.properties");
 });
 
-XPCOMUtils.defineLazyGetter(this, "NetworkHelper", function() {
-  return devtools.require("devtools/toolkit/webconsole/network-helper");
-});
-
 const HAR_VERSION = "1.1";
 
 /**
  * This object is responsible for building HAR file. See HAR spec:
  * https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html
  * http://www.softwareishard.com/blog/har-12-spec/
  *
  * @param {Object} options configuration object
new file mode 100644
--- /dev/null
+++ b/browser/devtools/netmonitor/har/har-collector.js
@@ -0,0 +1,456 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { Cu, Ci, Cc } = require("chrome");
+const { defer, all } = require("sdk/core/promise");
+const { setTimeout, clearTimeout } = require("sdk/timers");
+const { makeInfallible } = require("devtools/toolkit/DevToolsUtils.js");
+
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+
+// Helper tracer. Should be generic sharable by other modules (bug 1171927)
+const trace = {
+  log: function(...args) {
+  }
+}
+
+/**
+ * This object is responsible for collecting data related to all
+ * HTTP requests executed by the page (including inner iframes).
+ */
+function HarCollector(options) {
+  this.webConsoleClient = options.webConsoleClient;
+  this.debuggerClient = options.debuggerClient;
+  this.collector = options.collector;
+
+  this.onNetworkEvent = this.onNetworkEvent.bind(this);
+  this.onNetworkEventUpdate = this.onNetworkEventUpdate.bind(this);
+  this.onRequestHeaders = this.onRequestHeaders.bind(this);
+  this.onRequestCookies = this.onRequestCookies.bind(this);
+  this.onRequestPostData = this.onRequestPostData.bind(this);
+  this.onResponseHeaders = this.onResponseHeaders.bind(this);
+  this.onResponseCookies = this.onResponseCookies.bind(this);
+  this.onResponseContent = this.onResponseContent.bind(this);
+  this.onEventTimings = this.onEventTimings.bind(this);
+
+  this.onPageLoadTimeout = this.onPageLoadTimeout.bind(this);
+
+  this.clear();
+}
+
+HarCollector.prototype = {
+  // Connection
+
+  start: function() {
+    this.debuggerClient.addListener("networkEvent", this.onNetworkEvent);
+    this.debuggerClient.addListener("networkEventUpdate", this.onNetworkEventUpdate);
+  },
+
+  stop: function() {
+    this.debuggerClient.removeListener("networkEvent", this.onNetworkEvent);
+    this.debuggerClient.removeListener("networkEventUpdate", this.onNetworkEventUpdate);
+  },
+
+  clear: function() {
+    // Any pending requests events will be ignored (they turn
+    // into zombies, since not present in the files array).
+    this.files = new Map();
+    this.items = [];
+    this.firstRequestStart = -1;
+    this.lastRequestStart = -1;
+    this.requests = [];
+  },
+
+  waitForHarLoad: function() {
+    // There should be yet another timeout 'devtools.netmonitor.har.pageLoadTimeout'
+    // that should force export even if page isn't fully loaded.
+    let deferred = defer();
+    this.waitForResponses().then(() => {
+      trace.log("HarCollector.waitForHarLoad; DONE HAR loaded!");
+      deferred.resolve(this);
+    });
+
+    return deferred.promise;
+  },
+
+  waitForResponses: function() {
+    trace.log("HarCollector.waitForResponses; " + this.requests.length);
+
+    // All requests for additional data must be received to have complete
+    // HTTP info to generate the result HAR file. So, wait for all current
+    // promises. Note that new promises (requests) can be generated during the
+    // process of HTTP data collection.
+    return waitForAll(this.requests).then(() => {
+      // All responses are received from the backend now. We yet need to
+      // wait for a little while to see if a new request appears. If yes,
+      // lets's start gathering HTTP data again. If no, we can declare
+      // the page loaded.
+      // If some new requests appears in the meantime the promise will
+      // be rejected and we need to wait for responses all over again.
+      return this.waitForTimeout().then(() => {
+        // Page loaded!
+      }, () => {
+        trace.log("HarCollector.waitForResponses; NEW requests " +
+          "appeared during page timeout!");
+
+        // New requests executed, let's wait again.
+        return this.waitForResponses();
+      })
+    });
+  },
+
+  // Page Loaded Timeout
+
+  /**
+   * The page is loaded when there are no new requests within given period
+   * of time. The time is set in preferences:
+   * 'devtools.netmonitor.har.pageLoadedTimeout'
+   */
+  waitForTimeout: function() {
+    // The auto-export is not done if the timeout is set to zero (or less).
+    // This is useful in cases where the export is done manually through
+    // API exposed to the content.
+    let timeout = Services.prefs.getIntPref(
+      "devtools.netmonitor.har.pageLoadedTimeout");
+
+    trace.log("HarCollector.waitForTimeout; " + timeout);
+
+    this.pageLoadDeferred = defer();
+
+    if (timeout <= 0) {
+      this.pageLoadDeferred.resolve();
+      return this.pageLoadDeferred.promise;
+    }
+
+    this.pageLoadTimeout = setTimeout(this.onPageLoadTimeout, timeout);
+
+    return this.pageLoadDeferred.promise;
+  },
+
+  onPageLoadTimeout: function() {
+    trace.log("HarCollector.onPageLoadTimeout;");
+
+    // Ha, page has been loaded. Resolve the final timeout promise.
+    this.pageLoadDeferred.resolve();
+  },
+
+  resetPageLoadTimeout: function() {
+    // Remove the current timeout.
+    if (this.pageLoadTimeout) {
+      trace.log("HarCollector.resetPageLoadTimeout;");
+
+      clearTimeout(this.pageLoadTimeout);
+      this.pageLoadTimeout = null;
+    }
+
+    // Reject the current page load promise
+    if (this.pageLoadDeferred) {
+      this.pageLoadDeferred.reject();
+      this.pageLoadDeferred = null;
+    }
+  },
+
+  // Collected Data
+
+  getFile: function(actorId) {
+    return this.files.get(actorId);
+  },
+
+  getItems: function() {
+    return this.items;
+  },
+
+  // Event Handlers
+
+  onNetworkEvent: function(type, packet) {
+    // Skip events from different console actors.
+    if (packet.from != this.webConsoleClient.actor) {
+      return;
+    }
+
+    trace.log("HarCollector.onNetworkEvent; " + type, packet);
+
+    let { actor, startedDateTime, method, url, isXHR } = packet.eventActor;
+    let startTime = Date.parse(startedDateTime);
+
+    if (this.firstRequestStart == -1) {
+      this.firstRequestStart = startTime;
+    }
+
+    if (this.lastRequestEnd < startTime) {
+      this.lastRequestEnd = startTime;
+    }
+
+    let file = this.getFile(actor);
+    if (file) {
+      Cu.reportError("HarCollector.onNetworkEvent; ERROR " +
+        "existing file conflict!");
+      return;
+    }
+
+    file = {
+      startedDeltaMillis: startTime - this.firstRequestStart,
+      startedMillis: startTime,
+      method: method,
+      url: url,
+      isXHR: isXHR
+    };
+
+    this.files.set(actor, file);
+
+    // Mimic the Net panel data structure
+    this.items.push({
+      attachment: file
+    });
+  },
+
+  onNetworkEventUpdate: function(type, packet) {
+    let actor = packet.from;
+
+    // Skip events from unknown actors (not in the list).
+    // There could also be zombie requests received after the target is closed.
+    let file = this.getFile(packet.from);
+    if (!file) {
+      Cu.reportError("HarCollector.onNetworkEventUpdate; ERROR " +
+        "Unknown event actor: " + type, packet);
+      return;
+    }
+
+    trace.log("HarCollector.onNetworkEventUpdate; " +
+      packet.updateType, packet);
+
+    let includeResponseBodies = Services.prefs.getBoolPref(
+      "devtools.netmonitor.har.includeResponseBodies");
+
+    let request;
+    switch (packet.updateType) {
+      case "requestHeaders":
+        request = this.getData(actor, "getRequestHeaders", this.onRequestHeaders);
+        break;
+      case "requestCookies":
+        request = this.getData(actor, "getRequestCookies", this.onRequestCookies);
+        break;
+      case "requestPostData":
+        request = this.getData(actor, "getRequestPostData", this.onRequestPostData);
+        break;
+      case "responseHeaders":
+        request = this.getData(actor, "getResponseHeaders", this.onResponseHeaders);
+        break;
+      case "responseCookies":
+        request = this.getData(actor, "getResponseCookies", this.onResponseCookies);
+        break;
+      case "responseStart":
+        file.httpVersion = packet.response.httpVersion;
+        file.status = packet.response.status;
+        file.statusText = packet.response.statusText;
+        break;
+      case "responseContent":
+        file.contentSize = packet.contentSize;
+        file.mimeType = packet.mimeType;
+        file.transferredSize = packet.transferredSize;
+
+        if (includeResponseBodies) {
+          request = this.getData(actor, "getResponseContent", this.onResponseContent);
+        }
+        break;
+      case "eventTimings":
+        request = this.getData(actor, "getEventTimings", this.onEventTimings);
+        break;
+    }
+
+    if (request) {
+      this.requests.push(request);
+    }
+
+    this.resetPageLoadTimeout();
+  },
+
+  getData: function(actor, method, callback) {
+    let deferred = defer();
+
+    if (!this.webConsoleClient[method]) {
+      Cu.reportError("HarCollector.getData; ERROR " +
+        "Unknown method!");
+      return;
+    }
+
+    let file = this.getFile(actor);
+
+    trace.log("HarCollector.getData; REQUEST " + method +
+      ", " + file.url, file);
+
+    this.webConsoleClient[method](actor, response => {
+      trace.log("HarCollector.getData; RESPONSE " + method +
+        ", " + file.url, response);
+
+      callback(response);
+      deferred.resolve(response);
+    });
+
+    return deferred.promise;
+  },
+
+  /**
+   * Handles additional information received for a "requestHeaders" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onRequestHeaders: function(response) {
+    let file = this.getFile(response.from);
+    file.requestHeaders = response;
+
+    this.getLongHeaders(response.headers);
+  },
+
+  /**
+   * Handles additional information received for a "requestCookies" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onRequestCookies: function(response) {
+    let file = this.getFile(response.from);
+    file.requestCookies = response;
+
+    this.getLongHeaders(response.cookies);
+  },
+
+  /**
+   * Handles additional information received for a "requestPostData" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onRequestPostData: function(response) {
+    trace.log("HarCollector.onRequestPostData;", response);
+
+    let file = this.getFile(response.from);
+    file.requestPostData = response;
+
+    // Resolve long string
+    let text = response.postData.text;
+    if (typeof text == "object") {
+      this.getString(text).then(value => {
+          response.postData.text = value;
+      })
+    }
+  },
+
+  /**
+   * Handles additional information received for a "responseHeaders" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onResponseHeaders: function(response) {
+    let file = this.getFile(response.from);
+    file.responseHeaders = response;
+
+    this.getLongHeaders(response.headers);
+  },
+
+  /**
+   * Handles additional information received for a "responseCookies" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onResponseCookies: function(response) {
+    let file = this.getFile(response.from);
+    file.responseCookies = response;
+
+    this.getLongHeaders(response.cookies);
+  },
+
+  /**
+   * Handles additional information received for a "responseContent" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onResponseContent: function(response) {
+    let file = this.getFile(response.from);
+    file.mimeType = "text/plain";
+    file.responseContent = response;
+
+    // Resolve long string
+    let text = response.content.text;
+    if (typeof text == "object") {
+      this.getString(text).then(value => {
+        response.content.text = value;
+      })
+    }
+  },
+
+  /**
+   * Handles additional information received for a "eventTimings" packet.
+   *
+   * @param object response
+   *        The message received from the server.
+   */
+  onEventTimings: function(response) {
+    let file = this.getFile(response.from);
+    file.eventTimings = response;
+
+    let totalTime = response.totalTime;
+    file.totalTime = totalTime;
+    file.endedMillis = file.startedMillis + totalTime;
+  },
+
+  // Helpers
+
+  getLongHeaders: makeInfallible(function(headers) {
+    for (let header of headers) {
+      if (typeof header.value == "object") {
+        this.getString(header.value).then(value => {
+          header.value = value;
+        });
+      }
+    }
+  }),
+
+  /**
+   * Fetches the full text of a LongString.
+   *
+   * @param object | string aStringGrip
+   *        The long string grip containing the corresponding actor.
+   *        If you pass in a plain string (by accident or because you're lazy),
+   *        then a promise of the same string is simply returned.
+   * @return object Promise
+   *         A promise that is resolved when the full string contents
+   *         are available, or rejected if something goes wrong.
+   */
+  getString: function(stringGrip) {
+    let promise = this.collector.getString(stringGrip);
+    this.requests.push(promise);
+    return promise;
+  }
+};
+
+// Helpers
+
+/**
+ * Helper function that allows to wait for array of promises. It is
+ * possible to dynamically add new promises in the provided array.
+ * The function will wait even for the newly added promises.
+ * (this isn't possible with the default Promise.all);
+ */
+function waitForAll(promises) {
+  // Remove all from the original array and get clone of it.
+  let clone = promises.splice(0, promises.length);
+
+  // Wait for all promises in the given array.
+  return all(clone).then(() => {
+    // If there are new promises (in the original array)
+    // to wait for - chain them!
+    if (promises.length) {
+      return waitForAll(promises);
+    }
+  });
+}
+
+// Exports from this module
+exports.HarCollector = HarCollector;
--- a/browser/devtools/netmonitor/har/har-exporter.js
+++ b/browser/devtools/netmonitor/har/har-exporter.js
@@ -11,16 +11,22 @@ const { HarBuilder } = require("./har-bu
 
 XPCOMUtils.defineLazyGetter(this, "clipboardHelper", function() {
   return Cc["@mozilla.org/widget/clipboardhelper;1"].
     getService(Ci.nsIClipboardHelper);
 });
 
 var uid = 1;
 
+// Helper tracer. Should be generic sharable by other modules (bug 1171927)
+const trace = {
+  log: function(...args) {
+  }
+}
+
 /**
  * This object represents the main public API designed to access
  * Network export logic. Clients, such as the Network panel itself,
  * should use this API to export collected HTTP data from the panel.
  */
 const HarExporter = {
   // Public API
 
@@ -72,16 +78,18 @@ const HarExporter = {
     // presses cancel.
     let file = HarUtils.getTargetFile(options.defaultFileName,
       options.jsonp, options.compress);
 
     if (!file) {
       return resolve();
     }
 
+    trace.log("HarExporter.save; " + options.defaultFileName, options);
+
     return this.fetchHarData(options).then(jsonString => {
       if (!HarUtils.saveToFile(file, jsonString, options.compress)) {
         let msg = "Failed to save HAR file at: " + options.defaultFileName;
         Cu.reportError(msg);
       }
       return jsonString;
     });
   },
--- a/browser/devtools/netmonitor/har/moz.build
+++ b/browser/devtools/netmonitor/har/moz.build
@@ -1,12 +1,15 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 EXTRA_JS_MODULES.devtools.netmonitor.har += [
+    'har-automation.js',
     'har-builder.js',
+    'har-collector.js',
     'har-exporter.js',
     'har-utils.js',
+    'toolbox-overlay.js',
 ]
 
 BROWSER_CHROME_MANIFESTS += ['test/browser.ini']
new file mode 100644
--- /dev/null
+++ b/browser/devtools/netmonitor/har/toolbox-overlay.js
@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { Cu, Ci } = require("chrome");
+const { devtools } = Cu.import("resource://gre/modules/devtools/Loader.jsm", {});
+
+loader.lazyRequireGetter(this, "HarAutomation", "devtools/netmonitor/har/har-automation", true);
+
+// Map of all created overlays. There is always one instance of
+// an overlay per Toolbox instance (i.e. one per browser tab).
+const overlays = new WeakMap();
+
+/**
+ * This object is responsible for initialization and cleanup for HAR
+ * export feature. It represents an overlay for the Toolbox
+ * following the same life time by listening to its events.
+ *
+ * HAR APIs are designed for integration with tools (such as Selenium)
+ * that automates the browser. Primarily, it is for automating web apps
+ * and getting HAR file for every loaded page.
+ */
+function ToolboxOverlay(toolbox) {
+  this.toolbox = toolbox;
+
+  this.onInit = this.onInit.bind(this);
+  this.onDestroy = this.onDestroy.bind(this);
+
+  this.toolbox.on("ready", this.onInit);
+  this.toolbox.on("destroy", this.onDestroy);
+}
+
+ToolboxOverlay.prototype = {
+  /**
+   * Executed when the toolbox is ready.
+   */
+  onInit: function() {
+    let autoExport = Services.prefs.getBoolPref(
+      "devtools.netmonitor.har.enableAutoExportToFile");
+
+    if (!autoExport) {
+      return;
+    }
+
+    this.initAutomation();
+  },
+
+  /**
+   * Executed when the toolbox is destroyed.
+   */
+  onDestroy: function(eventId, toolbox) {
+    this.destroyAutomation();
+  },
+
+  // Automation
+
+  initAutomation: function() {
+    this.automation = new HarAutomation(this.toolbox);
+  },
+
+  destroyAutomation: function() {
+    if (this.automation) {
+      this.automation.destroy();
+    }
+  },
+};
+
+// Registration
+function register(toolbox) {
+  if (overlays.has(toolbox)) {
+    throw Error("Theere is an existing overlay for the toolbox");
+  }
+
+  // Instantiate an overlay for the toolbox.
+  let overlay = new ToolboxOverlay(toolbox);
+  overlays.set(toolbox, overlay);
+}
+
+// Exports from this module
+exports.register = register;