Bug 1362786 - (2) Run NMParser in worker r=kmag
authorDoug Thayer <dothayer@mozilla.com>
Fri, 21 Jul 2017 14:14:17 -0700
changeset 419923 71b419e1788dd017f3f3fe6bf76ae04115698a66
parent 419922 242a12eae20e469b3d8e6cfdc0ae34f9a1752c5d
child 419924 c1a15605e8a2662bd9c8a09435fd7f862d1f8776
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmag
bugs1362786
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1362786 - (2) Run NMParser in worker r=kmag Pulls out the NMParser work (parsing nm results and turning them into an ArrayBuffer'd map of addresses to symbols) into a worker. For OSX we will still need to do some work to run c++filt in the background, but this gets us most of the way there. Without a Subprocess.jsm usable from a worker, we'll have to bounce data back to the main thread in order to bounce it to the c++filt worker. MozReview-Commit-ID: LZi7J1qGpmh
browser/components/extensions/ParseBreakpadSymbols-worker.js
browser/components/extensions/ParseNMSymbols-worker.js
browser/components/extensions/ParseSymbols-worker.js
browser/components/extensions/ParseSymbols.jsm
browser/components/extensions/ext-geckoProfiler.js
browser/components/extensions/moz.build
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ParseBreakpadSymbols-worker.js
@@ -0,0 +1,58 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env worker */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm");
+importScripts("resource:///modules/ParseSymbols.jsm");
+
+async function fetchSymbolFile(url) {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(`got error status ${response.status}`);
+  }
+
+  return response.text();
+}
+
+function parse(text) {
+  const syms = new Map();
+
+  // Lines look like this:
+  //
+  // PUBLIC 3fc74 0 test_public_symbol
+  //
+  // FUNC 40330 8e 0 test_func_symbol
+  const symbolRegex = /\nPUBLIC ([0-9a-f]+) [0-9a-f]+ (.*)|\nFUNC ([0-9a-f]+) [0-9a-f]+ [0-9a-f]+ (.*)/g;
+
+  let match;
+  let approximateLength = 0;
+  while ((match = symbolRegex.exec(text))) {
+    const [address0, symbol0, address1, symbol1] = match.slice(1);
+    const address = parseInt(address0 || address1, 16);
+    const sym = (symbol0 || symbol1).trimRight();
+    syms.set(address, sym);
+    approximateLength += sym.length;
+  }
+
+  return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
+}
+
+onmessage = async e => {
+  try {
+    let text;
+    if (e.data.filepath) {
+      text = await OS.File.read(e.data.filepath, {encoding: "utf-8"});
+    } else if (e.data.url) {
+      text = await fetchSymbolFile(e.data.url);
+    }
+
+    const result = parse(text);
+    postMessage({result}, result.map(r => r.buffer));
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+  close();
+};
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ParseNMSymbols-worker.js
@@ -0,0 +1,85 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env worker */
+
+"use strict";
+
+importScripts("resource:///modules/ParseSymbols.jsm");
+
+class WorkerNMParser {
+  constructor() {
+    this._decoder = new TextDecoder();
+    this._addrToSymMap = new Map();
+    this._approximateLength = 0;
+  }
+
+  consume(arrayBuffer) {
+    const data = this._decoder.decode(arrayBuffer, {stream: true});
+    const lineRegex = /.*\n?/g;
+    const buffer = this._currentLine + data;
+
+    let match;
+    while ((match = lineRegex.exec(buffer))) {
+      let [line] = match;
+      if (line[line.length - 1] === "\n") {
+        this._processLine(line);
+      } else {
+        this._currentLine = line;
+        break;
+      }
+    }
+  }
+
+  finish() {
+    this._processLine(this._currentLine);
+    return {syms: this._addrToSymMap, approximateLength: this._approximateLength};
+  }
+
+  _processLine(line) {
+    // Example lines:
+    // 00000000028c9888 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 0000000003a33730 r mozilla::OggDemuxer::~OggDemuxer()::{lambda()#1}::operator()() const::__func__
+    // 0000000003a33930 r mozilla::VPXDecoder::Drain()::{lambda()#1}::operator()() const::__func__
+    //
+    // Some lines have the form
+    // <address> ' ' <letter> ' ' <symbol>
+    // and some have the form
+    // <address> ' ' <symbol>
+    // The letter has a meaning, but we ignore it.
+
+    const regex = /([^ ]+) (?:. )?(.*)/;
+    let match = regex.exec(line);
+    if (match) {
+      const [, address, symbol] = match;
+      this._addrToSymMap.set(parseInt(address, 16), symbol);
+      this._approximateLength += symbol.length;
+    }
+  }
+}
+
+let nmParser;
+onmessage = async e => {
+  try {
+    if (!nmParser) {
+      nmParser = new WorkerNMParser();
+    }
+    if (e.data.finish) {
+      const {syms, approximateLength} = nmParser.finish();
+      let result;
+      if (e.data.isDarwin) {
+        result = ParseSymbols.convertSymsMapToDemanglerFormat(syms);
+      } else {
+        result = ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
+      }
+
+      postMessage({result}, result.map(r => r.buffer));
+      close();
+    } else {
+      nmParser.consume(e.data.buffer);
+    }
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+};
deleted file mode 100644
--- a/browser/components/extensions/ParseSymbols-worker.js
+++ /dev/null
@@ -1,58 +0,0 @@
-/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
-/* vim: set sts=2 sw=2 et tw=80: */
-/* eslint-env worker */
-
-"use strict";
-
-importScripts("resource://gre/modules/osfile.jsm");
-importScripts("resource:///modules/ParseSymbols.jsm");
-
-async function fetchSymbolFile(url) {
-  const response = await fetch(url);
-
-  if (!response.ok) {
-    throw new Error(`got error status ${response.status}`);
-  }
-
-  return response.text();
-}
-
-function parse(text) {
-  const syms = new Map();
-
-  // Lines look like this:
-  //
-  // PUBLIC 3fc74 0 test_public_symbol
-  //
-  // FUNC 40330 8e 0 test_func_symbol
-  const symbolRegex = /\nPUBLIC ([0-9a-f]+) [0-9a-f]+ (.*)|\nFUNC ([0-9a-f]+) [0-9a-f]+ [0-9a-f]+ (.*)/g;
-
-  let match;
-  let approximateLength = 0;
-  while ((match = symbolRegex.exec(text))) {
-    const [address0, symbol0, address1, symbol1] = match.slice(1);
-    const address = parseInt(address0 || address1, 16);
-    const sym = (symbol0 || symbol1).trimRight();
-    syms.set(address, sym);
-    approximateLength += sym.length;
-  }
-
-  return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
-}
-
-onmessage = async e => {
-  try {
-    let text;
-    if (e.data.filepath) {
-      text = await OS.File.read(e.data.filepath, {encoding: "utf-8"});
-    } else if (e.data.url) {
-      text = await fetchSymbolFile(e.data.url);
-    }
-
-    const result = parse(text);
-    postMessage({result}, result.map(r => r.buffer));
-  } catch (error) {
-    postMessage({error: error.toString()});
-  }
-  close();
-};
--- a/browser/components/extensions/ParseSymbols.jsm
+++ b/browser/components/extensions/ParseSymbols.jsm
@@ -39,11 +39,23 @@ function convertSymsMapToExpectedSymForm
 
   const symsArray = addresses.map(addr => syms.get(addr));
   const {index, buffer} =
     convertStringArrayToUint8BufferWithIndex(symsArray, approximateSymLength);
 
   return [new Uint32Array(addresses), index, buffer];
 }
 
+function convertSymsMapToDemanglerFormat(syms) {
+  const addresses = Array.from(syms.keys());
+  addresses.sort((a, b) => a - b);
+
+  const symsArray = addresses.map(addr => syms.get(addr));
+  const textEncoder = new TextEncoder();
+  const buffer = textEncoder.encode(symsArray.join("\n"));
+
+  return [new Uint32Array(addresses), buffer];
+}
+
 var ParseSymbols = {
   convertSymsMapToExpectedSymFormat,
+  convertSymsMapToDemanglerFormat,
 };
--- a/browser/components/extensions/ext-geckoProfiler.js
+++ b/browser/components/extensions/ext-geckoProfiler.js
@@ -1,105 +1,83 @@
 /* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
 /* vim: set sts=2 sw=2 et tw=80: */
 "use strict";
 
 // The ext-* files are imported into the same scopes.
 /* import-globals-from ../../../toolkit/components/extensions/ext-toolkit.js */
 
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.importGlobalProperties(["fetch", "TextEncoder"]);
+Cu.importGlobalProperties(["fetch", "TextEncoder", "TextDecoder"]);
 
 XPCOMUtils.defineLazyModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ParseSymbols", "resource:///modules/ParseSymbols.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Subprocess", "resource://gre/modules/Subprocess.jsm");
 
 const PREF_ASYNC_STACK = "javascript.options.asyncstack";
 const PREF_SYMBOLS_URL = "extensions.geckoProfiler.symbols.url";
 const PREF_GET_SYMBOL_RULES = "extensions.geckoProfiler.getSymbolRules";
 
 const ASYNC_STACKS_ENABLED = Services.prefs.getBoolPref(PREF_ASYNC_STACK, false);
 
 var {
   ExtensionError,
 } = ExtensionUtils;
 
 const parseSym = data => {
-  const worker = new ChromeWorker("resource://app/modules/ParseSymbols-worker.js");
+  const worker = new ChromeWorker("resource://app/modules/ParseBreakpadSymbols-worker.js");
   const promise = new Promise((resolve, reject) => {
     worker.onmessage = (e) => {
       if (e.data.error) {
         reject(e.data.error);
       } else {
         resolve(e.data.result);
       }
     };
   });
   worker.postMessage(data);
   return promise;
 };
 
 class NMParser {
   constructor() {
-    this._addrToSymMap = new Map();
-    this._approximateLength = 0;
+    this._worker = new ChromeWorker("resource://app/modules/ParseNMSymbols-worker.js");
   }
 
-  consume(data) {
-    const lineRegex = /.*\n?/g;
-    const buffer = this._currentLine + data;
-
-    let match;
-    while ((match = lineRegex.exec(buffer))) {
-      let [line] = match;
-      if (line[line.length - 1] === "\n") {
-        this._processLine(line);
-      } else {
-        this._currentLine = line;
-        break;
-      }
-    }
+  consume(buffer) {
+    this._worker.postMessage({buffer}, [buffer]);
   }
 
   finish() {
-    this._processLine(this._currentLine);
-    return {syms: this._addrToSymMap, approximateLength: this._approximateLength};
-  }
-
-  _processLine(line) {
-    // Example lines:
-    // 00000000028c9888 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 0000000003a33730 r mozilla::OggDemuxer::~OggDemuxer()::{lambda()#1}::operator()() const::__func__
-    // 0000000003a33930 r mozilla::VPXDecoder::Drain()::{lambda()#1}::operator()() const::__func__
-    //
-    // Some lines have the form
-    // <address> ' ' <letter> ' ' <symbol>
-    // and some have the form
-    // <address> ' ' <symbol>
-    // The letter has a meaning, but we ignore it.
-
-    const regex = /([^ ]+) (?:. )?(.*)/;
-    let match = regex.exec(line);
-    if (match) {
-      const [, address, symbol] = match;
-      this._addrToSymMap.set(parseInt(address, 16), symbol);
-      this._approximateLength += symbol.length;
-    }
+    const promise = new Promise((resolve, reject) => {
+      this._worker.onmessage = (e) => {
+        if (e.data.error) {
+          reject(e.data.error);
+        } else {
+          resolve(e.data.result);
+        }
+      };
+    });
+    this._worker.postMessage({
+      finish: true,
+      isDarwin: Services.appinfo.OS === "Darwin",
+    });
+    return promise;
   }
 }
 
 class CppFiltParser {
   constructor(length) {
+    this._decoder = new TextDecoder();
     this._index = 0;
     this._results = new Array(length);
   }
 
-  consume(data) {
+  consume(arrayBuffer) {
+    const data = this._decoder.decode(arrayBuffer, {stream: true});
     const lineRegex = /.*\n?/g;
     const buffer = this._currentLine + data;
 
     let match;
     while ((match = lineRegex.exec(buffer))) {
       let [line] = match;
       if (line[line.length - 1] === "\n") {
         this._processLine(line);
@@ -117,17 +95,17 @@ class CppFiltParser {
 
   _processLine(line) {
     this._results[this._index++] = line.trimRight();
   }
 }
 
 const readAllData = async function(pipe, processData) {
   let data;
-  while ((data = await pipe.readString())) {
+  while ((data = await pipe.read()) && data.byteLength) {
     processData(data);
   }
 };
 
 const spawnProcess = async function(name, cmdArgs, processData, stdin = null) {
   const opts = {
     command: await Subprocess.pathSearch(name),
     arguments: cmdArgs,
@@ -152,29 +130,32 @@ const getSymbolsFromNM = async function(
   } else {
     // Mac's `nm` doesn't support the demangle option, so we have to
     // post-process the symbols with c++filt.
     args.unshift("--demangle");
   }
 
   await spawnProcess("nm", args, data => parser.consume(data));
   await spawnProcess("nm", ["-D", ...args], data => parser.consume(data));
-  let {syms, approximateLength} = parser.finish();
+  let result = await parser.finish();
+  if (Services.appinfo.OS !== "Darwin") {
+    return result;
+  }
 
-  if (Services.appinfo.OS === "Darwin") {
-    const keys = Array.from(syms.keys());
-    const values = keys.map(k => syms.get(k));
-    const demangler = new CppFiltParser(keys.length);
-    await spawnProcess("c++filt", [], data => demangler.consume(data), values.join("\n"));
-    const newSymbols = demangler.finish();
-    approximateLength = 0;
-    for (let [i, symbol] of newSymbols.entries()) {
-      approximateLength += symbol.length;
-      syms.set(keys[i], symbol);
-    }
+  const syms = new Map();
+  const [addresses, symbolsJoinedBuffer] = result;
+  const decoder = new TextDecoder();
+  const symbolsJoined = decoder.decode(symbolsJoinedBuffer);
+  const demangler = new CppFiltParser(addresses.length);
+  await spawnProcess("c++filt", [], data => demangler.consume(data), symbolsJoined);
+  const newSymbols = demangler.finish();
+  let approximateLength = 0;
+  for (let [i, symbol] of newSymbols.entries()) {
+    approximateLength += symbol.length;
+    syms.set(addresses[i], symbol);
   }
 
   return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
 };
 
 const pathComponentsForSymbolFile = (debugName, breakpadId) => {
   const symName = debugName.replace(/(\.pdb)?$/, ".sym");
   return [debugName, breakpadId, symName];
--- a/browser/components/extensions/moz.build
+++ b/browser/components/extensions/moz.build
@@ -10,17 +10,18 @@ with Files("**"):
 JAR_MANIFESTS += ['jar.mn']
 
 EXTRA_COMPONENTS += [
     'extensions-browser.manifest',
 ]
 
 EXTRA_JS_MODULES += [
     'ExtensionPopups.jsm',
-    'ParseSymbols-worker.js',
+    'ParseBreakpadSymbols-worker.js',
+    'ParseNMSymbols-worker.js',
     'ParseSymbols.jsm',
 ]
 
 DIRS += ['schemas']
 
 BROWSER_CHROME_MANIFESTS += [
     'test/browser/browser-remote.ini',
     'test/browser/browser.ini',