Bug 1509549 - Add a ProfilerGetSymbols module which can dump symbols with the help of a dynamically-loaded WebAssembly module. r=kmag
authorMarkus Stange <mstange@themasta.com>
Thu, 07 Feb 2019 19:36:43 +0000
changeset 458213 a291f111e24e9fa36e03823040fd3350a58067a3
parent 458212 54eafeca2123c137c52cbfe54df3dbdd85617c5b
child 458214 ba7bb5f95f438cca5fdef2091034e1d0312b09bc
push id111775
push useropoprus@mozilla.com
push dateFri, 08 Feb 2019 10:16:11 +0000
treeherdermozilla-inbound@fe34a6921349 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmag
bugs1509549
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509549 - Add a ProfilerGetSymbols module which can dump symbols with the help of a dynamically-loaded WebAssembly module. r=kmag The module can dump ELF binaries, Mach-O binaries, and pdb files. So it works for all our supported platforms. The module is currently hosted on https://zealous-rosalind-a98ce8.netlify.com/ , which is a netlify server that serves files from the following repo: https://github.com/mstange/profiler-assets To make all of this look a bit more official, I'm planning on doing two things: - Move the github repo under the devtools-html organization - Get a firefox.com subdomain such as profiler-assets.firefox.com for hosting Depends on D13004 Differential Revision: https://phabricator.services.mozilla.com/D13005
.eslintignore
browser/components/extensions/ProfilerGetSymbols-worker.js
browser/components/extensions/ProfilerGetSymbols.jsm
browser/components/extensions/moz.build
browser/components/extensions/profiler_get_symbols.js
tools/lint/eslint/modules.json
--- a/.eslintignore
+++ b/.eslintignore
@@ -60,16 +60,18 @@ browser/base/content/test/general/gZipOf
 browser/base/content/test/urlbar/file_blank_but_not_blank.html
 # Test files that are really json not js, and don't need to be linted.
 browser/components/sessionstore/test/unit/data/sessionstore_valid.js
 browser/components/sessionstore/test/unit/data/sessionstore_invalid.js
 # This file is split into two in order to keep it as a valid json file
 # for documentation purposes (policies.json) but to be accessed by the
 # code as a .jsm (schema.jsm)
 browser/components/enterprisepolicies/schemas/schema.jsm
+# Ignore generated code from wasm-bindgen
+browser/components/extensions/profiler_get_symbols.js
 # generated & special files in cld2
 browser/components/translation/cld2/**
 # Screenshots is imported as a system add-on and has
 # their own lint rules currently.
 browser/extensions/screenshots/**
 browser/extensions/pdfjs/content/build**
 browser/extensions/pdfjs/content/web**
 # generated or library files in pocket
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ProfilerGetSymbols-worker.js
@@ -0,0 +1,128 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env mozilla/chrome-worker */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm",
+              "resource://app/modules/profiler_get_symbols.js");
+
+// This worker uses the wasm module that was generated from https://github.com/mstange/profiler-get-symbols.
+// See ProfilerGetSymbols.jsm for more information.
+//
+// The worker instantiates the module, reads the binary into wasm memory, runs
+// the wasm code, and returns the symbol table or an error. Then it shuts down
+// itself.
+
+// Read an open OS.File instance into the Uint8Array dataBuf.
+function readFileInto(file, dataBuf) {
+  // Ideally we'd be able to call file.readTo(dataBuf) here, but readTo no
+  // longer exists.
+  // So instead, we copy the file over into wasm memory in 4MB chunks. This
+  // will take 425 invocations for a a 1.7GB file (such as libxul.so for a
+  // Firefox for Android build) and not take up too much memory per call.
+  const dataBufLen = dataBuf.byteLength;
+  const chunkSize = 4 * 1024 * 1024;
+  let pos = 0;
+  while (pos < dataBufLen) {
+    const chunkData = file.read({bytes: chunkSize});
+    const chunkBytes = chunkData.byteLength;
+    if (chunkBytes === 0) {
+      break;
+    }
+
+    dataBuf.set(chunkData, pos);
+    pos += chunkBytes;
+  }
+}
+
+onmessage = async e => {
+  try {
+    const {binaryPath, debugPath, breakpadId, module} = e.data;
+
+    if (!(module instanceof WebAssembly.Module)) {
+      throw new Error("invalid WebAssembly module");
+    }
+
+    // Instantiate the WASM module.
+    await wasm_bindgen(module);
+
+    const {CompactSymbolTable, wasm} = wasm_bindgen;
+
+    const binaryFile = OS.File.open(binaryPath, {read: true});
+    const binaryDataBufLen = binaryFile.stat().size;
+
+    // Read the binary file into WASM memory.
+    const binaryDataBufPtr = wasm.__wbindgen_malloc(binaryDataBufLen);
+    const binaryDataBuf = new Uint8Array(wasm.memory.buffer, binaryDataBufPtr, binaryDataBufLen);
+    readFileInto(binaryFile, binaryDataBuf);
+    binaryFile.close();
+
+    // Do the same for the debug file, if it is supplied and different from the
+    // binary file. This is only the case on Windows.
+    let debugDataBufLen = binaryDataBufLen;
+    let debugDataBufPtr = binaryDataBufPtr;
+    if (debugPath && debugPath !== binaryPath) {
+      const debugFile = OS.File.open(debugPath, {read: true});
+      debugDataBufLen = debugFile.stat().size;
+      debugDataBufPtr = wasm.__wbindgen_malloc(debugDataBufLen);
+      const debugDataBuf = new Uint8Array(wasm.memory.buffer, debugDataBufPtr, debugDataBufLen);
+      readFileInto(debugFile, debugDataBuf);
+      debugFile.close();
+    }
+
+    // Call get_compact_symbol_table. We're calling the raw wasm function
+    // instead of the binding function that wasm-bindgen generated for us,
+    // because the generated function doesn't let us pass binaryDataBufPtr
+    // or debugDataBufPtr and would force another copy.
+    //
+    // The rust definition of get_compact_symbol_table is:
+    //
+    // #[wasm_bindgen]
+    // pub fn get_compact_symbol_table(binary_data: &[u8], debug_data: &[u8], breakpad_id: &str, dest: &mut CompactSymbolTable) -> bool
+    //
+    // It gets exposed as a wasm function with the following signature:
+    //
+    // pub fn get_compact_symbol_table(binaryDataBufPtr: u32, binaryDataBufLen: u32, debugDataBufPtr: u32, debugDataBufLen: u32, breakpadIdPtr: u32, breakpadIdLen: u32, destPtr: u32) -> u32
+    //
+    // We're relying on implementation details of wasm-bindgen here. The above
+    // is true as of wasm-bindgen 0.2.32.
+
+    // Convert the breakpadId string into bytes in wasm memory.
+    const breakpadIdBuf = new TextEncoder("utf-8").encode(breakpadId);
+    const breakpadIdLen = breakpadIdBuf.length;
+    const breakpadIdPtr = wasm.__wbindgen_malloc(breakpadIdLen);
+    new Uint8Array(wasm.memory.buffer).set(breakpadIdBuf, breakpadIdPtr);
+
+    const output = new CompactSymbolTable();
+    let succeeded;
+    try {
+      succeeded =
+        wasm.get_compact_symbol_table(binaryDataBufPtr, binaryDataBufLen,
+                                      debugDataBufPtr, debugDataBufLen,
+                                      breakpadIdPtr, breakpadIdLen,
+                                      output.ptr) !== 0;
+    } catch (e) {
+      succeeded = false;
+    }
+
+    wasm.__wbindgen_free(breakpadIdPtr, breakpadIdLen);
+    wasm.__wbindgen_free(binaryDataBufPtr, binaryDataBufLen);
+    if (debugDataBufPtr != binaryDataBufPtr) {
+      wasm.__wbindgen_free(debugDataBufPtr, debugDataBufLen);
+    }
+
+    if (!succeeded) {
+      output.free();
+      throw new Error("get_compact_symbol_table returned false");
+    }
+
+    const result = [output.take_addr(), output.take_index(), output.take_buffer()];
+    output.free();
+
+    postMessage({result}, result.map(r => r.buffer));
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+  close();
+};
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ProfilerGetSymbols.jsm
@@ -0,0 +1,95 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+"use strict";
+
+const EXPORTED_SYMBOLS = ["ProfilerGetSymbols"];
+
+ChromeUtils.defineModuleGetter(this, "setTimeout",
+                               "resource://gre/modules/Timer.jsm");
+ChromeUtils.defineModuleGetter(this, "clearTimeout",
+                               "resource://gre/modules/Timer.jsm");
+
+Cu.importGlobalProperties(["fetch"]);
+
+// This module obtains symbol tables for binaries.
+// It does so with the help of a WASM module which gets pulled in from the
+// internet on demand. We're doing this purely for the purposes of saving on
+// code size. The contents of the WASM module are expected to be static, they
+// are checked against the hash specified below.
+// The WASM code is run on a ChromeWorker thread. It takes the raw byte
+// contents of the to-be-dumped binary (and of an additional optional pdb file
+// on Windows) as its input, and returns a set of typed arrays which make up
+// the symbol table.
+
+// Don't let the strange looking URLs and strings below scare you.
+// The hash check ensures that the contents of the wasm module are what we
+// expect them to be.
+// The source code is at https://github.com/mstange/profiler-get-symbols/ .
+
+// Generated from https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681
+const WASM_MODULE_URL =
+  "https://zealous-rosalind-a98ce8.netlify.com/wasm/4af7553b4848038c5a1e33a15ec107094bd0572e16fe2a367235bcb50a630b148ac4fe1d165859d7a9bb4ca4e2572c0e.wasm";
+const WASM_MODULE_INTEGRITY =
+  "sha384-SvdVO0hIA4xaHjOhXsEHCUvQVy4W/io2cjW8tQpjCxSKxP4dFlhZ16m7TKTiVywO";
+
+const EXPIRY_TIME_IN_MS = 5 * 60 * 1000; // 5 minutes
+
+let gCachedWASMModulePromise = null;
+let gCachedWASMModuleExpiryTimer = 0;
+
+function clearCachedWASMModule() {
+  gCachedWASMModulePromise = null;
+  gCachedWASMModuleExpiryTimer = 0;
+}
+
+function getWASMProfilerGetSymbolsModule() {
+  if (!gCachedWASMModulePromise) {
+    gCachedWASMModulePromise = (async function() {
+      const request = new Request(WASM_MODULE_URL, {
+        integrity: WASM_MODULE_INTEGRITY,
+        credentials: "omit",
+      });
+      return WebAssembly.compileStreaming(fetch(request));
+    })();
+  }
+
+  // Reset expiry timer.
+  clearTimeout(gCachedWASMModuleExpiryTimer);
+  gCachedWASMModuleExpiryTimer = setTimeout(clearCachedWASMModule, EXPIRY_TIME_IN_MS);
+
+  return gCachedWASMModulePromise;
+}
+
+this.ProfilerGetSymbols = {
+  /**
+   * Obtain symbols for the binary at the specified location.
+   *
+   * @param {string} binaryPath The absolute path to the binary on the local
+   *   file system.
+   * @param {string} debugPath The absolute path to the binary's pdb file on the
+   *   local file system if on Windows, otherwise the same as binaryPath.
+   * @param {string} breakpadId The breakpadId for the binary whose symbols
+   *   should be obtained. This is used for two purposes: 1) to locate the
+   *   correct single-arch binary in "FatArch" files, and 2) to make sure the
+   *   binary at the given path is actually the one that we want. If no ID match
+   *   is found, this function throws (rejects the promise).
+   * @returns {Promise} The symbol table in SymbolTableAsTuple format, see the
+   *   documentation for nsIProfiler.getSymbolTable.
+   */
+  async getSymbolTable(binaryPath, debugPath, breakpadId) {
+    const module = await getWASMProfilerGetSymbolsModule();
+
+    return new Promise((resolve, reject) => {
+      const worker =
+        new ChromeWorker("resource://app/modules/ProfilerGetSymbols-worker.js");
+      worker.onmessage = (e) => {
+        if (e.data.error) {
+          reject(e.data.error);
+          return;
+        }
+        resolve(e.data.result);
+      };
+      worker.postMessage({binaryPath, debugPath, breakpadId, module});
+    });
+  },
+};
--- a/browser/components/extensions/moz.build
+++ b/browser/components/extensions/moz.build
@@ -14,16 +14,19 @@ EXTRA_COMPONENTS += [
 ]
 
 EXTRA_JS_MODULES += [
     'ExtensionControlledPopup.jsm',
     'ExtensionPopups.jsm',
     'ParseBreakpadSymbols-worker.js',
     'ParseNMSymbols-worker.js',
     'ParseSymbols.jsm',
+    'profiler_get_symbols.js',
+    'ProfilerGetSymbols-worker.js',
+    'ProfilerGetSymbols.jsm',
 ]
 
 DIRS += ['schemas']
 
 BROWSER_CHROME_MANIFESTS += [
     'test/browser/browser-private.ini',
     'test/browser/browser-remote.ini',
     'test/browser/browser.ini',
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/profiler_get_symbols.js
@@ -0,0 +1,186 @@
+//
+// THIS FILE IS AUTOGENERATED by wasm-bindgen.
+//
+// Generated from:
+// https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681
+// by following the instructions in that repository's Readme.md
+//
+
+(function() {
+    var wasm;
+    const __exports = {};
+
+
+    let cachegetUint32Memory = null;
+    function getUint32Memory() {
+        if (cachegetUint32Memory === null || cachegetUint32Memory.buffer !== wasm.memory.buffer) {
+            cachegetUint32Memory = new Uint32Array(wasm.memory.buffer);
+        }
+        return cachegetUint32Memory;
+    }
+
+    function getArrayU32FromWasm(ptr, len) {
+        return getUint32Memory().subarray(ptr / 4, ptr / 4 + len);
+    }
+
+    let cachedGlobalArgumentPtr = null;
+    function globalArgumentPtr() {
+        if (cachedGlobalArgumentPtr === null) {
+            cachedGlobalArgumentPtr = wasm.__wbindgen_global_argument_ptr();
+        }
+        return cachedGlobalArgumentPtr;
+    }
+
+    let cachegetUint8Memory = null;
+    function getUint8Memory() {
+        if (cachegetUint8Memory === null || cachegetUint8Memory.buffer !== wasm.memory.buffer) {
+            cachegetUint8Memory = new Uint8Array(wasm.memory.buffer);
+        }
+        return cachegetUint8Memory;
+    }
+
+    function getArrayU8FromWasm(ptr, len) {
+        return getUint8Memory().subarray(ptr / 1, ptr / 1 + len);
+    }
+
+    function passArray8ToWasm(arg) {
+        const ptr = wasm.__wbindgen_malloc(arg.length * 1);
+        getUint8Memory().set(arg, ptr / 1);
+        return [ptr, arg.length];
+    }
+
+    let cachedTextEncoder = new TextEncoder('utf-8');
+
+    function passStringToWasm(arg) {
+
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = wasm.__wbindgen_malloc(buf.length);
+        getUint8Memory().set(buf, ptr);
+        return [ptr, buf.length];
+    }
+    /**
+    * @param {Uint8Array} arg0
+    * @param {Uint8Array} arg1
+    * @param {string} arg2
+    * @param {CompactSymbolTable} arg3
+    * @returns {boolean}
+    */
+    __exports.get_compact_symbol_table = function(arg0, arg1, arg2, arg3) {
+        const [ptr0, len0] = passArray8ToWasm(arg0);
+        const [ptr1, len1] = passArray8ToWasm(arg1);
+        const [ptr2, len2] = passStringToWasm(arg2);
+        try {
+            return (wasm.get_compact_symbol_table(ptr0, len0, ptr1, len1, ptr2, len2, arg3.ptr)) !== 0;
+
+        } finally {
+            wasm.__wbindgen_free(ptr0, len0 * 1);
+            wasm.__wbindgen_free(ptr1, len1 * 1);
+            wasm.__wbindgen_free(ptr2, len2 * 1);
+
+        }
+
+    };
+
+    function freeCompactSymbolTable(ptr) {
+
+        wasm.__wbg_compactsymboltable_free(ptr);
+    }
+    /**
+    */
+    class CompactSymbolTable {
+
+        free() {
+            const ptr = this.ptr;
+            this.ptr = 0;
+            freeCompactSymbolTable(ptr);
+        }
+
+        /**
+        * @returns {}
+        */
+        constructor() {
+            this.ptr = wasm.compactsymboltable_new();
+        }
+        /**
+        * @returns {Uint32Array}
+        */
+        take_addr() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_addr(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU32FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 4);
+            return realRet;
+
+        }
+        /**
+        * @returns {Uint32Array}
+        */
+        take_index() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_index(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU32FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 4);
+            return realRet;
+
+        }
+        /**
+        * @returns {Uint8Array}
+        */
+        take_buffer() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_buffer(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU8FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 1);
+            return realRet;
+
+        }
+    }
+    __exports.CompactSymbolTable = CompactSymbolTable;
+
+    let cachedTextDecoder = new TextDecoder('utf-8');
+
+    function getStringFromWasm(ptr, len) {
+        return cachedTextDecoder.decode(getUint8Memory().subarray(ptr, ptr + len));
+    }
+
+    __exports.__wbindgen_throw = function(ptr, len) {
+        throw new Error(getStringFromWasm(ptr, len));
+    };
+
+    function init(path_or_module) {
+        let instantiation;
+        const imports = { './profiler_get_symbols': __exports };
+        if (path_or_module instanceof WebAssembly.Module) {
+            instantiation = WebAssembly.instantiate(path_or_module, imports)
+            .then(instance => {
+            return { instance, module: path_or_module }
+        });
+    } else {
+        const data = fetch(path_or_module);
+        if (typeof WebAssembly.instantiateStreaming === 'function') {
+            instantiation = WebAssembly.instantiateStreaming(data, imports);
+        } else {
+            instantiation = data
+            .then(response => response.arrayBuffer())
+            .then(buffer => WebAssembly.instantiate(buffer, imports));
+        }
+    }
+    return instantiation.then(({instance}) => {
+        wasm = init.wasm = instance.exports;
+        return;
+    });
+};
+self.wasm_bindgen = Object.assign(init, __exports);
+})();
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -165,16 +165,17 @@
   "PhoneNumberMetaData.jsm": ["PHONE_NUMBER_META_DATA"],
   "PluginProvider.jsm": [],
   "PointerAdapter.jsm": ["PointerRelay", "PointerAdapter"],
   "policies.js": ["ErrorHandler", "SyncScheduler"],
   "prefs.js": ["Branch", "MarionettePrefs"],
   "prefs.js": ["PrefsEngine", "PrefRec"],
   "prefs.jsm": ["Preference"],
   "pretty-fast.js": ["prettyFast"],
+  "profiler_get_symbols.js": ["wasm_bindgen"],
   "PromiseWorker.jsm": ["BasePromiseWorker"],
   "PushCrypto.jsm": ["PushCrypto", "concatArray"],
   "quit.js": ["goQuitApplication"],
   "record.js": ["WBORecord", "RecordManager", "CryptoWrapper", "CollectionKeyManager", "Collection"],
   "recursive_importA.jsm": ["foo", "bar"],
   "recursive_importB.jsm": ["baz", "qux"],
   "Reducers.jsm": ["reducers", "INITIAL_STATE", "insertPinned", "TOP_SITES_DEFAULT_ROWS", "TOP_SITES_MAX_SITES_PER_ROW"],
   "Redux.jsm": ["redux"],