Bug 1289194 - Experimental LibFuzzer integration. r=glandium
☠☠ backed out by 33e7ae9b3104 ☠ ☠
authorChristian Holler (:decoder) <choller@mozilla.com>
Thu, 01 Sep 2016 15:07:01 +0200
changeset 411019 95e68b473e91d27a6c469c4a4765aa04e784c800
parent 411018 e753af61eb0221f850b557127ad99768d9c0e0a9
child 411020 13a770064f3ee6f6da31c085059a443bff1517d6
push id28817
push userpaul@paul.cx
push dateWed, 07 Sep 2016 12:47:55 +0000
reviewersglandium
bugs1289194
milestone51.0a1
Bug 1289194 - Experimental LibFuzzer integration. r=glandium
browser/app/moz.build
browser/app/nsBrowserApp.cpp
moz.build
toolkit/moz.configure
toolkit/xre/nsAppRunner.cpp
tools/fuzzing/libfuzzer/FuzzerCustomMain.cpp
tools/fuzzing/libfuzzer/Makefile.in
tools/fuzzing/libfuzzer/clone_libfuzzer.sh
tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.cpp
tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.h
tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.cpp
tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.h
tools/fuzzing/libfuzzer/harness/moz.build
tools/fuzzing/libfuzzer/moz.build
tools/fuzzing/moz.build
xpcom/build/nsXULAppAPI.h
--- a/browser/app/moz.build
+++ b/browser/app/moz.build
@@ -28,16 +28,19 @@ LOCAL_INCLUDES += [
     '/xpcom/base',
     '/xpcom/build',
 ]
 
 USE_LIBS += [
     'mozglue',
 ]
 
+if CONFIG['LIBFUZZER']:
+  USE_LIBS += [ 'fuzzer' ]
+
 if CONFIG['_MSC_VER']:
     # Always enter a Windows program through wmain, whether or not we're
     # a console application.
     WIN32_EXE_LDFLAGS += ['-ENTRY:wmainCRTStartup']
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     RCINCLUDE = 'splash.rc'
     DEFINES['MOZ_PHOENIX'] = True
--- a/browser/app/nsBrowserApp.cpp
+++ b/browser/app/nsBrowserApp.cpp
@@ -124,33 +124,52 @@ XRE_TelemetryAccumulateType XRE_Telemetr
 XRE_StartupTimelineRecordType XRE_StartupTimelineRecord;
 XRE_mainType XRE_main;
 XRE_StopLateWriteChecksType XRE_StopLateWriteChecks;
 XRE_XPCShellMainType XRE_XPCShellMain;
 XRE_GetProcessTypeType XRE_GetProcessType;
 XRE_SetProcessTypeType XRE_SetProcessType;
 XRE_InitChildProcessType XRE_InitChildProcess;
 XRE_EnableSameExecutableForContentProcType XRE_EnableSameExecutableForContentProc;
+#ifdef LIBFUZZER
+XRE_LibFuzzerSetMainType XRE_LibFuzzerSetMain;
+XRE_LibFuzzerGetFuncsType XRE_LibFuzzerGetFuncs;
+#endif
 
 static const nsDynamicFunctionLoad kXULFuncs[] = {
     { "XRE_GetFileFromPath", (NSFuncPtr*) &XRE_GetFileFromPath },
     { "XRE_CreateAppData", (NSFuncPtr*) &XRE_CreateAppData },
     { "XRE_FreeAppData", (NSFuncPtr*) &XRE_FreeAppData },
     { "XRE_TelemetryAccumulate", (NSFuncPtr*) &XRE_TelemetryAccumulate },
     { "XRE_StartupTimelineRecord", (NSFuncPtr*) &XRE_StartupTimelineRecord },
     { "XRE_main", (NSFuncPtr*) &XRE_main },
     { "XRE_StopLateWriteChecks", (NSFuncPtr*) &XRE_StopLateWriteChecks },
     { "XRE_XPCShellMain", (NSFuncPtr*) &XRE_XPCShellMain },
     { "XRE_GetProcessType", (NSFuncPtr*) &XRE_GetProcessType },
     { "XRE_SetProcessType", (NSFuncPtr*) &XRE_SetProcessType },
     { "XRE_InitChildProcess", (NSFuncPtr*) &XRE_InitChildProcess },
     { "XRE_EnableSameExecutableForContentProc", (NSFuncPtr*) &XRE_EnableSameExecutableForContentProc },
+#ifdef LIBFUZZER
+    { "XRE_LibFuzzerSetMain", (NSFuncPtr*) &XRE_LibFuzzerSetMain },
+    { "XRE_LibFuzzerGetFuncs", (NSFuncPtr*) &XRE_LibFuzzerGetFuncs },
+#endif
     { nullptr, nullptr }
 };
 
+#ifdef LIBFUZZER
+int libfuzzer_main(int argc, char **argv);
+
+/* This wrapper is used by the libFuzzer main to call into libxul */
+
+void libFuzzerGetFuncs(const char* moduleName, LibFuzzerInitFunc* initFunc,
+                       LibFuzzerTestingFunc* testingFunc) {
+  return XRE_LibFuzzerGetFuncs(moduleName, initFunc, testingFunc);
+}
+#endif
+
 static int do_main(int argc, char* argv[], char* envp[], nsIFile *xreDirectory)
 {
   nsCOMPtr<nsIFile> appini;
   nsresult rv;
   uint32_t mainFlags = 0;
 
   // Allow firefox.exe to launch XULRunner apps via -app <application.ini>
   // Note that -app must be the *first* argument.
@@ -251,16 +270,21 @@ static int do_main(int argc, char* argv[
   if (!brokerServices) {
     Output("Couldn't initialize the broker services.\n");
     return 255;
   }
 #endif
   appData.sandboxBrokerServices = brokerServices;
 #endif
 
+#ifdef LIBFUZZER
+  if (getenv("LIBFUZZER"))
+    XRE_LibFuzzerSetMain(argc, argv, libfuzzer_main);
+#endif
+
   return XRE_main(argc, argv, &appData, mainFlags);
 }
 
 static bool
 FileExists(const char *path)
 {
 #ifdef XP_WIN
   wchar_t wideDir[MAX_PATH];
--- a/moz.build
+++ b/moz.build
@@ -52,16 +52,17 @@ if not CONFIG['JS_STANDALONE']:
 
 DIRS += [
     'config/external/fdlibm',
     'config/external/nspr',
     'config/external/zlib',
     'memory',
     'mfbt',
     'mozglue',
+    'tools/fuzzing',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     DIRS += ['xpcom/xpidl']
 
 if CONFIG['USE_ICU']:
     DIRS += ['config/external/icu']
 
--- a/toolkit/moz.configure
+++ b/toolkit/moz.configure
@@ -759,8 +759,20 @@ def skia_includes(skia, skia_gpu):
         includes += [
             '/gfx/skia/skia/include/gpu',
             '/gfx/skia/skia/include/utils',
         ]
 
     return includes
 
 set_config('SKIA_INCLUDES', skia_includes)
+
+# Support various fuzzing options
+# ==============================================================
+option('--enable-libfuzzer', help='Enable libfuzzer support')
+
+@depends('--enable-libfuzzer')
+def enable_libfuzzer(value):
+    if value:
+        return True
+
+set_config('LIBFUZZER', enable_libfuzzer)
+set_define('LIBFUZZER', enable_libfuzzer)
--- a/toolkit/xre/nsAppRunner.cpp
+++ b/toolkit/xre/nsAppRunner.cpp
@@ -253,16 +253,28 @@ nsString gAbsoluteArgv0Path;
 // build backends.
 #include "buildid.h"
 #endif
 
 #ifdef MOZ_LINKER
 extern "C" MFBT_API bool IsSignalHandlingBroken();
 #endif
 
+#ifdef LIBFUZZER
+#include "LibFuzzerRunner.h"
+
+namespace mozilla {
+LibFuzzerRunner* libFuzzerRunner = 0;
+} // namespace mozilla
+
+extern "C" MOZ_EXPORT void XRE_LibFuzzerSetMain(int argc, char** argv, LibFuzzerMain main) {
+  mozilla::libFuzzerRunner->setParams(argc, argv, main);
+}
+#endif
+
 namespace mozilla {
 int (*RunGTest)() = 0;
 } // namespace mozilla
 
 using namespace mozilla;
 using mozilla::Unused;
 using mozilla::scache::StartupCache;
 using mozilla::dom::ContentParent;
@@ -3637,16 +3649,23 @@ XREMain::XRE_mainStartup(bool* aExitFlag
 
   // Open the display ourselves instead of using gtk_init, so that we can
   // close it without fear that one day gtk might clean up the display it
   // opens.
   if (!gtk_parse_args(&gArgc, &gArgv))
     return 1;
 #endif /* MOZ_WIDGET_GTK */
 
+#ifdef LIBFUZZER
+  if (PR_GetEnv("LIBFUZZER")) {
+    *aExitFlag = true;
+    return mozilla::libFuzzerRunner->Run();
+  }
+#endif
+
   if (PR_GetEnv("MOZ_RUN_GTEST")) {
     int result;
 #ifdef XP_WIN
     UseParentConsole();
 #endif
     // RunGTest will only be set if we're in xul-unit
     if (mozilla::RunGTest) {
       gIsGtest = true;
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/FuzzerCustomMain.cpp
@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * * This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <cstdlib>
+
+#include "FuzzerInterface.h"
+#include "FuzzerInternal.h"
+#include "harness/LibFuzzerRegistry.h"
+
+/* This is a wrapper defined in browser/app/nsBrowserApp.cpp,
+ * encapsulating the XRE_ equivalent defined in libxul */
+extern void libFuzzerGetFuncs(const char*, LibFuzzerInitFunc*,
+                                 LibFuzzerTestingFunc*);
+
+int libfuzzer_main(int argc, char **argv) {
+  LibFuzzerInitFunc initFunc = nullptr;
+  LibFuzzerTestingFunc testingFunc = nullptr;
+
+  libFuzzerGetFuncs(getenv("LIBFUZZER"), &initFunc, &testingFunc);
+
+  if (initFunc) {
+    int ret = initFunc(&argc, &argv);
+    if (ret) {
+      fprintf(stderr, "LibFuzzer: Error: Initialize callback failed\n");
+      return ret;
+    }
+  }
+
+  if (!testingFunc) {
+      fprintf(stderr, "LibFuzzer: Error: No testing callback found\n");
+      return 1;
+  }
+
+  return fuzzer::FuzzerDriver(&argc, &argv, testingFunc);
+}
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/Makefile.in
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+include $(topsrcdir)/config/rules.mk
+
+# According to the LLVM docs, LibFuzzer isn't supposed to be built with any
+# sanitizer flags and in fact, building it with ASan coverage currently causes
+# Clang 3.9+ to crash, so we filter out all sanitizer-related flags here.
+CXXFLAGS := $(filter-out -fsanitize%,$(CXXFLAGS))
+CFLAGS := $(filter-out -fsanitize%,$(CFLAGS))
+LDFLAGS := $(filter-out -fsanitize%,$(LDFLAGS))
new file mode 100755
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+mkdir tmp/
+git clone --no-checkout --depth 1 https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer tmp/
+mv tmp/.git .
+rm -Rf tmp
+git reset --hard HEAD
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.cpp
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * * This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LibFuzzerRegistry.h"
+
+extern "C" {
+    void MOZ_EXPORT XRE_LibFuzzerGetFuncs(const char* moduleName, LibFuzzerInitFunc* initFunc, LibFuzzerTestingFunc* testingFunc) {
+        std::string moduleNameStr(moduleName);
+        mozilla::LibFuzzerFunctions funcs = mozilla::LibFuzzerRegistry::getInstance().getModuleFunctions(moduleNameStr);
+        *initFunc = funcs.first;
+        *testingFunc = funcs.second;
+    }
+}
+
+namespace mozilla {
+
+LibFuzzerRegistry& LibFuzzerRegistry::getInstance() {
+    static LibFuzzerRegistry instance;
+    return instance;
+}
+
+void LibFuzzerRegistry::registerModule(std::string moduleName, LibFuzzerInitFunc initFunc, LibFuzzerTestingFunc testingFunc) {
+    moduleMap.insert(std::pair<std::string, LibFuzzerFunctions>(moduleName,LibFuzzerFunctions(initFunc, testingFunc)));
+}
+
+LibFuzzerFunctions LibFuzzerRegistry::getModuleFunctions(std::string& moduleName) {
+    return moduleMap[moduleName];
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * * This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _LibFuzzerRegistry_h__
+#define _LibFuzzerRegistry_h__
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "mozilla/Attributes.h"
+
+typedef int(*LibFuzzerMain)(int, char**);
+typedef int(*LibFuzzerInitFunc)(int*, char***);
+typedef int(*LibFuzzerTestingFunc)(const uint8_t*, size_t);
+
+namespace mozilla {
+
+typedef std::pair<LibFuzzerInitFunc, LibFuzzerTestingFunc> LibFuzzerFunctions;
+
+class LibFuzzerRegistry {
+    public:
+        MOZ_EXPORT static LibFuzzerRegistry& getInstance();
+        MOZ_EXPORT void registerModule(std::string moduleName, LibFuzzerInitFunc initFunc, LibFuzzerTestingFunc testingFunc);
+        MOZ_EXPORT LibFuzzerFunctions getModuleFunctions(std::string& moduleName);
+
+        LibFuzzerRegistry(LibFuzzerRegistry const&) = delete;
+        void operator=(LibFuzzerRegistry const&) = delete;
+
+    private:
+        LibFuzzerRegistry() {};
+        std::map<std::string, LibFuzzerFunctions> moduleMap;
+};
+
+} // namespace mozilla
+
+
+#endif // _LibFuzzerRegistry_h__
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.cpp
@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * * This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LibFuzzerRunner.h"
+#include "mozilla/Attributes.h"
+#include "testing/TestHarness.h"
+#include "prenv.h"
+
+namespace mozilla {
+
+// We use a static var 'libFuzzerRunner' defined in nsAppRunner.cpp.
+// libFuzzerRunner is initialized to nullptr but if LibFuzzer (this file)
+// is linked in then libFuzzerRunner will be set here indicating that
+// we want to call into LibFuzzer's main.
+class _InitLibFuzzer {
+public:
+  _InitLibFuzzer() {
+    libFuzzerRunner = new LibFuzzerRunner();
+  }
+} InitLibFuzzer;
+
+int LibFuzzerRunner::Run() {
+  ScopedXPCOM xpcom("LibFuzzer");
+  return mFuzzerMain(mArgc, mArgv);
+}
+
+typedef int(*LibFuzzerMain)(int, char**);
+
+void LibFuzzerRunner::setParams(int argc, char** argv, LibFuzzerMain main) {
+  mArgc = argc;
+  mArgv = argv;
+  mFuzzerMain = main;
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.h
@@ -0,0 +1,23 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * * This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+namespace mozilla {
+
+typedef int(*LibFuzzerMain)(int, char**);
+
+class LibFuzzerRunner {
+public:
+  int Run();
+  void setParams(int argc, char** argv, LibFuzzerMain main);
+
+private:
+  int mArgc;
+  char** mArgv;
+  LibFuzzerMain mFuzzerMain;
+};
+
+extern LibFuzzerRunner* libFuzzerRunner;
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/harness/moz.build
@@ -0,0 +1,19 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Library('fuzzer-runner')
+
+SOURCES += [
+    'LibFuzzerRegistry.cpp',
+    'LibFuzzerRunner.cpp',
+]
+
+EXPORTS += [
+    'LibFuzzerRegistry.h',
+    'LibFuzzerRunner.h',
+]
+
+FINAL_LIBRARY = "xul"
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/libfuzzer/moz.build
@@ -0,0 +1,26 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Library('fuzzer')
+
+DIRS += [
+  'harness',
+]
+
+SOURCES += [
+    'FuzzerCrossOver.cpp',
+    'FuzzerCustomMain.cpp',
+    'FuzzerDriver.cpp',
+    'FuzzerExtFunctionsDlsym.cpp',
+    'FuzzerExtFunctionsWeak.cpp',
+    'FuzzerIO.cpp',
+    'FuzzerLoop.cpp',
+    'FuzzerMutate.cpp',
+    'FuzzerSHA1.cpp',
+    'FuzzerTracePC.cpp',
+    'FuzzerTraceState.cpp',
+    'FuzzerUtil.cpp',
+]
new file mode 100644
--- /dev/null
+++ b/tools/fuzzing/moz.build
@@ -0,0 +1,10 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+if CONFIG['LIBFUZZER']:
+  DIRS += [
+    'libfuzzer',
+  ]
--- a/xpcom/build/nsXULAppAPI.h
+++ b/xpcom/build/nsXULAppAPI.h
@@ -515,9 +515,21 @@ XRE_API(int,
         XRE_XPCShellMain, (int argc, char** argv, char** envp,
                            const XREShellData* aShellData))
 
 #if MOZ_WIDGET_GTK == 2
 XRE_API(void,
         XRE_GlibInit, ())
 #endif
 
+
+#ifdef LIBFUZZER
+#include "LibFuzzerRegistry.h"
+
+XRE_API(void,
+        XRE_LibFuzzerSetMain, (int, char**, LibFuzzerMain))
+
+XRE_API(void,
+        XRE_LibFuzzerGetFuncs, (const char*, LibFuzzerInitFunc*,
+                                LibFuzzerTestingFunc*))
+#endif // LIBFUZZER
+
 #endif // _nsXULAppAPI_h__