☠☠ backed out by de805196bbc4 ☠ ☠ | |
author | Nicholas Nethercote <nnethercote@mozilla.com> |
Thu, 09 Oct 2014 19:28:33 -0700 | |
changeset 238474 | 835fbe63da4aad49f636b68077a9261b55a54572 |
parent 238473 | 6edcdf87a37d5eefb26cc14c251561b0d960f3b4 |
child 238475 | 94c5d968e7e802a423c2634ed15bea2dcff2d6ef |
push id | 660 |
push user | raliiev@mozilla.com |
push date | Wed, 18 Feb 2015 20:30:48 +0000 |
treeherder | mozilla-release@49e493494178 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | glandium |
bugs | 1076446 |
milestone | 36.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/memory/replace/dmd/DMD.cpp +++ b/memory/replace/dmd/DMD.cpp @@ -222,28 +222,16 @@ StatusMsg(const char* aFmt, ...) /* static */ void InfallibleAllocPolicy::ExitOnFailure(const void* aP) { if (!aP) { MOZ_CRASH("DMD out of memory; aborting"); } } -class FpWriteFunc : public JSONWriteFunc -{ -public: - explicit FpWriteFunc(FILE* aFp) : mFp(aFp) {} - ~FpWriteFunc() { fclose(mFp); } - - void Write(const char* aStr) { fputs(aStr, mFp); } - -private: - FILE* mFp; -}; - static double Percent(size_t part, size_t whole) { return (whole == 0) ? 0 : 100 * (double)part / whole; } // Commifies the number. static char* @@ -284,46 +272,38 @@ class Options const T mDefault; const T mMax; T mActual; NumOption(T aDefault, T aMax) : mDefault(aDefault), mMax(aMax), mActual(aDefault) {} }; - enum Mode { - Normal, // run normally - Test // do some basic correctness tests - }; - char* mDMDEnvVar; // a saved copy, for later printing NumOption<size_t> mSampleBelowSize; NumOption<uint32_t> mMaxFrames; bool mShowDumpStats; - Mode mMode; void BadArg(const char* aArg); static const char* ValueIfMatch(const char* aArg, const char* aOptionName); static bool GetLong(const char* aArg, const char* aOptionName, long aMin, long aMax, long* aValue); static bool GetBool(const char* aArg, const char* aOptionName, bool* aValue); public: explicit Options(const char* aDMDEnvVar); const char* DMDEnvVar() const { return mDMDEnvVar; } size_t SampleBelowSize() const { return mSampleBelowSize.mActual; } size_t MaxFrames() const { return mMaxFrames.mActual; } size_t ShowDumpStats() const { return mShowDumpStats; } - void SetSampleBelowSize(size_t aN) { mSampleBelowSize.mActual = aN; } - - bool IsTestMode() const { return mMode == Test; } + void SetSampleBelowSize(size_t aSize) { mSampleBelowSize.mActual = aSize; } }; static Options *gOptions; //--------------------------------------------------------------------------- // The global lock //--------------------------------------------------------------------------- @@ -1273,18 +1253,17 @@ Options::GetBool(const char* aArg, const // of 4096, for example, then our alloc counter would only take on even // values, because jemalloc always rounds up requests sizes. In contrast, a // prime size will explore all possible values of the alloc counter. // Options::Options(const char* aDMDEnvVar) : mDMDEnvVar(InfallibleAllocPolicy::strdup_(aDMDEnvVar)), mSampleBelowSize(4093, 100 * 100 * 1000), mMaxFrames(StackTrace::MaxFrames, StackTrace::MaxFrames), - mShowDumpStats(false), - mMode(Normal) + mShowDumpStats(false) { char* e = mDMDEnvVar; if (strcmp(e, "1") != 0) { bool isEnd = false; while (!isEnd) { // Consume leading whitespace. while (isspace(*e)) { e++; @@ -1309,21 +1288,16 @@ Options::Options(const char* aDMDEnvVar) mSampleBelowSize.mActual = myLong; } else if (GetLong(arg, "--max-frames", 1, mMaxFrames.mMax, &myLong)) { mMaxFrames.mActual = myLong; } else if (GetBool(arg, "--show-dump-stats", &myBool)) { mShowDumpStats = myBool; - } else if (strcmp(arg, "--mode=normal") == 0) { - mMode = Options::Normal; - } else if (strcmp(arg, "--mode=test") == 0) { - mMode = Options::Test; - } else if (strcmp(arg, "") == 0) { // This can only happen if there is trailing whitespace. Ignore. MOZ_ASSERT(isEnd); } else { BadArg(arg); } @@ -1349,48 +1323,32 @@ Options::BadArg(const char* aArg) StatusMsg(" --sample-below=<1..%d> Sample blocks smaller than this [%d]\n", int(mSampleBelowSize.mMax), int(mSampleBelowSize.mDefault)); StatusMsg(" (prime numbers are recommended)\n"); StatusMsg(" --max-frames=<1..%d> Max. depth of stack traces [%d]\n", int(mMaxFrames.mMax), int(mMaxFrames.mDefault)); StatusMsg(" --show-dump-stats=<yes|no> Show stats about dumps? [no]\n"); - StatusMsg(" --mode=<normal|test> Mode of operation [normal]\n"); StatusMsg("\n"); exit(1); } //--------------------------------------------------------------------------- // DMD start-up //--------------------------------------------------------------------------- #ifdef XP_MACOSX static void NopStackWalkCallback(uint32_t aFrameNumber, void* aPc, void* aSp, void* aClosure) { } #endif -// Note that fopen() can allocate. -static FILE* -OpenOutputFile(const char* aFilename) -{ - FILE* fp = fopen(aFilename, "w"); - if (!fp) { - StatusMsg("can't create %s file: %s\n", aFilename, strerror(errno)); - exit(1); - } - return fp; -} - -static void RunTestMode(UniquePtr<FpWriteFunc> aF1, UniquePtr<FpWriteFunc> aF2, - UniquePtr<FpWriteFunc> aF3, UniquePtr<FpWriteFunc> aF4); - // WARNING: this function runs *very* early -- before all static initializers // have run. For this reason, non-scalar globals such as gStateLock and // gStackTraceTable are allocated dynamically (so we can guarantee their // construction in this function) rather than statically. static void Init(const malloc_table_t* aMallocTable) { MOZ_ASSERT(!gIsDMDRunning); @@ -1434,41 +1392,17 @@ Init(const malloc_table_t* aMallocTable) gStackTraceTable = InfallibleAllocPolicy::new_<StackTraceTable>(); gStackTraceTable->init(8192); gBlockTable = InfallibleAllocPolicy::new_<BlockTable>(); gBlockTable->init(8192); } - if (gOptions->IsTestMode()) { - // Do all necessary allocations before setting gIsDMDRunning so those - // allocations don't show up in our results. Once gIsDMDRunning is set we - // are intercepting malloc et al. in earnest. - // - // These files are written to $CWD. It would probably be better to write - // them to "TmpD" using the directory service, but that would require - // linking DMD with XPCOM. - auto f1 = MakeUnique<FpWriteFunc>(OpenOutputFile("full-empty.json")); - auto f2 = MakeUnique<FpWriteFunc>(OpenOutputFile("full-unsampled1.json")); - auto f3 = MakeUnique<FpWriteFunc>(OpenOutputFile("full-unsampled2.json")); - auto f4 = MakeUnique<FpWriteFunc>(OpenOutputFile("full-sampled.json")); - gIsDMDRunning = true; - - StatusMsg("running test mode...\n"); - RunTestMode(Move(f1), Move(f2), Move(f3), Move(f4)); - StatusMsg("finished test mode; DMD is now disabled again\n"); - - // Continue running so that the xpcshell test can complete, but DMD no - // longer needs to be running. - gIsDMDRunning = false; - - } else { - gIsDMDRunning = true; - } + gIsDMDRunning = true; } //--------------------------------------------------------------------------- // DMD reporting and unreporting //--------------------------------------------------------------------------- static void ReportHelper(const void* aPtr, bool aReportedOnAlloc) @@ -1834,266 +1768,23 @@ AnalyzeReports(JSONWriter& aWriter) AnalyzeReportsImpl(aWriter); ClearReports(); } //--------------------------------------------------------------------------- // Testing //--------------------------------------------------------------------------- -// This function checks that heap blocks that have the same stack trace but -// different (or no) reporters get aggregated separately. -void Foo(int aSeven) +MOZ_EXPORT void +SetSampleBelowSize(size_t aSize) { - char* a[6]; - for (int i = 0; i < aSeven - 1; i++) { - a[i] = (char*) replace_malloc(128 - 16*i); - } - - for (int i = 0; i < aSeven - 5; i++) { - Report(a[i]); // reported - } - Report(a[2]); // reported - Report(a[3]); // reported - // a[4], a[5] unreported -} - -// This stops otherwise-unused variables from being optimized away. -static void -UseItOrLoseIt(void* aPtr, int aSeven) -{ - char buf[64]; - int n = sprintf(buf, "%p\n", aPtr); - if (n == 20 + aSeven) { - fprintf(stderr, "well, that is surprising"); - } + gOptions->SetSampleBelowSize(aSize); } -// The output from this function feeds into DMD's xpcshell test. -static void -RunTestMode(UniquePtr<FpWriteFunc> aF1, UniquePtr<FpWriteFunc> aF2, - UniquePtr<FpWriteFunc> aF3, UniquePtr<FpWriteFunc> aF4) +MOZ_EXPORT void +ClearBlocks() { - // This test relies on the compiler not doing various optimizations, such as - // eliding unused replace_malloc() calls or unrolling loops with fixed - // iteration counts. So we want a constant value that the compiler can't - // determine statically, and we use that in various ways to prevent the above - // optimizations from happening. - // - // This code always sets |seven| to the value 7. It works because we know - // that "--mode=test" must be within the DMD environment variable if we reach - // here, but the compiler almost certainly does not. - // - char* env = getenv("DMD"); - char* p1 = strstr(env, "--mode=t"); - char* p2 = strstr(p1, "test"); - int seven = p2 - p1; - - // The first part of this test requires sampling to be disabled. - gOptions->SetSampleBelowSize(1); - - //--------- - - // AnalyzeReports 1. Zero for everything. - JSONWriter writer1(Move(aF1)); - AnalyzeReports(writer1); - - //--------- - - // AnalyzeReports 2: 1 freed, 9 out of 10 unreported. - // AnalyzeReports 3: still present and unreported. - int i; - char* a = nullptr; - for (i = 0; i < seven + 3; i++) { - a = (char*) replace_malloc(100); - UseItOrLoseIt(a, seven); - } - replace_free(a); - - // Note: 8 bytes is the smallest requested size that gives consistent - // behaviour across all platforms with jemalloc. - // AnalyzeReports 2: reported. - // AnalyzeReports 3: thrice-reported. - char* a2 = (char*) replace_malloc(8); - Report(a2); - - // AnalyzeReports 2: reported. - // AnalyzeReports 3: reportedness carries over, due to ReportOnAlloc. - char* b = (char*) replace_malloc(10); - ReportOnAlloc(b); - - // ReportOnAlloc, then freed. - // AnalyzeReports 2: freed, irrelevant. - // AnalyzeReports 3: freed, irrelevant. - char* b2 = (char*) replace_malloc(1); - ReportOnAlloc(b2); - replace_free(b2); - - // AnalyzeReports 2: reported 4 times. - // AnalyzeReports 3: freed, irrelevant. - char* c = (char*) replace_calloc(10, 3); - Report(c); - for (int i = 0; i < seven - 4; i++) { - Report(c); - } - - // AnalyzeReports 2: ignored. - // AnalyzeReports 3: irrelevant. - Report((void*)(intptr_t)i); - - // jemalloc rounds this up to 8192. - // AnalyzeReports 2: reported. - // AnalyzeReports 3: freed. - char* e = (char*) replace_malloc(4096); - e = (char*) replace_realloc(e, 4097); - Report(e); - - // First realloc is like malloc; second realloc is shrinking. - // AnalyzeReports 2: reported. - // AnalyzeReports 3: re-reported. - char* e2 = (char*) replace_realloc(nullptr, 1024); - e2 = (char*) replace_realloc(e2, 512); - Report(e2); - - // First realloc is like malloc; second realloc creates a min-sized block. - // XXX: on Windows, second realloc frees the block. - // AnalyzeReports 2: reported. - // AnalyzeReports 3: freed, irrelevant. - char* e3 = (char*) replace_realloc(nullptr, 1023); -//e3 = (char*) replace_realloc(e3, 0); - MOZ_ASSERT(e3); - Report(e3); - - // AnalyzeReports 2: freed, irrelevant. - // AnalyzeReports 3: freed, irrelevant. - char* f = (char*) replace_malloc(64); - replace_free(f); - - // AnalyzeReports 2: ignored. - // AnalyzeReports 3: irrelevant. - Report((void*)(intptr_t)0x0); - - // AnalyzeReports 2: mixture of reported and unreported. - // AnalyzeReports 3: all unreported. - Foo(seven); - Foo(seven); - - // AnalyzeReports 2: twice-reported. - // AnalyzeReports 3: twice-reported. - char* g1 = (char*) replace_malloc(77); - ReportOnAlloc(g1); - ReportOnAlloc(g1); - - // AnalyzeReports 2: twice-reported. - // AnalyzeReports 3: once-reported. - char* g2 = (char*) replace_malloc(78); - Report(g2); - ReportOnAlloc(g2); - - // AnalyzeReports 2: twice-reported. - // AnalyzeReports 3: once-reported. - char* g3 = (char*) replace_malloc(79); - ReportOnAlloc(g3); - Report(g3); - - // All the odd-ball ones. - // AnalyzeReports 2: all unreported. - // AnalyzeReports 3: all freed, irrelevant. - // XXX: no memalign on Mac -//void* x = memalign(64, 65); // rounds up to 128 -//UseItOrLoseIt(x, seven); - // XXX: posix_memalign doesn't work on B2G -//void* y; -//posix_memalign(&y, 128, 129); // rounds up to 256 -//UseItOrLoseIt(y, seven); - // XXX: valloc doesn't work on Windows. -//void* z = valloc(1); // rounds up to 4096 -//UseItOrLoseIt(z, seven); -//aligned_alloc(64, 256); // XXX: C11 only - - // AnalyzeReports 2. - JSONWriter writer2(Move(aF2)); - AnalyzeReports(writer2); - - //--------- - - Report(a2); - Report(a2); - replace_free(c); - replace_free(e); - Report(e2); - replace_free(e3); -//replace_free(x); -//replace_free(y); -//replace_free(z); - - // AnalyzeReports 3. - JSONWriter writer3(Move(aF3)); - AnalyzeReports(writer3); - - //--------- - - // Clear all knowledge of existing blocks to give us a clean slate. gBlockTable->clear(); - - gOptions->SetSampleBelowSize(128); - - char* s; - - // This equals the sample size, and so is reported exactly. It should be - // listed before records of the same size that are sampled. - s = (char*) replace_malloc(128); - UseItOrLoseIt(s, seven); - - // This exceeds the sample size, and so is reported exactly. - s = (char*) replace_malloc(144); - UseItOrLoseIt(s, seven); - - // These together constitute exactly one sample. - for (int i = 0; i < seven + 9; i++) { - s = (char*) replace_malloc(8); - UseItOrLoseIt(s, seven); - } - MOZ_ASSERT(gSmallBlockActualSizeCounter == 0); - - // These fall 8 bytes short of a full sample. - for (int i = 0; i < seven + 8; i++) { - s = (char*) replace_malloc(8); - UseItOrLoseIt(s, seven); - } - MOZ_ASSERT(gSmallBlockActualSizeCounter == 120); - - // This exceeds the sample size, and so is recorded exactly. - s = (char*) replace_malloc(256); - UseItOrLoseIt(s, seven); - MOZ_ASSERT(gSmallBlockActualSizeCounter == 120); - - // This gets more than to a full sample from the |i < 15| loop above. - s = (char*) replace_malloc(96); - UseItOrLoseIt(s, seven); - MOZ_ASSERT(gSmallBlockActualSizeCounter == 88); - - // This gets to another full sample. - for (int i = 0; i < seven - 2; i++) { - s = (char*) replace_malloc(8); - UseItOrLoseIt(s, seven); - } - MOZ_ASSERT(gSmallBlockActualSizeCounter == 0); - - // This allocates 16, 32, ..., 128 bytes, which results in a heap block - // record that contains a mix of sample and non-sampled blocks, and so should - // be printed with '~' signs. - for (int i = 1; i <= seven + 1; i++) { - s = (char*) replace_malloc(i * 16); - UseItOrLoseIt(s, seven); - } - MOZ_ASSERT(gSmallBlockActualSizeCounter == 64); - - // At the end we're 64 bytes into the current sample so we report ~1,424 - // bytes of allocation overall, which is 64 less than the real value 1,488. - - // AnalyzeReports 4. - JSONWriter writer4(Move(aF4)); - AnalyzeReports(writer4); + gSmallBlockActualSizeCounter = 0; } } // namespace dmd } // namespace mozilla
--- a/memory/replace/dmd/DMD.h +++ b/memory/replace/dmd/DMD.h @@ -142,12 +142,20 @@ SizeOf(Sizes* aSizes); // Prints a status message prefixed with "DMD[<pid>]". Use sparingly. MOZ_EXPORT void StatusMsg(const char* aFmt, ...); // Indicates whether or not DMD is running. MOZ_EXPORT bool IsRunning(); +// Sets the sample-below size. Only used for testing purposes. +MOZ_EXPORT void +SetSampleBelowSize(size_t aSize); + +// Clears all records of live allocations. Only used for testing purposes. +MOZ_EXPORT void +ClearBlocks(); + } // namespace mozilla } // namespace dmd #endif /* DMD_h___ */
--- a/memory/replace/dmd/dmd.py +++ b/memory/replace/dmd/dmd.py @@ -1,9 +1,9 @@ -#! /usr/bin/python +#! /usr/bin/env python # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. '''This script analyzes a JSON file emitted by DMD.''' from __future__ import print_function, division @@ -339,25 +339,27 @@ def main(): def out(*arguments, **kwargs): print(*arguments, file=args.output, **kwargs) def printStack(traceTable, frameTable, traceKey): frameKeys = traceTable[traceKey] fmt = ' #{:02d}{:}' if args.filter_stacks_for_testing: - # If any frame has "DMD.cpp" or "replace_malloc.c" in its - # description -- as should be the case for every stack trace when - # running DMD in test mode -- we replace the entire trace with a - # single, predictable frame. There is too much variation in the - # stack traces across different machines and platforms to do more - # specific matching. + # When running SmokeDMD.cpp, every stack trace should contain at + # least one frame that contains 'DMD.cpp', from either |DMD.cpp| or + # |SmokeDMD.cpp|. (Or 'dmd.cpp' on Windows.) If we see such a + # frame, we replace the entire stack trace with a single, + # predictable frame. There is too much variation in the stack + # traces across different machines and platforms to do more precise + # matching, but this level of matching will result in failure if + # stack fixing fails completely. for frameKey in frameKeys: frameDesc = frameTable[frameKey] - if 'DMD.cpp' in frameDesc or 'replace_malloc.c' in frameDesc: + if 'DMD.cpp' in frameDesc or 'dmd.cpp' in frameDesc: out(fmt.format(1, ': ... DMD.cpp ...')) return # The frame number is always '#00' (see DMD.h for why), so we have to # replace that with the correct frame number. for n, frameKey in enumerate(traceTable[traceKey], start=1): out(fmt.format(n, frameTable[frameKey][3:]))
--- a/memory/replace/dmd/moz.build +++ b/memory/replace/dmd/moz.build @@ -28,12 +28,10 @@ if CONFIG['MOZ_OPTIMIZE']: DISABLE_STL_WRAPPING = True if CONFIG['OS_ARCH'] == 'WINNT': OS_LIBS += [ 'dbghelp', ] -XPCSHELL_TESTS_MANIFESTS += [ - 'test/xpcshell.ini', -] +TEST_DIRS += ['test']
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/SmokeDMD.cpp @@ -0,0 +1,325 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This program is used by the DMD xpcshell test. It is run under DMD and +// produces some output. The xpcshell test then post-processes and checks this +// output. +// +// Note that this file does not have "Test" or "test" in its name, because that +// will cause the build system to not record breakpad symbols for it, which +// will stop the post-processing (which includes stack fixing) from working +// correctly. + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> + +#include "mozilla/Assertions.h" +#include "mozilla/JSONWriter.h" +#include "mozilla/UniquePtr.h" +#include "DMD.h" + +using mozilla::JSONWriter; +using mozilla::MakeUnique; +using namespace mozilla::dmd; + +class FpWriteFunc : public mozilla::JSONWriteFunc +{ +public: + explicit FpWriteFunc(const char* aFilename) + { + mFp = fopen(aFilename, "w"); + if (!mFp) { + fprintf(stderr, "SmokeDMD: can't create %s file: %s\n", + aFilename, strerror(errno)); + exit(1); + } + } + + ~FpWriteFunc() { fclose(mFp); } + + void Write(const char* aStr) { fputs(aStr, mFp); } + +private: + FILE* mFp; +}; + +// This stops otherwise-unused variables from being optimized away. +static void +UseItOrLoseIt(void* aPtr, int aSeven) +{ + char buf[64]; + int n = sprintf(buf, "%p\n", aPtr); + if (n == 20 + aSeven) { + fprintf(stderr, "well, that is surprising"); + } +} + +// This function checks that heap blocks that have the same stack trace but +// different (or no) reporters get aggregated separately. +void Foo(int aSeven) +{ + char* a[6]; + for (int i = 0; i < aSeven - 1; i++) { + a[i] = (char*) malloc(128 - 16*i); + } + + for (int i = 0; i < aSeven - 5; i++) { + Report(a[i]); // reported + } + Report(a[2]); // reported + Report(a[3]); // reported + // a[4], a[5] unreported +} + +void +RunTests() +{ + // These files are written to $CWD. + auto f1 = MakeUnique<FpWriteFunc>("full-empty.json"); + auto f2 = MakeUnique<FpWriteFunc>("full-unsampled1.json"); + auto f3 = MakeUnique<FpWriteFunc>("full-unsampled2.json"); + auto f4 = MakeUnique<FpWriteFunc>("full-sampled.json"); + + // This test relies on the compiler not doing various optimizations, such as + // eliding unused malloc() calls or unrolling loops with fixed iteration + // counts. So we compile it with -O0 (or equivalent), which probably prevents + // that. We also use the following variable for various loop iteration + // counts, just in case compilers might unroll very small loops even with + // -O0. + int seven = 7; + + // Make sure that DMD is actually running; it is initialized on the first + // allocation. + int *x = (int*)malloc(100); + UseItOrLoseIt(x, seven); + MOZ_RELEASE_ASSERT(IsRunning()); + + // The first part of this test requires sampling to be disabled. + SetSampleBelowSize(1); + + // The file manipulations above may have done some heap allocations. + // Clear all knowledge of existing blocks to give us a clean slate. + ClearBlocks(); + + //--------- + + // AnalyzeReports 1. Zero for everything. + JSONWriter writer1(Move(f1)); + AnalyzeReports(writer1); + + //--------- + + // AnalyzeReports 2: 1 freed, 9 out of 10 unreported. + // AnalyzeReports 3: still present and unreported. + int i; + char* a = nullptr; + for (i = 0; i < seven + 3; i++) { + a = (char*) malloc(100); + UseItOrLoseIt(a, seven); + } + free(a); + + // Note: 8 bytes is the smallest requested size that gives consistent + // behaviour across all platforms with jemalloc. + // AnalyzeReports 2: reported. + // AnalyzeReports 3: thrice-reported. + char* a2 = (char*) malloc(8); + Report(a2); + + // AnalyzeReports 2: reported. + // AnalyzeReports 3: reportedness carries over, due to ReportOnAlloc. + char* b = (char*) malloc(10); + ReportOnAlloc(b); + + // ReportOnAlloc, then freed. + // AnalyzeReports 2: freed, irrelevant. + // AnalyzeReports 3: freed, irrelevant. + char* b2 = (char*) malloc(1); + ReportOnAlloc(b2); + free(b2); + + // AnalyzeReports 2: reported 4 times. + // AnalyzeReports 3: freed, irrelevant. + char* c = (char*) calloc(10, 3); + Report(c); + for (int i = 0; i < seven - 4; i++) { + Report(c); + } + + // AnalyzeReports 2: ignored. + // AnalyzeReports 3: irrelevant. + Report((void*)(intptr_t)i); + + // jemalloc rounds this up to 8192. + // AnalyzeReports 2: reported. + // AnalyzeReports 3: freed. + char* e = (char*) malloc(4096); + e = (char*) realloc(e, 4097); + Report(e); + + // First realloc is like malloc; second realloc is shrinking. + // AnalyzeReports 2: reported. + // AnalyzeReports 3: re-reported. + char* e2 = (char*) realloc(nullptr, 1024); + e2 = (char*) realloc(e2, 512); + Report(e2); + + // First realloc is like malloc; second realloc creates a min-sized block. + // XXX: on Windows, second realloc frees the block. + // AnalyzeReports 2: reported. + // AnalyzeReports 3: freed, irrelevant. + char* e3 = (char*) realloc(nullptr, 1023); +//e3 = (char*) realloc(e3, 0); + MOZ_ASSERT(e3); + Report(e3); + + // AnalyzeReports 2: freed, irrelevant. + // AnalyzeReports 3: freed, irrelevant. + char* f = (char*) malloc(64); + free(f); + + // AnalyzeReports 2: ignored. + // AnalyzeReports 3: irrelevant. + Report((void*)(intptr_t)0x0); + + // AnalyzeReports 2: mixture of reported and unreported. + // AnalyzeReports 3: all unreported. + Foo(seven); + + // AnalyzeReports 2: twice-reported. + // AnalyzeReports 3: twice-reported. + char* g1 = (char*) malloc(77); + ReportOnAlloc(g1); + ReportOnAlloc(g1); + + // AnalyzeReports 2: mixture of reported and unreported. + // AnalyzeReports 3: all unreported. + // Nb: this Foo() call is not adjacent to the previous one, because that has + // been seen to cause compilers to give them the same stacks, which can break + // the test. + Foo(seven); + + // AnalyzeReports 2: twice-reported. + // AnalyzeReports 3: once-reported. + char* g2 = (char*) malloc(78); + Report(g2); + ReportOnAlloc(g2); + + // AnalyzeReports 2: twice-reported. + // AnalyzeReports 3: once-reported. + char* g3 = (char*) malloc(79); + ReportOnAlloc(g3); + Report(g3); + + // All the odd-ball ones. + // AnalyzeReports 2: all unreported. + // AnalyzeReports 3: all freed, irrelevant. + // XXX: no memalign on Mac +//void* w = memalign(64, 65); // rounds up to 128 +//UseItOrLoseIt(w, seven); + + // XXX: posix_memalign doesn't work on B2G +//void* x; +//posix_memalign(&y, 128, 129); // rounds up to 256 +//UseItOrLoseIt(x, seven); + + // XXX: valloc doesn't work on Windows. +//void* y = valloc(1); // rounds up to 4096 +//UseItOrLoseIt(y, seven); + + // XXX: C11 only +//void* z = aligned_alloc(64, 256); +//UseItOrLoseIt(z, seven); + + // AnalyzeReports 2. + JSONWriter writer2(Move(f2)); + AnalyzeReports(writer2); + + //--------- + + Report(a2); + Report(a2); + free(c); + free(e); + Report(e2); + free(e3); +//free(w); +//free(x); +//free(y); +//free(z); + + // AnalyzeReports 3. + JSONWriter writer3(Move(f3)); + AnalyzeReports(writer3); + + //--------- + + // The first part of this test requires sampling to be disabled. + SetSampleBelowSize(128); + + // Clear all knowledge of existing blocks to give us a clean slate. + ClearBlocks(); + + char* s; + + // This equals the sample size, and so is reported exactly. It should be + // listed before records of the same size that are sampled. + s = (char*) malloc(128); + UseItOrLoseIt(s, seven); + + // This exceeds the sample size, and so is reported exactly. + s = (char*) malloc(144); + UseItOrLoseIt(s, seven); + + // These together constitute exactly one sample. + for (int i = 0; i < seven + 9; i++) { + s = (char*) malloc(8); + UseItOrLoseIt(s, seven); + } + + // These fall 8 bytes short of a full sample. + for (int i = 0; i < seven + 8; i++) { + s = (char*) malloc(8); + UseItOrLoseIt(s, seven); + } + + // This exceeds the sample size, and so is recorded exactly. + s = (char*) malloc(256); + UseItOrLoseIt(s, seven); + + // This gets more than to a full sample from the |i < seven + 8| loop above. + s = (char*) malloc(96); + UseItOrLoseIt(s, seven); + + // This gets to another full sample. + for (int i = 0; i < seven - 2; i++) { + s = (char*) malloc(8); + UseItOrLoseIt(s, seven); + } + + // This allocates 16, 32, ..., 128 bytes, which results in a heap block + // record that contains a mix of sample and non-sampled blocks, and so should + // be printed with '~' signs. + for (int i = 1; i <= seven + 1; i++) { + s = (char*) malloc(i * 16); + UseItOrLoseIt(s, seven); + } + + // At the end we're 64 bytes into the current sample so we report ~1,424 + // bytes of allocation overall, which is 64 less than the real value 1,488. + + // AnalyzeReports 4. + JSONWriter writer4(Move(f4)); + AnalyzeReports(writer4); +} + +int main() +{ + RunTests(); + + return 0; +}
--- a/memory/replace/dmd/test/full-heap-empty-expected.txt +++ b/memory/replace/dmd/test/full-heap-empty-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- # no live heap blocks #-----------------------------------------------------------------
--- a/memory/replace/dmd/test/full-heap-sampled-expected.txt +++ b/memory/replace/dmd/test/full-heap-sampled-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 128 } #----------------------------------------------------------------- Live { ~4 blocks in heap block record 1 of 7 ~512 bytes (~512 requested / ~0 slop)
--- a/memory/replace/dmd/test/full-heap-unsampled1-expected.txt +++ b/memory/replace/dmd/test/full-heap-unsampled1-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- Live { 1 block in heap block record 1 of 12 8,192 bytes (4,097 requested / 4,095 slop)
--- a/memory/replace/dmd/test/full-heap-unsampled2-expected.txt +++ b/memory/replace/dmd/test/full-heap-unsampled2-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- Live { 9 blocks in heap block record 1 of 9 1,008 bytes (900 requested / 108 slop)
--- a/memory/replace/dmd/test/full-reports-empty-expected.txt +++ b/memory/replace/dmd/test/full-reports-empty-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- # no twice-reported heap blocks #-----------------------------------------------------------------
--- a/memory/replace/dmd/test/full-reports-sampled-expected.txt +++ b/memory/replace/dmd/test/full-reports-sampled-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 128 } #----------------------------------------------------------------- # no twice-reported heap blocks #-----------------------------------------------------------------
--- a/memory/replace/dmd/test/full-reports-unsampled1-expected.txt +++ b/memory/replace/dmd/test/full-reports-unsampled1-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- Twice-reported { 1 block in heap block record 1 of 4 80 bytes (79 requested / 1 slop)
--- a/memory/replace/dmd/test/full-reports-unsampled2-expected.txt +++ b/memory/replace/dmd/test/full-reports-unsampled2-expected.txt @@ -1,12 +1,12 @@ #----------------------------------------------------------------- Invocation { - $DMD = '--mode=test' + $DMD = '1' Sample-below size = 1 } #----------------------------------------------------------------- Twice-reported { 1 block in heap block record 1 of 2 80 bytes (77 requested / 3 slop)
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/moz.build @@ -0,0 +1,26 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SimplePrograms([ + 'SmokeDMD', +]) + +# See the comment at the top of SmokeDMD.cpp:RunTests(). +if CONFIG['OS_ARCH'] == 'WINNT': + CXXFLAGS += ['-Og-'] +else: + CXXFLAGS += ['-O0'] + +DEFINES['MOZ_NO_MOZALLOC'] = True + +DISABLE_STL_WRAPPING = True + +USE_LIBS += ['dmd'] + +XPCSHELL_TESTS_MANIFESTS += [ + 'xpcshell.ini', +] +
--- a/memory/replace/dmd/test/test_dmd.js +++ b/memory/replace/dmd/test/test_dmd.js @@ -10,79 +10,131 @@ const {classes: Cc, interfaces: Ci, util Cu.import("resource://gre/modules/FileUtils.jsm"); // The xpcshell test harness sets PYTHON so we can read it here. let gEnv = Cc["@mozilla.org/process/environment;1"] .getService(Ci.nsIEnvironment); let gPythonName = gEnv.get("PYTHON"); -// If we're testing locally, the script is in "CurProcD". Otherwise, it is in -// another location that we have to find. -let gDmdScriptFile = FileUtils.getFile("CurProcD", ["dmd.py"]); -if (!gDmdScriptFile.exists()) { - gDmdScriptFile = FileUtils.getFile("CurWorkD", []); - while (gDmdScriptFile.path.contains("xpcshell")) { - gDmdScriptFile = gDmdScriptFile.parent; +// If we're testing locally, the executable file is in "CurProcD". Otherwise, +// it is in another location that we have to find. +function getExecutable(aFilename) { + let file = FileUtils.getFile("CurProcD", [aFilename]); + if (!file.exists()) { + file = FileUtils.getFile("CurWorkD", []); + while (file.path.contains("xpcshell")) { + file = file.parent; + } + file.append("bin"); + file.append(aFilename); } - gDmdScriptFile.append("bin"); - gDmdScriptFile.append("dmd.py"); + return file; +} + +let gIsWindows = Cc["@mozilla.org/xre/app-info;1"] + .getService(Ci.nsIXULRuntime).OS === "WINNT"; +let gDmdTestFile = getExecutable("SmokeDMD" + (gIsWindows ? ".exe" : "")); + +let gDmdScriptFile = getExecutable("dmd.py"); + +function readFile(aFile) { + var fstream = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + var cstream = Cc["@mozilla.org/intl/converter-input-stream;1"] + .createInstance(Ci.nsIConverterInputStream); + fstream.init(aFile, -1, 0, 0); + cstream.init(fstream, "UTF-8", 0, 0); + + var data = ""; + let (str = {}) { + let read = 0; + do { + // Read as much as we can and put it in str.value. + read = cstream.readString(0xffffffff, str); + data += str.value; + } while (read != 0); + } + cstream.close(); // this closes fstream + return data.replace(/\r/g, ""); // normalize line endings +} + +function runProcess(aExeFile, aArgs) { + let process = Cc["@mozilla.org/process/util;1"] + .createInstance(Components.interfaces.nsIProcess); + process.init(aExeFile); + process.run(/* blocking = */true, aArgs, aArgs.length); + return process.exitValue; } function test(aJsonFile, aPrefix, aOptions) { // DMD writes the JSON files to CurWorkD, so we do likewise here with // |actualFile| for consistency. It is removed once we've finished. let expectedFile = FileUtils.getFile("CurWorkD", [aPrefix + "-expected.txt"]); let actualFile = FileUtils.getFile("CurWorkD", [aPrefix + "-actual.txt"]); // Run dmd.py on the JSON file, producing |actualFile|. - let pythonFile = new FileUtils.File(gPythonName); - let pythonProcess = Cc["@mozilla.org/process/util;1"] - .createInstance(Components.interfaces.nsIProcess); - pythonProcess.init(pythonFile); - let args = [ gDmdScriptFile.path, "--filter-stacks-for-testing", "-o", actualFile.path ]; args = args.concat(aOptions); args.push(aJsonFile.path); - pythonProcess.run(/* blocking = */true, args, args.length); + runProcess(new FileUtils.File(gPythonName), args); - // Compare |expectedFile| with |actualFile|. Difference are printed to - // stdout. + // Compare |expectedFile| with |actualFile|. We produce nice diffs with + // /usr/bin/diff on systems that have it (Mac and Linux). Otherwise (Windows) + // we do a string compare of the file contents and then print them both if + // they don't match. + + let success; + try { + let rv = runProcess(new FileUtils.File("/usr/bin/diff"), + ["-u", expectedFile.path, actualFile.path]); + success = rv == 0; - let diffFile = new FileUtils.File("/usr/bin/diff"); - let diffProcess = Cc["@mozilla.org/process/util;1"] - .createInstance(Components.interfaces.nsIProcess); - // XXX: this doesn't work on Windows (bug 1076446). - diffProcess.init(diffFile); + } catch (e) { + let expectedData = readFile(expectedFile); + let actualData = readFile(actualFile); + success = expectedData === actualData; + if (!success) { + expectedData = expectedData.split("\n"); + actualData = actualData.split("\n"); + for (let i = 0; i < expectedData.length; i++) { + print("EXPECTED:" + expectedData[i]); + } + for (let i = 0; i < actualData.length; i++) { + print(" ACTUAL:" + actualData[i]); + } + } + } - args = ["-u", expectedFile.path, actualFile.path]; - diffProcess.run(/* blocking = */true, args, args.length); - let success = diffProcess.exitValue == 0; ok(success, aPrefix); actualFile.remove(true); } function run_test() { let jsonFile; // These tests do full end-to-end testing of DMD, i.e. both the C++ code that // generates the JSON output, and the script that post-processes that output. - // The test relies on DMD's test mode executing beforehand, in order to - // produce the relevant JSON files. // // Run these synchronously, because test() updates the full*.json files // in-place (to fix stacks) when it runs dmd.py, and that's not safe to do // asynchronously. + + gEnv.set("DMD", "1"); + gEnv.set(gEnv.get("DMD_PRELOAD_VAR"), gEnv.get("DMD_PRELOAD_VALUE")); + + runProcess(gDmdTestFile, []); + let fullTestNames = ["empty", "unsampled1", "unsampled2", "sampled"]; for (let i = 0; i < fullTestNames.length; i++) { let name = fullTestNames[i]; jsonFile = FileUtils.getFile("CurWorkD", ["full-" + name + ".json"]); test(jsonFile, "full-heap-" + name, ["--ignore-reports"]) test(jsonFile, "full-reports-" + name, []) jsonFile.remove(true); }
--- a/memory/replace/dmd/test/xpcshell.ini +++ b/memory/replace/dmd/test/xpcshell.ini @@ -17,12 +17,13 @@ support-files = script-sort-by-req-expected.txt script-sort-by-slop-expected.txt script-ignore-alloc-fns.json script-ignore-alloc-fns-expected.txt script-show-all-block-sizes.json script-show-all-block-sizes-expected.txt # Bug 1077230 explains why this test is disabled on Mac 10.6. -# Bug 1076446 is open for getting this test working on on Windows. +# Bug 1076446 comment 20 explains why this test is only enabled on Windows 5.1 +# (WinXP) and 6.1 (Win7), but not 6.2 (Win8). [test_dmd.js] dmd = true -run-if = os == 'linux' || os == 'mac' && os_version != '10.6' +run-if = os == 'linux' || os == 'mac' && os_version != '10.6' || os == 'win' && (os_version == '5.1' || os_version == '6.1')
--- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -823,21 +823,19 @@ class RunProgram(MachCommandBase): @CommandArgument('--dmd', action='store_true', group='DMD', help='Enable DMD. The following arguments have no effect without this.') @CommandArgument('--sample-below', default=None, type=str, group='DMD', help='Sample blocks smaller than this. Use 1 for no sampling. The default is 4093.') @CommandArgument('--max-frames', default=None, type=str, group='DMD', help='The maximum depth of stack traces. The default and maximum is 24.') @CommandArgument('--show-dump-stats', action='store_true', group='DMD', help='Show stats when doing dumps.') - @CommandArgument('--mode', choices=['normal', 'test'], group='DMD', - help='Mode of operation. The default is normal.') def run(self, params, remote, background, noprofile, debug, debugger, debugparams, slowscript, dmd, sample_below, max_frames, - show_dump_stats, mode): + show_dump_stats): try: binpath = self.get_binary_path('app') except Exception as e: print("It looks like your program isn't built.", "You can run |mach build| to build it.") print(e) return 1 @@ -897,18 +895,16 @@ class RunProgram(MachCommandBase): dmd_params = [] if sample_below: dmd_params.append('--sample-below=' + sample_below) if max_frames: dmd_params.append('--max-frames=' + max_frames) if show_dump_stats: dmd_params.append('--show-dump-stats=yes') - if mode: - dmd_params.append('--mode=' + mode) if dmd_params: dmd_env_var = " ".join(dmd_params) else: dmd_env_var = "1" bin_dir = os.path.dirname(binpath) lib_name = self.substs['DLL_PREFIX'] + 'dmd' + self.substs['DLL_SUFFIX']
--- a/testing/mochitest/Makefile.in +++ b/testing/mochitest/Makefile.in @@ -58,17 +58,20 @@ endif ifeq (windows,$(MOZ_WIDGET_TOOLKIT)) TEST_HARNESS_BINS += screenshot$(BIN_SUFFIX) ifdef MOZ_METRO TEST_HARNESS_BINS += metrotestharness$(BIN_SUFFIX) endif endif ifdef MOZ_DMD -TEST_HARNESS_BINS += dmd.py +TEST_HARNESS_BINS += \ + dmd.py \ + SmokeDMD$(BIN_SUFFIX) \ + $(NULL) endif # Components / typelibs that don't get packaged with # the build, but that we need for the test harness. TEST_HARNESS_COMPONENTS := \ test_necko.xpt \ $(NULL)
--- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -615,20 +615,20 @@ class XPCShellTestThread(Thread): # self.xrePath is <prefix>/Contents/Resources. # We need <prefix>/Contents/MacOS/libdmd.dylib. contents_dir = os.path.dirname(self.xrePath) libdmd = os.path.join(contents_dir, 'MacOS', 'libdmd.dylib') elif sys.platform == 'win32': preloadEnvVar = 'MOZ_REPLACE_MALLOC_LIB' libdmd = os.path.join(self.xrePath, 'dmd.dll') - self.env['DMD'] = '--mode=test' self.env['PYTHON'] = sys.executable self.env['BREAKPAD_SYMBOLS_PATH'] = self.symbolsPath - self.env[preloadEnvVar] = libdmd + self.env['DMD_PRELOAD_VAR'] = preloadEnvVar + self.env['DMD_PRELOAD_VALUE'] = libdmd testTimeoutInterval = HARNESS_TIMEOUT # Allow a test to request a multiple of the timeout if it is expected to take long if 'requesttimeoutfactor' in self.test_object: testTimeoutInterval *= int(self.test_object['requesttimeoutfactor']) testTimer = None if not self.interactive and not self.debuggerInfo:
--- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -591,18 +591,21 @@ NO_PKG_FILES += \ content_unit_tests \ necko_unit_tests \ *.dSYM \ $(NULL) # If a manifest has not been supplied, the following # files should be excluded from the package too ifndef MOZ_PKG_MANIFEST -NO_PKG_FILES += \ - ssltunnel* +NO_PKG_FILES += ssltunnel* +endif + +ifdef MOZ_DMD +NO_PKG_FILES += SmokeDMD endif # browser/locales/Makefile uses this makefile for its variable defs, but # doesn't want the libs:: rule. ifndef PACKAGER_NO_LIBS libs:: make-package endif