☠☠ backed out by 1e4b92de7bc5 ☠ ☠ | |
author | Nicholas Nethercote <nnethercote@mozilla.com> |
Thu, 16 Oct 2014 19:06:45 -0700 | |
changeset 238475 | 94c5d968e7e802a423c2634ed15bea2dcff2d6ef |
parent 238474 | 835fbe63da4aad49f636b68077a9261b55a54572 |
child 238476 | 450c187cbc1bb6f9b9c92096ef56f63a7d1cafc8 |
push id | 660 |
push user | raliiev@mozilla.com |
push date | Wed, 18 Feb 2015 20:30:48 +0000 |
treeherder | mozilla-release@49e493494178 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | mccr8 |
bugs | 1014343 |
milestone | 36.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/memory/replace/dmd/dmd.py +++ b/memory/replace/dmd/dmd.py @@ -53,43 +53,105 @@ allocatorFns = [ 'pod_realloc', # This one necessary to fully filter some sequences of allocation functions # that happen in practice. Note that ??? entries that follow non-allocation # functions won't be stripped, as explained above. '???', ] class Record(object): + '''A record is an aggregation of heap blocks that have identical stack + traces. It can also be used to represent the difference between two + records.''' + def __init__(self): self.numBlocks = 0 self.reqSize = 0 self.slopSize = 0 self.usableSize = 0 self.isSampled = False + self.allocatedAtDesc = None + self.reportedAtDescs = [] self.usableSizes = collections.defaultdict(int) + def isZero(self, args): + return self.numBlocks == 0 and \ + self.reqSize == 0 and \ + self.slopSize == 0 and \ + self.usableSize == 0 and \ + (not args.show_all_block_sizes or len(self.usableSizes) == 0) + + def negate(self): + self.numBlocks = -self.numBlocks + self.reqSize = -self.reqSize + self.slopSize = -self.slopSize + self.usableSize = -self.usableSize + + negatedUsableSizes = collections.defaultdict(int) + for (usableSize, isSampled), count in self.usableSizes.items(): + negatedUsableSizes[(-usableSize, isSampled)] = count + self.usableSizes = negatedUsableSizes + + def subtract(self, r): + # We should only be calling this on records with matching stack traces. + # Check this. + assert self.allocatedAtDesc == r.allocatedAtDesc + assert self.reportedAtDescs == r.reportedAtDescs + + self.numBlocks -= r.numBlocks + self.reqSize -= r.reqSize + self.slopSize -= r.slopSize + self.usableSize -= r.usableSize + self.isSampled = self.isSampled or r.isSampled + + usableSizes1 = self.usableSizes + usableSizes2 = r.usableSizes + usableSizes3 = collections.defaultdict(int) + for usableSize, isSampled in usableSizes1: + counts1 = usableSizes1[usableSize, isSampled] + if (usableSize, isSampled) in usableSizes2: + counts2 = usableSizes2[usableSize, isSampled] + del usableSizes2[usableSize, isSampled] + counts3 = counts1 - counts2 + if counts3 != 0: + if counts3 < 0: + usableSize = -usableSize + counts3 = -counts3 + usableSizes3[usableSize, isSampled] = counts3 + else: + usableSizes3[usableSize, isSampled] = counts1 + + for usableSize, isSampled in usableSizes2: + usableSizes3[-usableSize, isSampled] = \ + usableSizes2[usableSize, isSampled] + + self.usableSizes = usableSizes3 + @staticmethod def cmpByIsSampled(r1, r2): # Treat sampled as smaller than non-sampled. return cmp(r2.isSampled, r1.isSampled) @staticmethod def cmpByUsableSize(r1, r2): # Sort by usable size, then req size, then by isSampled. - return cmp(r1.usableSize, r2.usableSize) or Record.cmpByReqSize(r1, r2) + return cmp(abs(r1.usableSize), abs(r2.usableSize)) or \ + Record.cmpByReqSize(r1, r2) @staticmethod def cmpByReqSize(r1, r2): # Sort by req size, then by isSampled. - return cmp(r1.reqSize, r2.reqSize) or Record.cmpByIsSampled(r1, r2) + return cmp(abs(r1.reqSize), abs(r2.reqSize)) or \ + Record.cmpByIsSampled(r1, r2) @staticmethod def cmpBySlopSize(r1, r2): # Sort by slop size, then by isSampled. - return cmp(r1.slopSize, r2.slopSize) or Record.cmpByIsSampled(r1, r2) + return cmp(abs(r1.slopSize), abs(r2.slopSize)) or \ + Record.cmpByIsSampled(r1, r2) sortByChoices = { 'usable': Record.cmpByUsableSize, # the default 'req': Record.cmpByReqSize, 'slop': Record.cmpBySlopSize, } @@ -100,17 +162,19 @@ def parseCommandLine(): value = int(string) if value < 1 or value > 24: msg = '{:s} is not in the range 1..24'.format(string) raise argparse.ArgumentTypeError(msg) return value description = ''' Analyze heap data produced by DMD. -If no files are specified, read from stdin; input can be gzipped. +If one file is specified, analyze it; if two files are specified, analyze the +difference. +Input files can be gzipped. Write to stdout unless -o/--output is specified. Stack traces are fixed to show function names, filenames and line numbers unless --no-fix-stacks is specified; stack fixing modifies the original file and may take some time. If specified, the BREAKPAD_SYMBOLS_PATH environment variable is used to find breakpad symbols for stack fixing. ''' p = argparse.ArgumentParser(description=description) @@ -135,17 +199,21 @@ variable is used to find breakpad symbol help='show individual block sizes for each record') p.add_argument('--no-fix-stacks', action='store_true', help='do not fix stacks') p.add_argument('--filter-stacks-for-testing', action='store_true', help='filter stack traces; only useful for testing purposes') - p.add_argument('input_file') + p.add_argument('input_file', + help='a file produced by DMD') + + p.add_argument('input_file2', nargs='?', + help='a file produced by DMD; if present, it is diff\'d with input_file') return p.parse_args(sys.argv[1:]) # Fix stacks if necessary: first write the output to a tempfile, then replace # the original file with it. def fixStackTraces(inputFilename, isZipped, opener): # This append() call is needed to make the import statements work when this @@ -190,28 +258,26 @@ def fixStackTraces(inputFilename, isZipp for line in inputFile: tmpFile.write(fix(line)) tmpFile.close() shutil.move(tmpFilename, inputFilename) -def main(): - args = parseCommandLine() - +def getDigestFromFile(args, inputFile): # Handle gzipped input if necessary. - isZipped = args.input_file.endswith('.gz') + isZipped = inputFile.endswith('.gz') opener = gzip.open if isZipped else open # Fix stack traces unless otherwise instructed. if not args.no_fix_stacks: - fixStackTraces(args.input_file, isZipped, opener) + fixStackTraces(inputFile, isZipped, opener) - with opener(args.input_file, 'rb') as f: + with opener(inputFile, 'rb') as f: j = json.load(f) if j['version'] != outputVersion: raise Exception("'version' property isn't '{:d}'".format(outputVersion)) # Extract the main parts of the JSON object. invocation = j['invocation'] dmdEnvVar = invocation['dmdEnvVar'] @@ -240,16 +306,41 @@ def main(): if numSkippedFrames > 0: traceTable[traceKey] = frameKeys[numSkippedFrames:] # Trim the number of frames. for traceKey, frameKeys in traceTable.items(): if len(frameKeys) > args.max_frames: traceTable[traceKey] = frameKeys[:args.max_frames] + def buildTraceDescription(traceTable, frameTable, traceKey): + frameKeys = traceTable[traceKey] + fmt = ' #{:02d}{:}' + + if args.filter_stacks_for_testing: + # When running SmokeDMD.cpp, every stack trace should contain at + # least one frame that contains 'DMD.cpp', from either |DMD.cpp| or + # |SmokeDMD.cpp|. (Or 'dmd.cpp' on Windows.) If we see such a + # frame, we replace the entire stack trace with a single, + # predictable frame. There is too much variation in the stack + # traces across different machines and platforms to do more precise + # matching, but this level of matching will result in failure if + # stack fixing fails completely. + for frameKey in frameKeys: + frameDesc = frameTable[frameKey] + if 'DMD.cpp' in frameDesc or 'dmd.cpp' in frameDesc: + return [fmt.format(1, ': ... DMD.cpp ...')] + + # The frame number is always '#00' (see DMD.h for why), so we have to + # replace that with the correct frame number. + desc = [] + for n, frameKey in enumerate(traceTable[traceKey], start=1): + desc.append(fmt.format(n, frameTable[frameKey][3:])) + return desc + # Aggregate blocks into records. All sufficiently similar blocks go into a # single record. if args.ignore_reports: liveRecords = collections.defaultdict(Record) else: unreportedRecords = collections.defaultdict(Record) onceReportedRecords = collections.defaultdict(Record) @@ -259,34 +350,43 @@ def main(): heapBlocks = 0 for block in blockList: # For each block we compute a |recordKey|, and all blocks with the same # |recordKey| are aggregated into a single record. The |recordKey| is # derived from the block's 'alloc' and 'reps' (if present) stack # traces. # - # Each stack trace has a key in the JSON file. But we don't use that - # key to construct |recordKey|; instead we use the frame keys. - # This is because the stack trimming done for --max-frames can cause - # stack traces with distinct trace keys to end up with the same frame - # keys, and these should be considered equivalent. E.g. if we have - # distinct traces T1:[A,B,C] and T2:[A,B,D] and we trim the final frame - # of each they should be considered equivalent. - allocatedAt = block['alloc'] + # We use frame descriptions (e.g. "#00: foo (X.cpp:99)") when comparing + # traces for equality. We can't use trace keys or frame keys because + # they're not comparable across different DMD runs (which is relevant + # when doing diffs). + # + # Using frame descriptions also fits in with the stack trimming done + # for --max-frames, which requires that stack traces with common + # beginnings but different endings to be considered equivalent. E.g. if + # we have distinct traces T1:[A:D1,B:D2,C:D3] and T2:[X:D1,Y:D2,Z:D4] + # and we trim the final frame of each they should be considered + # equivalent because the untrimmed frame descriptions (D1 and D2) + # match. + def makeRecordKeyPart(traceKey): + return str(map(lambda frameKey: frameTable[frameKey], + traceTable[traceKey])) + + allocatedAtTraceKey = block['alloc'] if args.ignore_reports: - recordKey = str(traceTable[allocatedAt]) + recordKey = makeRecordKeyPart(allocatedAtTraceKey) records = liveRecords else: - recordKey = str(traceTable[allocatedAt]) + recordKey = makeRecordKeyPart(allocatedAtTraceKey) if 'reps' in block: - reportedAts = block['reps'] - for reportedAt in reportedAts: - recordKey += str(traceTable[reportedAt]) - if len(reportedAts) == 1: + reportedAtTraceKeys = block['reps'] + for reportedAtTraceKey in reportedAtTraceKeys: + recordKey += makeRecordKeyPart(reportedAtTraceKey) + if len(reportedAtTraceKeys) == 1: records = onceReportedRecords else: records = twiceReportedRecords else: records = unreportedRecords record = records[recordKey] @@ -307,25 +407,102 @@ def main(): heapUsableSize += usableSize heapBlocks += 1 record.numBlocks += 1 record.reqSize += reqSize record.slopSize += slopSize record.usableSize += usableSize record.isSampled = record.isSampled or isSampled - record.allocatedAt = block['alloc'] + if record.allocatedAtDesc == None: + record.allocatedAtDesc = \ + buildTraceDescription(traceTable, frameTable, + allocatedAtTraceKey) + if args.ignore_reports: pass else: - if 'reps' in block: - record.reportedAts = block['reps'] + if 'reps' in block and record.reportedAtDescs == []: + f = lambda k: buildTraceDescription(traceTable, frameTable, k) + record.reportedAtDescs = map(f, reportedAtTraceKeys) record.usableSizes[(usableSize, isSampled)] += 1 - # Print records. + # All the processed data for a single DMD file is called a "digest". + digest = {} + digest['dmdEnvVar'] = dmdEnvVar + digest['sampleBelowSize'] = sampleBelowSize + digest['heapUsableSize'] = heapUsableSize + digest['heapBlocks'] = heapBlocks + digest['heapIsSampled'] = heapIsSampled + if args.ignore_reports: + digest['liveRecords'] = liveRecords + else: + digest['unreportedRecords'] = unreportedRecords + digest['onceReportedRecords'] = onceReportedRecords + digest['twiceReportedRecords'] = twiceReportedRecords + return digest + + +def diffRecords(args, records1, records2): + records3 = {} + + # Process records1. + for k in records1: + r1 = records1[k] + if k in records2: + # This record is present in both records1 and records2. + r2 = records2[k] + del records2[k] + r2.subtract(r1) + if not r2.isZero(args): + records3[k] = r2 + else: + # This record is present only in records1. + r1.negate() + records3[k] = r1 + + for k in records2: + # This record is present only in records2. + records3[k] = records2[k] + + return records3 + + +def diffDigests(args, d1, d2): + d3 = {} + d3['dmdEnvVar'] = (d1['dmdEnvVar'], d2['dmdEnvVar']) + d3['sampleBelowSize'] = (d1['sampleBelowSize'], d2['sampleBelowSize']) + d3['heapUsableSize'] = d2['heapUsableSize'] - d1['heapUsableSize'] + d3['heapBlocks'] = d2['heapBlocks'] - d1['heapBlocks'] + d3['heapIsSampled'] = d2['heapIsSampled'] or d1['heapIsSampled'] + if args.ignore_reports: + d3['liveRecords'] = diffRecords(args, d1['liveRecords'], + d2['liveRecords']) + else: + d3['unreportedRecords'] = diffRecords(args, d1['unreportedRecords'], + d2['unreportedRecords']) + d3['onceReportedRecords'] = diffRecords(args, d1['onceReportedRecords'], + d2['onceReportedRecords']) + d3['twiceReportedRecords'] = diffRecords(args, d1['twiceReportedRecords'], + d2['twiceReportedRecords']) + return d3 + + +def printDigest(args, digest): + dmdEnvVar = digest['dmdEnvVar'] + sampleBelowSize = digest['sampleBelowSize'] + heapUsableSize = digest['heapUsableSize'] + heapIsSampled = digest['heapIsSampled'] + heapBlocks = digest['heapBlocks'] + if args.ignore_reports: + liveRecords = digest['liveRecords'] + else: + unreportedRecords = digest['unreportedRecords'] + onceReportedRecords = digest['onceReportedRecords'] + twiceReportedRecords = digest['twiceReportedRecords'] separator = '#' + '-' * 65 + '\n' def number(n, isSampled): '''Format a number, with comma as a separator and a '~' prefix if it's sampled.''' return '{:}{:,d}'.format('~' if isSampled else '', n) @@ -334,39 +511,19 @@ def main(): def plural(n): return '' if n == 1 else 's' # Prints to stdout, or to file if -o/--output was specified. def out(*arguments, **kwargs): print(*arguments, file=args.output, **kwargs) - def printStack(traceTable, frameTable, traceKey): - frameKeys = traceTable[traceKey] - fmt = ' #{:02d}{:}' - - if args.filter_stacks_for_testing: - # When running SmokeDMD.cpp, every stack trace should contain at - # least one frame that contains 'DMD.cpp', from either |DMD.cpp| or - # |SmokeDMD.cpp|. (Or 'dmd.cpp' on Windows.) If we see such a - # frame, we replace the entire stack trace with a single, - # predictable frame. There is too much variation in the stack - # traces across different machines and platforms to do more precise - # matching, but this level of matching will result in failure if - # stack fixing fails completely. - for frameKey in frameKeys: - frameDesc = frameTable[frameKey] - if 'DMD.cpp' in frameDesc or 'dmd.cpp' in frameDesc: - out(fmt.format(1, ': ... DMD.cpp ...')) - return - - # The frame number is always '#00' (see DMD.h for why), so we have to - # replace that with the correct frame number. - for n, frameKey in enumerate(traceTable[traceKey], start=1): - out(fmt.format(n, frameTable[frameKey][3:])) + def printStack(traceDesc): + for frameDesc in traceDesc: + out(frameDesc) def printRecords(recordKind, records, heapUsableSize): RecordKind = recordKind.capitalize() out(separator) numRecords = len(records) cmpRecords = sortByChoices[args.sort_by] sortedRecords = sorted(records.values(), cmp=cmpRecords, reverse=True) kindBlocks = 0 @@ -409,66 +566,79 @@ def main(): pass else: out(' {:4.2f}% of {:} ({:4.2f}% cumulative)'. format(perc(record.usableSize, kindUsableSize), recordKind, perc(kindCumulativeUsableSize, kindUsableSize))) if args.show_all_block_sizes: - usableSizes = sorted(record.usableSizes.items(), reverse=True) + abscmp = lambda ((usableSize1, _1a), _1b), \ + ((usableSize2, _2a), _2b): \ + cmp(abs(usableSize1), abs(usableSize2)) + usableSizes = sorted(record.usableSizes.items(), cmp=abscmp, + reverse=True) out(' Individual block sizes: ', end='') - isFirst = True - for (usableSize, isSampled), count in usableSizes: - if not isFirst: - out('; ', end='') - out('{:}'.format(number(usableSize, isSampled)), end='') - if count > 1: - out(' x {:,d}'.format(count), end='') - isFirst = False + if len(usableSizes) == 0: + out('(no change)', end='') + else: + isFirst = True + for (usableSize, isSampled), count in usableSizes: + if not isFirst: + out('; ', end='') + out('{:}'.format(number(usableSize, isSampled)), end='') + if count > 1: + out(' x {:,d}'.format(count), end='') + isFirst = False out() out(' Allocated at {') - printStack(traceTable, frameTable, record.allocatedAt) + printStack(record.allocatedAtDesc) out(' }') if args.ignore_reports: pass else: - if hasattr(record, 'reportedAts'): - for n, reportedAt in enumerate(record.reportedAts): - again = 'again ' if n > 0 else '' - out(' Reported {:}at {{'.format(again)) - printStack(traceTable, frameTable, reportedAt) - out(' }') + for n, reportedAtDesc in enumerate(record.reportedAtDescs): + again = 'again ' if n > 0 else '' + out(' Reported {:}at {{'.format(again)) + printStack(reportedAtDesc) + out(' }') out('}\n') return (kindUsableSize, kindBlocks) - # Print header. + def printInvocation(n, dmdEnvVar, sampleBelowSize): + out('Invocation{:} {{'.format(n)) + out(' $DMD = \'' + dmdEnvVar + '\'') + out(' Sample-below size = ' + str(sampleBelowSize)) + out('}\n') + + # Print invocation(s). out(separator) - out('Invocation {') - out(' $DMD = \'' + dmdEnvVar + '\'') - out(' Sample-below size = ' + str(sampleBelowSize)) - out('}\n') + if type(dmdEnvVar) is not tuple: + printInvocation('', dmdEnvVar, sampleBelowSize) + else: + printInvocation(' 1', dmdEnvVar[0], sampleBelowSize[0]) + printInvocation(' 2', dmdEnvVar[1], sampleBelowSize[1]) # Print records. if args.ignore_reports: liveUsableSize, liveBlocks = \ printRecords('live', liveRecords, heapUsableSize) else: twiceReportedUsableSize, twiceReportedBlocks = \ printRecords('twice-reported', twiceReportedRecords, heapUsableSize) unreportedUsableSize, unreportedBlocks = \ - printRecords('unreported', unreportedRecords, heapUsableSize) + printRecords('unreported', unreportedRecords, heapUsableSize) onceReportedUsableSize, onceReportedBlocks = \ - printRecords('once-reported', onceReportedRecords, heapUsableSize) + printRecords('once-reported', onceReportedRecords, heapUsableSize) # Print summary. out(separator) out('Summary {') if args.ignore_reports: out(' Total: {:} bytes in {:} blocks'. format(number(liveUsableSize, heapIsSampled), number(liveBlocks, heapIsSampled))) @@ -496,10 +666,20 @@ def main(): format('Twice-reported:', number(twiceReportedUsableSize, heapIsSampled), perc(twiceReportedUsableSize, heapUsableSize), number(twiceReportedBlocks, heapIsSampled), perc(twiceReportedBlocks, heapBlocks))) out('}\n') +def main(): + args = parseCommandLine() + digest = getDigestFromFile(args, args.input_file) + if args.input_file2: + digest2 = getDigestFromFile(args, args.input_file2) + digest = diffDigests(args, digest, digest2) + printDigest(args, digest) + + if __name__ == '__main__': main() +
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/script-diff-basic-expected.txt @@ -0,0 +1,109 @@ +#----------------------------------------------------------------- + +Invocation 1 { + $DMD = '--sample-below=127' + Sample-below size = 127 +} + +Invocation 2 { + $DMD = '--sample-below=63' + Sample-below size = 63 +} + +#----------------------------------------------------------------- + +Twice-reported { + ~-1 blocks in heap block record 1 of 1 + ~-1,088 bytes (~-1,064 requested / ~-24 slop) + 15.46% of the heap (15.46% cumulative) + 100.00% of twice-reported (100.00% cumulative) + Allocated at { + #01: F (F.cpp:99) + } + Reported at { + #01: R1 (R1.cpp:99) + } + Reported again at { + #01: R2 (R2.cpp:99) + } +} + +#----------------------------------------------------------------- + +Unreported { + 4 blocks in heap block record 1 of 4 + 16,384 bytes (16,384 requested / 0 slop) + -232.76% of the heap (-232.76% cumulative) + 371.01% of unreported (371.01% cumulative) + Allocated at { + #01: E (E.cpp:99) + } +} + +Unreported { + ~7 blocks in heap block record 2 of 4 + ~-11,968 bytes (~-12,016 requested / ~48 slop) + 170.02% of the heap (-62.74% cumulative) + -271.01% of unreported (100.00% cumulative) + Allocated at { + #01: F (F.cpp:99) + } +} + +Unreported { + 0 blocks in heap block record 3 of 4 + 0 bytes (-384 requested / 384 slop) + -0.00% of the heap (-62.74% cumulative) + 0.00% of unreported (100.00% cumulative) + Allocated at { + #01: C (C.cpp:99) + } +} + +Unreported { + -2 blocks in heap block record 4 of 4 + 0 bytes (0 requested / 0 slop) + -0.00% of the heap (-62.74% cumulative) + 0.00% of unreported (100.00% cumulative) + Allocated at { + #01: B (B.cpp:99) + } +} + +#----------------------------------------------------------------- + +Once-reported { + -3 blocks in heap block record 1 of 2 + -10,240 bytes (-10,192 requested / -48 slop) + 145.48% of the heap (145.48% cumulative) + 98.77% of once-reported (98.77% cumulative) + Allocated at { + #01: D (D.cpp:99) + } + Reported at { + #01: R1 (R1.cpp:99) + } +} + +Once-reported { + ~-1 blocks in heap block record 2 of 2 + ~-127 bytes (~-151 requested / ~24 slop) + 1.80% of the heap (147.28% cumulative) + 1.23% of once-reported (100.00% cumulative) + Allocated at { + #01: F (F.cpp:99) + } + Reported at { + #01: R1 (R1.cpp:99) + } +} + +#----------------------------------------------------------------- + +Summary { + Total: ~-7,039 bytes (100.00%) in ~4 blocks (100.00%) + Unreported: ~4,416 bytes (-62.74%) in ~9 blocks (225.00%) + Once-reported: ~-10,367 bytes (147.28%) in ~-4 blocks (-100.00%) + Twice-reported: ~-1,088 bytes ( 15.46%) in ~-1 blocks (-25.00%) +} +
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/script-diff-options-expected.txt @@ -0,0 +1,80 @@ +#----------------------------------------------------------------- + +Invocation 1 { + $DMD = '--sample-below=127' + Sample-below size = 127 +} + +Invocation 2 { + $DMD = '--sample-below=63' + Sample-below size = 63 +} + +#----------------------------------------------------------------- + +Live { + 4 blocks in heap block record 1 of 6 + 16,384 bytes (16,384 requested / 0 slop) + -232.76% of the heap (-232.76% cumulative) + Individual block sizes: 4,096 x 4 + Allocated at { + #01: E (E.cpp:99) + } +} + +Live { + ~5 blocks in heap block record 2 of 6 + ~-13,183 bytes (~-13,231 requested / ~48 slop) + 187.29% of the heap (-45.48% cumulative) + Individual block sizes: -15,360; 2,048; -1,024; 512 x 2; 128; ~-127 x 3; 64 x 4; ~63 x 2 + Allocated at { + #01: F (F.cpp:99) + } +} + +Live { + -3 blocks in heap block record 3 of 6 + -10,240 bytes (-10,192 requested / -48 slop) + 145.48% of the heap (100.00% cumulative) + Individual block sizes: -4,096 x 2; -2,048 + Allocated at { + #01: D (D.cpp:99) + } +} + +Live { + 0 blocks in heap block record 4 of 6 + 0 bytes (-384 requested / 384 slop) + -0.00% of the heap (100.00% cumulative) + Individual block sizes: (no change) + Allocated at { + #01: C (C.cpp:99) + } +} + +Live { + 0 blocks in heap block record 5 of 6 + 0 bytes (0 requested / 0 slop) + -0.00% of the heap (100.00% cumulative) + Individual block sizes: 20,480; -16,384; -8,192; 4,096 + Allocated at { + #01: G (G.cpp:99) + } +} + +Live { + -2 blocks in heap block record 6 of 6 + 0 bytes (0 requested / 0 slop) + -0.00% of the heap (100.00% cumulative) + Individual block sizes: 8,192 x 2; -4,096 x 4 + Allocated at { + #01: B (B.cpp:99) + } +} + +#----------------------------------------------------------------- + +Summary { + Total: ~-7,039 bytes in ~4 blocks +} +
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/script-diff1.json @@ -0,0 +1,62 @@ +{ + "version": 1, + "invocation": { + "dmdEnvVar": "--sample-below=127", + "sampleBelowSize": 127 + }, + "blockList": [ + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + + {"req": 4096, "alloc": "B"}, + {"req": 4096, "alloc": "B"}, + {"req": 4096, "alloc": "B"}, + {"req": 4096, "alloc": "B"}, + + {"req": 4096, "alloc": "C"}, + {"req": 4096, "alloc": "C"}, + {"req": 4096, "alloc": "C"}, + {"req": 4096, "alloc": "C"}, + + {"req": 4096, "alloc": "D", "reps": ["R1"]}, + {"req": 4096, "alloc": "D", "reps": ["R1"]}, + {"req": 2000, "slop": 48, "alloc": "D", "reps": ["R1"]}, + + {"req": 15360, "alloc": "F"}, + {"req": 512, "alloc": "F"}, + {"req": 512, "alloc": "F"}, + { "alloc": "F"}, + {"req": 1024, "alloc": "F", "reps": ["R1"]}, + { "alloc": "F", "reps": ["R1"]}, + {"req": 1000, "slop": 24, "alloc": "F", "reps": ["R1", "R2"]}, + { "alloc": "F", "reps": ["R1", "R2"]}, + + {"req": 4096, "alloc": "G"}, + {"req": 8192, "alloc": "G"}, + {"req": 16384, "alloc": "G"} + ], + "traceTable": { + "A": ["AA"], + "B": ["BB"], + "C": ["CC"], + "D": ["DD"], + "E": ["EE"], + "F": ["FF"], + "G": ["GG"], + "R1": ["RR1"], + "R2": ["RR2"] + }, + "frameTable": { + "AA": "#00: A (A.cpp:99)", + "BB": "#00: B (B.cpp:99)", + "CC": "#00: C (C.cpp:99)", + "DD": "#00: D (D.cpp:99)", + "EE": "#00: E (E.cpp:99)", + "FF": "#00: F (F.cpp:99)", + "GG": "#00: G (G.cpp:99)", + "RR1": "#00: R1 (R1.cpp:99)", + "RR2": "#00: R2 (R2.cpp:99)" + } +}
new file mode 100644 --- /dev/null +++ b/memory/replace/dmd/test/script-diff2.json @@ -0,0 +1,66 @@ +{ + "version": 1, + "invocation": { + "dmdEnvVar": "--sample-below=63", + "sampleBelowSize": 63 + }, + "blockList": [ + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + {"req": 4096, "alloc": "A"}, + + {"req": 8192, "alloc": "B"}, + {"req": 8192, "alloc": "B"}, + + {"req": 4000, "slop": 96, "alloc": "C"}, + {"req": 4000, "slop": 96, "alloc": "C"}, + {"req": 4000, "slop": 96, "alloc": "C"}, + {"req": 4000, "slop": 96, "alloc": "C"}, + + {"req": 4096, "alloc": "E"}, + {"req": 4096, "alloc": "E"}, + {"req": 4096, "alloc": "E"}, + {"req": 4096, "alloc": "E"}, + + {"req": 2000, "slop": 48, "alloc": "F"}, + {"req": 1000, "slop": 24, "alloc": "F", "reps": ["R1"]}, + {"req": 512, "alloc": "F"}, + {"req": 512, "alloc": "F"}, + {"req": 512, "alloc": "F"}, + {"req": 512, "alloc": "F"}, + {"req": 128, "alloc": "F"}, + { "alloc": "F", "reps": ["R1", "R2"]}, + {"req": 64, "alloc": "F"}, + {"req": 64, "alloc": "F"}, + {"req": 64, "alloc": "F"}, + {"req": 64, "alloc": "F"}, + { "alloc": "F"}, + + {"req": 4096, "alloc": "G"}, + {"req": 4096, "alloc": "G"}, + {"req": 20480, "alloc": "G"} + ], + "traceTable": { + "A": ["AA"], + "B": ["BB"], + "C": ["CC"], + "D": ["DD"], + "E": ["EE"], + "F": ["FF"], + "G": ["GG"], + "R1": ["RR1"], + "R2": ["RR2"] + }, + "frameTable": { + "AA": "#00: A (A.cpp:99)", + "BB": "#00: B (B.cpp:99)", + "CC": "#00: C (C.cpp:99)", + "DD": "#00: D (D.cpp:99)", + "EE": "#00: E (E.cpp:99)", + "FF": "#00: F (F.cpp:99)", + "GG": "#00: G (G.cpp:99)", + "RR1": "#00: R1 (R1.cpp:99)", + "RR2": "#00: R2 (R2.cpp:99)" + } +}
--- a/memory/replace/dmd/test/test_dmd.js +++ b/memory/replace/dmd/test/test_dmd.js @@ -60,31 +60,29 @@ function readFile(aFile) { function runProcess(aExeFile, aArgs) { let process = Cc["@mozilla.org/process/util;1"] .createInstance(Components.interfaces.nsIProcess); process.init(aExeFile); process.run(/* blocking = */true, aArgs, aArgs.length); return process.exitValue; } -function test(aJsonFile, aPrefix, aOptions) { +function test(aPrefix, aArgs) { // DMD writes the JSON files to CurWorkD, so we do likewise here with // |actualFile| for consistency. It is removed once we've finished. let expectedFile = FileUtils.getFile("CurWorkD", [aPrefix + "-expected.txt"]); let actualFile = FileUtils.getFile("CurWorkD", [aPrefix + "-actual.txt"]); // Run dmd.py on the JSON file, producing |actualFile|. let args = [ gDmdScriptFile.path, "--filter-stacks-for-testing", "-o", actualFile.path - ]; - args = args.concat(aOptions); - args.push(aJsonFile.path); + ].concat(aArgs); runProcess(new FileUtils.File(gPythonName), args); // Compare |expectedFile| with |actualFile|. We produce nice diffs with // /usr/bin/diff on systems that have it (Mac and Linux). Otherwise (Windows) // we do a string compare of the file contents and then print them both if // they don't match. @@ -111,17 +109,17 @@ function test(aJsonFile, aPrefix, aOptio } ok(success, aPrefix); actualFile.remove(true); } function run_test() { - let jsonFile; + let jsonFile, jsonFile2; // These tests do full end-to-end testing of DMD, i.e. both the C++ code that // generates the JSON output, and the script that post-processes that output. // // Run these synchronously, because test() updates the full*.json files // in-place (to fix stacks) when it runs dmd.py, and that's not safe to do // asynchronously. @@ -129,46 +127,64 @@ function run_test() { gEnv.set(gEnv.get("DMD_PRELOAD_VAR"), gEnv.get("DMD_PRELOAD_VALUE")); runProcess(gDmdTestFile, []); let fullTestNames = ["empty", "unsampled1", "unsampled2", "sampled"]; for (let i = 0; i < fullTestNames.length; i++) { let name = fullTestNames[i]; jsonFile = FileUtils.getFile("CurWorkD", ["full-" + name + ".json"]); - test(jsonFile, "full-heap-" + name, ["--ignore-reports"]) - test(jsonFile, "full-reports-" + name, []) + test("full-heap-" + name, ["--ignore-reports", jsonFile.path]) + test("full-reports-" + name, [jsonFile.path]) jsonFile.remove(true); } // These tests only test the post-processing script. They use hand-written // JSON files as input. Ideally the JSON files would contain comments // explaining how they work, but JSON doesn't allow comments, so I've put // explanations here. // This just tests that stack traces of various lengths are truncated // appropriately. The number of records in the output is different for each // of the tested values. jsonFile = FileUtils.getFile("CurWorkD", ["script-max-frames.json"]); - test(jsonFile, "script-max-frames-8", ["-r", "--max-frames=8"]); - test(jsonFile, "script-max-frames-3", ["-r", "--max-frames=3", - "--no-fix-stacks"]); - test(jsonFile, "script-max-frames-1", ["-r", "--max-frames=1"]); + test("script-max-frames-8", + ["--ignore-reports", "--max-frames=8", jsonFile.path]); + test("script-max-frames-3", + ["--ignore-reports", "--max-frames=3", "--no-fix-stacks", + jsonFile.path]); + test("script-max-frames-1", + ["--ignore-reports", "--max-frames=1", jsonFile.path]); - // This test has three records that are shown in a different order for each + // This file has three records that are shown in a different order for each // of the different sort values. It also tests the handling of gzipped JSON // files. jsonFile = FileUtils.getFile("CurWorkD", ["script-sort-by.json.gz"]); - test(jsonFile, "script-sort-by-usable", ["-r", "--sort-by=usable"]); - test(jsonFile, "script-sort-by-req", ["-r", "--sort-by=req", - "--no-fix-stacks"]); - test(jsonFile, "script-sort-by-slop", ["-r", "--sort-by=slop"]); + test("script-sort-by-usable", + ["--ignore-reports", "--sort-by=usable", jsonFile.path]); + test("script-sort-by-req", + ["--ignore-reports", "--sort-by=req", "--no-fix-stacks", jsonFile.path]); + test("script-sort-by-slop", + ["--ignore-reports", "--sort-by=slop", jsonFile.path]); - // This test has several real stack traces taken from Firefox execution, each + // This file has several real stack traces taken from Firefox execution, each // of which tests a different allocator function (or functions). jsonFile = FileUtils.getFile("CurWorkD", ["script-ignore-alloc-fns.json"]); - test(jsonFile, "script-ignore-alloc-fns", ["-r", "--ignore-alloc-fns"]); + test("script-ignore-alloc-fns", + ["--ignore-reports", "--ignore-alloc-fns", jsonFile.path]); - // This test has numerous allocations of different sizes, some repeated, some + // This file has numerous allocations of different sizes, some repeated, some // sampled, that all end up in the same record. jsonFile = FileUtils.getFile("CurWorkD", ["script-show-all-block-sizes.json"]); - test(jsonFile, "script-show-all-block-sizes", ["-r", "--show-all-block-sizes"]); + test("script-show-all-block-sizes", + ["--ignore-reports", "--show-all-block-sizes", jsonFile.path]); + + // This tests diffs. The first invocation has no options, the second has + // several. + jsonFile = FileUtils.getFile("CurWorkD", ["script-diff1.json"]); + jsonFile2 = FileUtils.getFile("CurWorkD", ["script-diff2.json"]); + test("script-diff-basic", + [jsonFile.path, jsonFile2.path]); + test("script-diff-options", + ["--ignore-reports", "--show-all-block-sizes", + jsonFile.path, jsonFile2.path]); } +
--- a/memory/replace/dmd/test/xpcshell.ini +++ b/memory/replace/dmd/test/xpcshell.ini @@ -15,15 +15,19 @@ support-files = script-sort-by.json.gz script-sort-by-usable-expected.txt script-sort-by-req-expected.txt script-sort-by-slop-expected.txt script-ignore-alloc-fns.json script-ignore-alloc-fns-expected.txt script-show-all-block-sizes.json script-show-all-block-sizes-expected.txt + script-diff1.json + script-diff2.json + script-diff-basic-expected.txt + script-diff-options-expected.txt # Bug 1077230 explains why this test is disabled on Mac 10.6. # Bug 1076446 comment 20 explains why this test is only enabled on Windows 5.1 # (WinXP) and 6.1 (Win7), but not 6.2 (Win8). [test_dmd.js] dmd = true run-if = os == 'linux' || os == 'mac' && os_version != '10.6' || os == 'win' && (os_version == '5.1' || os_version == '6.1')