bug 383083 - post-process symbol files to add source file revision numbers. r=bsmedberg
authorted.mielczarek@gmail.com
Wed, 27 Jun 2007 03:25:14 -0700
changeset 2847 e0bc162d641283121f5493ff242e2b51b7f8c13b
parent 2846 acea1423be7c4f3a9bd82e10bcc5118a24689ebc
child 2848 8188d0ab768ad770ad511a3ae15fe65d6481c36a
push id1
push userbsmedberg@mozilla.com
push dateThu, 20 Mar 2008 16:49:24 +0000
treeherdermozilla-central@61007906a1f8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbsmedberg
bugs383083
milestone1.9a6pre
bug 383083 - post-process symbol files to add source file revision numbers. r=bsmedberg
Makefile.in
toolkit/airbag/tools/make_symbol_store.pl
toolkit/airbag/tools/symbolstore.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -132,56 +132,51 @@ ifdef MOZ_PROFILE
 	/bin/find $(DIST) -name "*.EXE" -exec splitsym {} \;
 	mkdir -p $(DIST)/$(BUILDID)
 	/bin/find $(DIST) -name "*.dbg" -exec mv {} $(DIST)/$(BUILDID) \;
 endif # MOZ_PROFILE
 endif # MOZILLA_OFFICIAL
 endif # WINNT
 
 ifeq ($(OS_ARCH),WINNT)
-SYM_FIND_CMD := /bin/find . -path dist -prune -o -name "*.exe" \
-    -o -name "*.dll" -o -name "*.EXE" | sed "s/\.[^\.]*$$/\.pdb/"
 # we want to copy PDB files on Windows
 MAKE_SYM_STORE_ARGS := -c
 DUMP_SYMS_BIN := $(topsrcdir)/toolkit/airbag/tools/win32/dump_syms.exe
+# PDB files don't get moved to dist, so we need to scan the whole objdir
+MAKE_SYM_STORE_PATH := .
 endif
 ifeq ($(OS_ARCH),Darwin)
 # need to pass arch flags for universal builds
 ifdef UNIVERSAL_BINARY
 MAKE_SYM_STORE_ARGS := -a "ppc i386"
-SYM_DIST := $(DIST)/universal
+MAKE_SYM_STORE_PATH := $(DIST)/universal
 else
 MAKE_SYM_STORE_ARGS := -a $(OS_TEST)
-SYM_DIST := $(DIST)
+MAKE_SYM_STORE_PATH := $(DIST)/bin
 endif
-# |file| is stupid on universal binaries, it produces one line of output
-# for the file, and one line of output for each architecture contained within.
-SYM_FIND_CMD := find -L $(SYM_DIST) -type f -a -perm -100 -o -name "*.dylib" \
-    | xargs file -L | grep "Mach-O" | grep -v "for architecture" | cut -f1 -d':'
 DUMP_SYMS_BIN := $(DIST)/host/bin/dump_syms
 endif
 ifeq ($(OS_ARCH),Linux)
 MAKE_SYM_STORE_ARGS :=
-SYM_FIND_CMD := find -L $(DIST)/bin -type f -a -perm -100 -o -name "*.so" \
-    | xargs file -L | grep "ELF" | cut -f1 -d':'
 DUMP_SYMS_BIN := $(DIST)/host/bin/dump_syms
+MAKE_SYM_STORE_PATH := $(DIST)/bin
 endif
 
 ifdef MOZ_SYMBOLS_EXTRA_BUILDID
 EXTRA_BUILDID := -$(MOZ_SYMBOLS_EXTRA_BUILDID)
 endif
 
 buildsymbols:
 ifdef MOZ_AIRBAG
 	echo building symbol store
 	mkdir -p $(DIST)/crashreporter-symbols/$(BUILDID)
-	$(SYM_FIND_CMD) | \
-	  xargs $(topsrcdir)/toolkit/airbag/tools/make_symbol_store.pl    \
-	  $(MAKE_SYM_STORE_ARGS) $(DUMP_SYMS_BIN)                         \
-	  $(DIST)/crashreporter-symbols/$(BUILDID) >                      \
+	$(PYTHON) $(topsrcdir)/toolkit/airbag/tools/symbolstore.py    \
+	  $(MAKE_SYM_STORE_ARGS) -s $(topsrcdir) $(DUMP_SYMS_BIN)     \
+	  $(DIST)/crashreporter-symbols/$(BUILDID)                    \
+	  $(MAKE_SYM_STORE_PATH) >                                    \
 	  $(DIST)/crashreporter-symbols/$(BUILDID)/$(MOZ_APP_NAME)-$(MOZ_APP_VERSION)-$(OS_ARCH)-$(BUILDID)$(EXTRA_BUILDID)-symbols.txt
 	echo packing symbols
 	mkdir -p $(topsrcdir)/../$(BUILDID)
 	cd $(DIST)/crashreporter-symbols/$(BUILDID) && \
           zip -r9D ../crashreporter-symbols-$(BUILDID).zip .
 	mv $(DIST)/crashreporter-symbols/crashreporter-symbols-$(BUILDID).zip \
           $(topsrcdir)/../$(BUILDID)
 endif # MOZ_AIRBAG
deleted file mode 100755
--- a/toolkit/airbag/tools/make_symbol_store.pl
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/perl -w
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Ted Mielczarek <ted.mielczarek@gmail.com>
-# Portions created by the Initial Developer are Copyright (C) 2006
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
-#
-# ***** END LICENSE BLOCK *****
-#
-# Usage: make_symbol_store.pl <params> <dump_syms path> <symbol store path>
-#                             <debug info files>
-#   Runs dump_syms on each debug info file specified on the command line,
-#   then places the resulting symbol file in the proper directory
-#   structure in the symbol store path.  Accepts multiple files
-#   on the command line, so can be called as part of a pipe using
-#   find <dir> | xargs make_symbol_store.pl <dump_syms> <storepath>
-#   Parameters accepted:
-#     -c           : Copy debug info files to the same directory structure
-#                    as sym files
-#     -a "<archs>" : Run dump_syms -a <arch> for each space separated
-#                    cpu architecture in <archs> (only on OS X)
-
-use FileHandle;
-use File::Path;
-use File::Copy;
-use File::Basename;
-
-print "Usage: make_symbol_store.pl <params>" .
-  "<dump_syms path> <storepath> <debug info files>\n"
-  and exit if scalar @ARGV < 3;
-
-# Given a symbol file generated by dump_syms,
-# and a directory to store the resulting symbol path,
-# move the symbol file into the directory structure
-# expected by the breakpad processor.  For details, see:
-# http://google-breakpad.googlecode.com/svn/trunk/src/processor/simple_symbol_supplier.h
-sub rename_symbol_file
-{
-    my ($symbol_file, $dest_path) = @_;
-    my $fh = FileHandle->new($symbol_file, "r");
-    return "" unless $fh;
-
-    my $line = <$fh>;
-    return "" unless $line;
-    $line =~ s/\s*$//;
-    $fh->close();
-    return "" unless $line =~ m/^MODULE/;
-
-    # the first line of a sym file looks like:
-    # MODULE os cpu identifier debug_file
-    my ($guid,$dbgfile) = (split(/ +/, $line))[3..4];
-    my $newpath = $dest_path . "/" . $dbgfile . "/" . $guid;
-    eval { mkpath($newpath) };
-    return "" if $@;
-
-    if(move($symbol_file, $newpath)) {
-        my $out = $newpath;
-        my ($f) = fileparse($symbol_file);
-        $out =~ s/^$dest_path//;
-        $out =~ s|^/||;
-        print "$out/$f\n";
-        return $newpath;
-    }
-
-    return "";
-}
-
-my $copy_dbg = 0;
-my @archs = ('');
-while (@ARGV && $ARGV[0] =~ m/^-/) {
-    my $arg = shift;
-    if ($arg eq '-c') {
-        $copy_dbg = 1;
-    }
-    elsif ($arg eq '-a') {
-        @archs = (split(/\s+/, shift));
-    }
-}
-
-my $dump_syms = shift;
-my $symbol_path = shift;
-foreach my $dbgfile (@ARGV) {
-    next unless -f $dbgfile;
-    # get filename without path or .pdb extension, if it exists
-    my ($sf) = fileparse($dbgfile, ".pdb");
-    my $symfile = $symbol_path . "/" . $sf . ".sym";
-    foreach my $arch (@archs) {
-      my $a = '';
-      $a = "-a $arch" if $arch ne '';
-      system("${dump_syms} ${a} ${dbgfile} > ${symfile}");
-      # remove empty sym file, probably no debug symbols in that file
-      if (-s $symfile == 0) {
-	  unlink($symfile);
-	  next;
-      }
-      my $newpath = rename_symbol_file $symfile, $symbol_path;
-      if ($copy_dbg && $newpath ne "") {
-        my $out = $newpath;
-        $out =~ s/^$symbol_path//;
-        $out =~ s|^/||;
-        print "$out/${sf}.pdb\n";
-        copy($dbgfile, $newpath);
-      }
-    }
-}
new file mode 100755
--- /dev/null
+++ b/toolkit/airbag/tools/symbolstore.py
@@ -0,0 +1,332 @@
+#!/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# The Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2007
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Ted Mielczarek <ted.mielczarek@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+#
+# Usage: symbolstore.py <params> <dump_syms path> <symbol store path>
+#                                <debug info files or dirs>
+#   Runs dump_syms on each debug info file specified on the command line,
+#   then places the resulting symbol file in the proper directory
+#   structure in the symbol store path.  Accepts multiple files
+#   on the command line, so can be called as part of a pipe using
+#   find <dir> | xargs symbolstore.pl <dump_syms> <storepath>
+#   But really, you might just want to pass it <dir>.
+#
+#   Parameters accepted:
+#     -c           : Copy debug info files to the same directory structure
+#                    as sym files
+#     -a "<archs>" : Run dump_syms -a <arch> for each space separated
+#                    cpu architecture in <archs> (only on OS X)
+#     -s <srcdir>  : Use <srcdir> as the top source directory to
+#                    generate relative filenames.
+
+import sys
+import os
+import re
+import shutil
+from optparse import OptionParser
+
+# Utility functions
+
+def GetCVSRevision(file):
+    """Given a full path to a file, look in CVS/Entries
+    for the CVS revision number"""
+    (path, filename) = os.path.split(file)
+    entries = os.path.join(path, "CVS", "Entries")
+    if not os.path.isfile(entries):
+        return None
+    f = open(entries, "r")
+    for line in f:
+        parts = line.split("/")
+        if len(parts) > 1 and parts[1] == filename:
+            return parts[2]
+    print >> sys.stderr, "Failed to get CVS Revision for %s" % filename
+    return None
+
+def GetCVSRoot(file):
+    """Given a full path to a file, look in CVS/Root
+    for the CVS Root"""
+    (path, filename) = os.path.split(file)
+    root = os.path.join(path, "CVS", "Root")
+    if not os.path.isfile(root):
+        return None
+    f = open(root, "r")
+    root_name = f.readline().strip()
+    f.close()
+    parts = root_name.split("@")
+    if len(parts) > 1:
+        # we don't want the extra colon
+        return parts[1].replace(":","")
+    print >> sys.stderr, "Failed to get CVS Root for %s" % filename
+    return None
+
+def GetVCSFilename(file, srcdir):
+    """Given a full path to a file, and the top source directory,
+    look for version control information about this file, and return
+    a specially formatted filename that contains the VCS type,
+    VCS location, relative filename, and revision number, formatted like:
+    vcs:vcs location:filename:revision
+    For example:
+    cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36"""
+    (path, filename) = os.path.split(file)
+    cvsdir = os.path.join(path, "CVS")
+    if os.path.isdir(cvsdir):
+        rev = GetCVSRevision(file)
+        root = GetCVSRoot(file)
+        if rev is not None and root is not None:
+            if srcdir is not None:
+                # strip the base path off
+                # but we actually want the last dir in srcdir
+                # also, we want forward slashes on win32 paths
+                file = os.path.normpath(file)
+                file = file.replace(srcdir, "", 1)
+                (head, tail) = os.path.split(srcdir)
+                if tail == "":
+                    tail = os.path.basename(head)
+                file = tail + file
+            file = file.replace("\\", "/")
+            return "cvs:%s:%s:%s" % (root, file, rev)
+    file = file.replace("\\", "/")
+    return file
+
+def GetPlatformSpecificDumper(**kwargs):
+    """This function simply returns a instance of a subclass of Dumper
+    that is appropriate for the current platform."""
+    return {'win32': Dumper_Win32,
+            'cygwin': Dumper_Win32,
+            'linux2': Dumper_Linux,
+            'darwin': Dumper_Mac}[sys.platform](**kwargs)
+
+class Dumper:
+    """This class can dump symbols from a file with debug info, and
+    store the output in a directory structure that is valid for use as
+    a Breakpad symbol server.  Requires a path to a dump_syms binary--
+    |dump_syms| and a directory to store symbols in--|symbol_path|.
+    Optionally takes a list of processor architectures to process from
+    each debug file--|archs|, the full path to the top source
+    directory--|srcdir|, for generating relative source file names,
+    and an option to copy debug info files alongside the dumped
+    symbol files--|copy_debug|, mostly useful for creating a
+    Microsoft Symbol Server from the resulting output.
+
+    You don't want to use this directly if you intend to call
+    ProcessDir.  Instead, call GetPlatformSpecificDumper to
+    get an instance of a subclass."""
+    def __init__(self, dump_syms, symbol_path,
+                 archs=None, srcdir=None, copy_debug=False):
+        self.dump_syms = dump_syms
+        self.symbol_path = symbol_path
+        if archs is None:
+            # makes the loop logic simpler
+            self.archs = ['']
+        else:
+            self.archs = ['-a %s' % a for a in archs.split()]
+        if srcdir is not None:
+            self.srcdir = os.path.normpath(srcdir)
+        else:
+            self.srcdir = None
+        self.copy_debug = copy_debug
+
+    # subclasses override this
+    def ShouldProcess(self, file):
+        return False
+
+    def RunFileCommand(self, file):
+        """Utility function, returns the output of file(1)"""
+        try:
+            # we use -L to read the targets of symlinks,
+            # and -b to print just the content, not the filename
+            return os.popen("file -Lb " + file).read()
+        except:
+            return ""
+
+    # This is a no-op except on Win32
+    def FixFilenameCase(self, file):
+        return file
+
+    def Process(self, file_or_dir):
+        "Process a file or all the (valid) files in a directory."
+        if os.path.isdir(file_or_dir):
+            return self.ProcessDir(file_or_dir)
+        elif os.path.isfile(file_or_dir):
+            return self.ProcessFile(file_or_dir)
+        # maybe it doesn't exist?
+        return False
+    
+    def ProcessDir(self, dir):
+        """Process all the valid files in this directory.  Valid files
+        are determined by calling ShouldProcess."""
+        result = True
+        for root, dirs, files in os.walk(dir):
+            for f in files:
+                fullpath = os.path.join(root, f)
+                if self.ShouldProcess(fullpath):
+                    if not self.ProcessFile(fullpath):
+                        result = False
+        return result
+    
+    def ProcessFile(self, file):
+        """Dump symbols from this file into a symbol file, stored
+        in the proper directory structure in  |symbol_path|."""
+        result = False
+        for arch in self.archs:
+            try:
+                cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r")
+                module_line = cmd.next()
+                if module_line.startswith("MODULE"):
+                    # MODULE os cpu guid debug_file
+                    (guid, debug_file) = (module_line.split())[3:5]
+                    # strip off .pdb extensions, and append .sym
+                    sym_file = re.sub("\.pdb$", "", debug_file) + ".sym"
+                    # we do want forward slashes here
+                    rel_path = os.path.join(debug_file,
+                                            guid,
+                                            sym_file).replace("\\", "/")
+                    full_path = os.path.normpath(os.path.join(self.symbol_path,
+                                                              rel_path))
+                    try:
+                        os.makedirs(os.path.dirname(full_path))
+                    except OSError: # already exists
+                        pass
+                    f = open(full_path, "w")
+                    f.write(module_line)
+                    # now process the rest of the output
+                    for line in cmd:
+                        if line.startswith("FILE"):
+                            # FILE index filename
+                            (x, index, filename) = line.split(None, 2)
+                            filename = self.FixFilenameCase(filename.rstrip())
+                            filename = GetVCSFilename(filename, self.srcdir)
+                            f.write("FILE %s %s\n" % (index, filename))
+                        else:
+                            # pass through all other lines unchanged
+                            f.write(line)
+                    f.close()
+                    cmd.close()
+                    # we output relative paths so callers can get a list of what
+                    # was generated
+                    print rel_path
+                    if self.copy_debug:
+                        rel_path = os.path.join(debug_file,
+                                                guid,
+                                                debug_file).replace("\\", "/")
+                        print rel_path
+                        full_path = os.path.normpath(os.path.join(self.symbol_path,
+                                                                  rel_path))
+                        shutil.copyfile(file, full_path)
+                    result = True
+            except StopIteration:
+                pass
+            except:
+                print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0]
+                raise
+            return result
+
+# Platform-specific subclasses.  For the most part, these just have
+# logic to determine what files to extract symbols from.
+
+class Dumper_Win32(Dumper):
+    def ShouldProcess(self, file):
+        """This function will allow processing of pdb files that have dll
+        or exe files with the same base name next to them."""
+        if file.endswith(".pdb"):
+            (path,ext) = os.path.splitext(file)
+            if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll"):
+                return True
+        return False
+    
+    def FixFilenameCase(self, file):
+        """Recent versions of Visual C++ put filenames into
+        PDB files as all lowercase.  If the file exists
+        on the local filesystem, fix it."""
+        (path, filename) = os.path.split(file)
+        if not os.path.isdir(path):
+            return file
+        lc_filename = filename.lower()
+        for f in os.listdir(path):
+            if f.lower() == lc_filename:
+                return os.path.join(path, f)
+        return file
+
+class Dumper_Linux(Dumper):
+    def ShouldProcess(self, file):
+        """This function will allow processing of files that are
+        executable, or end with the .so extension, and additionally
+        file(1) reports as being ELF files.  It expects to find the file
+        command in PATH."""
+        if file.endswith(".so") or os.access(file, os.X_OK):
+            return self.RunFileCommand(file).startswith("ELF")
+        return False
+
+class Dumper_Mac(Dumper):
+    def ShouldProcess(self, file):
+        """This function will allow processing of files that are
+        executable, or end with the .dylib extension, and additionally
+        file(1) reports as being Mach-O files.  It expects to find the file
+        command in PATH."""
+        if file.endswith(".dylib") or os.access(file, os.X_OK):
+            return self.RunFileCommand(file).startswith("Mach-O")
+        return False
+
+# Entry point if called as a standalone program
+def main():
+    parser = OptionParser(usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>")
+    parser.add_option("-c", "--copy",
+                      action="store_true", dest="copy_debug", default=False,
+                      help="Copy debug info files into the same directory structure as symbol files")
+    parser.add_option("-a", "--archs",
+                      action="store", dest="archs",
+                      help="Run dump_syms -a <arch> for each space separated cpu architecture in ARCHS (only on OS X)")
+    parser.add_option("-s", "--srcdir",
+                      action="store", dest="srcdir",
+                      help="Use SRCDIR to determine relative paths to source files")
+    (options, args) = parser.parse_args()
+
+    if len(args) < 3:
+        parser.error("not enough arguments")
+        exit(1)
+
+    dumper = GetPlatformSpecificDumper(dump_syms=args[0],
+                                       symbol_path=args[1],
+                                       copy_debug=options.copy_debug,
+                                       archs=options.archs,
+                                       srcdir=options.srcdir)
+    for arg in args[2:]:
+        dumper.Process(arg)
+
+# run main if run directly
+if __name__ == "__main__":
+    main()