Bug 408134 - "Source Server for Windows builds integrated into symbolstore.py" [p=lukasblakk@gmail.com (Lukas Blakk [lsblakk]) r=luser/ted a1.9=beltzner]
Tue, 26 Feb 2008 16:54:47 -0800
changeset 12298 a601d67a071725f57bb771912741f8ce19c39741
parent 12297 6db6583411045d5b2bf4b131a812f7273c3e6805
child 12299 37bc4f3a6de47c46649c47f98b82e348dcba0df5
push id1
push userbsmedberg@mozilla.com
push dateThu, 20 Mar 2008 16:49:24 +0000
treeherdermozilla-central@61007906a1f8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersluser, ted
Bug 408134 - "Source Server for Windows builds integrated into symbolstore.py" [p=lukasblakk@gmail.com (Lukas Blakk [lsblakk]) r=luser/ted a1.9=beltzner]
--- a/Makefile.in
+++ b/Makefile.in
@@ -135,16 +135,19 @@ ifdef MOZ_PROFILE
 	/bin/find $(DIST) -name "*.dbg" -exec mv {} $(DIST)/$(BUILDID) \;
 endif # MOZ_PROFILE
 endif # WINNT
 ifeq ($(OS_ARCH),WINNT)
 # we want to copy PDB files on Windows
 ifeq (,$(CYGWIN_WRAPPER))
 # this doesn't work with Cygwin Python
 MAKE_SYM_STORE_ARGS += --vcs-info
 DUMP_SYMS_BIN ?= $(topsrcdir)/toolkit/crashreporter/tools/win32/dump_syms.exe
 # PDB files don't get moved to dist, so we need to scan the whole objdir
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -275,47 +275,60 @@ def GetVCSFilename(file, srcdir):
 def GetPlatformSpecificDumper(**kwargs):
     """This function simply returns a instance of a subclass of Dumper
     that is appropriate for the current platform."""
     return {'win32': Dumper_Win32,
             'cygwin': Dumper_Win32,
             'linux2': Dumper_Linux,
             'darwin': Dumper_Mac}[sys.platform](**kwargs)
+def SourceIndex(fileStream, outputPath):
+    """Takes a list of files, writes info to a data block in a .stream file"""
+    # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+    # Create the srcsrv data block that indexes the pdb file
+    result = True
+    pdbStreamFile = open(outputPath, "w")
+    pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%CVS_WORKINGDIR%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% %var3%\r\nCVS_EXTRACT_TARGET=%targ%\%var2%\%fnbksl%(%var3%)\%fnfile%(%var1%)\r\nCVS_WORKING_DIR=%targ%\%var2%\%fnbksl%(%var3%)\r\nMYSERVER=%CVSROOT%\r\nSRCSRVTRG=%CVS_WORKING_DIR%\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''')
+    pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above
+    pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n")
+    pdbStreamFile.close()
+    return result
 class Dumper:
     """This class can dump symbols from a file with debug info, and
     store the output in a directory structure that is valid for use as
     a Breakpad symbol server.  Requires a path to a dump_syms binary--
     |dump_syms| and a directory to store symbols in--|symbol_path|.
     Optionally takes a list of processor architectures to process from
     each debug file--|archs|, the full path to the top source
     directory--|srcdir|, for generating relative source file names,
     and an option to copy debug info files alongside the dumped
     symbol files--|copy_debug|, mostly useful for creating a
     Microsoft Symbol Server from the resulting output.
     You don't want to use this directly if you intend to call
     ProcessDir.  Instead, call GetPlatformSpecificDumper to
     get an instance of a subclass."""
     def __init__(self, dump_syms, symbol_path,
-                 archs=None, srcdir=None, copy_debug=False, vcsinfo=False):
+                 archs=None, srcdir=None, copy_debug=False, vcsinfo=False, srcsrv=False):
         # popen likes absolute paths, at least on windows
         self.dump_syms = os.path.abspath(dump_syms)
         self.symbol_path = symbol_path
         if archs is None:
             # makes the loop logic simpler
             self.archs = ['']
             self.archs = ['-a %s' % a for a in archs.split()]
         if srcdir is not None:
             self.srcdir = os.path.normpath(srcdir)
             self.srcdir = None
         self.copy_debug = copy_debug
         self.vcsinfo = vcsinfo
+        self.srcsrv = srcsrv
     # subclasses override this
     def ShouldProcess(self, file):
         return False
     def RunFileCommand(self, file):
         """Utility function, returns the output of file(1)"""
@@ -324,16 +337,20 @@ class Dumper:
             return os.popen("file -Lb " + file).read()
             return ""
     # This is a no-op except on Win32
     def FixFilenameCase(self, file):
         return file
+    # This is a no-op except on Win32
+    def SourceServerIndexing(self, debug_file, guid, sourceFileStream):
+        return ""
     def Process(self, file_or_dir):
         "Process a file or all the (valid) files in a directory."
         if os.path.isdir(file_or_dir):
             return self.ProcessDir(file_or_dir)
         elif os.path.isfile(file_or_dir):
             return self.ProcessFile(file_or_dir)
         # maybe it doesn't exist?
         return False
@@ -349,16 +366,17 @@ class Dumper:
                     if not self.ProcessFile(fullpath):
                         result = False
         return result
     def ProcessFile(self, file):
         """Dump symbols from this file into a symbol file, stored
         in the proper directory structure in  |symbol_path|."""
         result = False
+        sourceFileStream = ''
         for arch in self.archs:
                 cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r")
                 module_line = cmd.next()
                 if module_line.startswith("MODULE"):
                     # MODULE os cpu guid debug_file
                     (guid, debug_file) = (module_line.split())[3:5]
                     # strip off .pdb extensions, and append .sym
@@ -376,18 +394,23 @@ class Dumper:
                     f = open(full_path, "w")
                     # now process the rest of the output
                     for line in cmd:
                         if line.startswith("FILE"):
                             # FILE index filename
                             (x, index, filename) = line.split(None, 2)
                             filename = self.FixFilenameCase(filename.rstrip())
+                            sourcepath = filename
                             if self.vcsinfo:
                                 filename = GetVCSFilename(filename, self.srcdir)
+                            # gather up files with cvs for indexing   
+                            if filename.startswith("cvs"):
+                                (ver, checkout, source_file, revision) = filename.split(":", 3)
+                                sourceFileStream += sourcepath + "*MYSERVER*" + source_file + '*' + revision + "\r\n"
                             f.write("FILE %s %s\n" % (index, filename))
                             # pass through all other lines unchanged
                     # we output relative paths so callers can get a list of what
                     # was generated
@@ -395,16 +418,19 @@ class Dumper:
                     if self.copy_debug:
                         rel_path = os.path.join(debug_file,
                                                 debug_file).replace("\\", "/")
                         print rel_path
                         full_path = os.path.normpath(os.path.join(self.symbol_path,
                         shutil.copyfile(file, full_path)
+                    if self.srcsrv:
+                        # Call on SourceServerIndexing
+                        result = self.SourceServerIndexing(debug_file, guid, sourceFileStream)
                     result = True
             except StopIteration:
                 print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0]
         return result
@@ -440,16 +466,36 @@ class Dumper_Win32(Dumper):
             for f in os.listdir(path):
                 if f.lower() == lc_filename:
                     result = os.path.join(path, f)
         # Cache the corrected version to avoid future filesystem hits.
         self.fixedFilenameCaseCache[file] = result
         return result
+    def SourceServerIndexing(self, debug_file, guid, sourceFileStream):
+        # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
+        cwd = os.getcwd()
+        streamFilename = debug_file + ".stream"
+        stream_output_path = os.path.join(cwd, streamFilename)
+        # Call SourceIndex to create the .stream file
+        result = SourceIndex(sourceFileStream, stream_output_path)
+        if self.copy_debug:
+            pdbstr_path = os.environ.get("PDBSTR_PATH")
+            pdbstr = os.path.normpath(pdbstr_path)
+            pdb_rel_path = os.path.join(debug_file, guid, debug_file)
+            pdb_filename = os.path.normpath(os.path.join(self.symbol_path, pdb_rel_path))
+            # move to the dir with the stream files to call pdbstr
+            os.chdir(os.path.dirname(stream_output_path))
+            os.spawnv(os.P_WAIT, pdbstr, [pdbstr, "-w", "-p:" + pdb_filename, "-i:" + streamFilename, "-s:srcsrv"])
+            # clean up all the .stream files when done
+            os.remove(stream_output_path)
+        return result
 class Dumper_Linux(Dumper):
     def ShouldProcess(self, file):
         """This function will allow processing of files that are
         executable, or end with the .so extension, and additionally
         file(1) reports as being ELF files.  It expects to find the file
         command in PATH."""
         if file.endswith(".so") or os.access(file, os.X_OK):
@@ -476,26 +522,37 @@ def main():
                       action="store", dest="archs",
                       help="Run dump_syms -a <arch> for each space separated cpu architecture in ARCHS (only on OS X)")
     parser.add_option("-s", "--srcdir",
                       action="store", dest="srcdir",
                       help="Use SRCDIR to determine relative paths to source files")
     parser.add_option("-v", "--vcs-info",
                       action="store_true", dest="vcsinfo",
                       help="Try to retrieve VCS info for each FILE listed in the output")
+    parser.add_option("-i", "--source-index",
+                      action="store_true", dest="srcsrv", default=False,
+                      help="Add source index information to debug files, making them suitable for use in a source server.")
     (options, args) = parser.parse_args()
+    #check to see if the pdbstr.exe exists
+    if options.srcsrv:
+        pdbstr = os.environ.get("PDBSTR_PATH")
+        if not os.path.exists(pdbstr):
+            print >> sys.stderr, "Invalid path to pdbstr.exe - please set/check PDBSTR_PATH.\n"
+            sys.exit(1)
     if len(args) < 3:
         parser.error("not enough arguments")
     dumper = GetPlatformSpecificDumper(dump_syms=args[0],
-                                       vcsinfo=options.vcsinfo)
+                                       vcsinfo=options.vcsinfo,
+                                       srcsrv=options.srcsrv)
     for arg in args[2:]:
 # run main if run directly
 if __name__ == "__main__":