Bug 424817 - "source server tweaks" [p=lukasblakk@gmail.com (Lukas Blakk [lsblakk]) r=luser/ted a1.9=beltzner]
Fri, 04 Apr 2008 01:54:44 -0700
changeset 13896 762dd6c7a762cd88757c5a3be8908caa34b76b5c
parent 13895 0d91957c2c7b8d0f7ffda3a220ac70bff1d3c4e8
child 13897 c652aaaa02159bfbe72500f51b751a89a9e4acb7
push idunknown
push userunknown
push dateunknown
reviewersluser, ted
Bug 424817 - "source server tweaks" [p=lukasblakk@gmail.com (Lukas Blakk [lsblakk]) r=luser/ted a1.9=beltzner]
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -62,16 +62,17 @@ from optparse import OptionParser
 # Utility classes
 class VCSFileInfo:
     """ A base class for version-controlled file information. Ensures that the
         following attributes are generated only once (successfully):
+            self.clean_root
         The attributes are generated by a single call to the GetRoot,
         GetRevision, and GetFilename methods. Those methods are explicitly not
         implemented here and must be implemented in derived classes. """
     def __init__(self, file):
@@ -86,34 +87,45 @@ class VCSFileInfo:
             failure on the off chance that a future call might succeed. """
         if name == "root":
             root = self.GetRoot()
             if root:
                 self.root = root
             return root
+        elif name == "clean_root":
+            clean_root = self.GetCleanRoot()
+            if clean_root:
+                self.clean_root = clean_root
+            return clean_root
         elif name == "revision":
             revision = self.GetRevision()
             if revision:
                 self.revision = revision
             return revision
         elif name == "filename":
             filename = self.GetFilename()
             if filename:
                 self.filename = filename
             return filename
         raise AttributeError
     def GetRoot(self):
+        """ This method should return the unmodified root for the file or 'None'
+            on failure. """
+        raise NotImplementedError
+    def GetCleanRoot(self):
         """ This method should return the repository root for the file or 'None'
             on failure. """
-        raise NotImplementedError
+        raise NotImplementedErrors
     def GetRevision(self):
         """ This method should return the revision number for the file or 'None'
             on failure. """
         raise NotImplementedError
     def GetFilename(self):
         """ This method should return the repository-specific filename for the
@@ -131,17 +143,23 @@ class CVSFileInfo(VCSFileInfo):
     def GetRoot(self):
         (path, filename) = os.path.split(self.file)
         root = os.path.join(path, "CVS", "Root")
         if not os.path.isfile(root):
             return None
         f = open(root, "r")
         root_name = f.readline().strip()
-        parts = root_name.split("@")
+        if root_name:
+            return root_name
+        print >> sys.stderr, "Failed to get CVS Root for %s" % filename
+        return None
+    def GetCleanRoot(self):
+        parts = self.root.split('@')
         if len(parts) > 1:
             # we don't want the extra colon
             return parts[1].replace(":","")
         print >> sys.stderr, "Failed to get CVS Root for %s" % filename
         return None
     def GetRevision(self):
         (path, filename) = os.path.split(self.file)
@@ -153,31 +171,31 @@ class CVSFileInfo(VCSFileInfo):
             parts = line.split("/")
             if len(parts) > 1 and parts[1] == filename:
                 return parts[2]
         print >> sys.stderr, "Failed to get CVS Revision for %s" % filename
         return None
     def GetFilename(self):
         file = self.file
-        if self.revision and self.root:
+        if self.revision and self.clean_root:
             if self.srcdir:
                 # strip the base path off
                 # but we actually want the last dir in srcdir
                 file = os.path.normpath(file)
                 # the lower() is to handle win32+vc8, where
                 # the source filenames come out all lowercase,
                 # but the srcdir can be mixed case
                 if file.lower().startswith(self.srcdir.lower()):
                     file = file[len(self.srcdir):]
                 (head, tail) = os.path.split(self.srcdir)
                 if tail == "":
                     tail = os.path.basename(head)
                 file = tail + file
-            return "cvs:%s:%s:%s" % (self.root, file, self.revision)
+            return "cvs:%s:%s:%s" % (self.clean_root, file, self.revision)
         return file
 class SVNFileInfo(VCSFileInfo):
     url = None
     repo = None
     svndata = {}
     # This regex separates protocol and optional username/password from a url.
@@ -215,16 +233,20 @@ class SVNFileInfo(VCSFileInfo):
         key = "Repository Root"
         if key in self.svndata:
             match = self.rootRegex.match(self.svndata[key])
             if match:
                 return match.group(1)
         print >> sys.stderr, "Failed to get SVN Root for %s" % self.file
         return None
+    # File bug to get this teased out from the current GetRoot, this is temporary
+    def GetCleanRoot(self):
+        return self.root
     def GetRevision(self):
         key = "Revision"
         if key in self.svndata:
             return self.svndata[key]
         print >> sys.stderr, "Failed to get SVN Revision for %s" % self.file
         return None
     def GetFilename(self):
@@ -240,59 +262,66 @@ class SVNFileInfo(VCSFileInfo):
 # A cache of files for which VCS info has already been determined. Used to
 # prevent extra filesystem activity or process launching.
 vcsFileInfoCache = {}
 def GetVCSFilename(file, srcdir):
     """Given a full path to a file, and the top source directory,
     look for version control information about this file, and return
-    a specially formatted filename that contains the VCS type,
+    a tuple containing
+    1) a specially formatted filename that contains the VCS type,
     VCS location, relative filename, and revision number, formatted like:
     vcs:vcs location:filename:revision
     For example:
-    cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36"""
+    cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36
+    2) the unmodified root information if it exists"""
     (path, filename) = os.path.split(file)
     if path == '' or filename == '':
         return file
     fileInfo = None
+    root = ''
     if file in vcsFileInfoCache:
         # Already cached this info, use it.
         fileInfo = vcsFileInfoCache[file]
         if os.path.isdir(os.path.join(path, "CVS")):
             fileInfo = CVSFileInfo(file, srcdir)
+            if fileInfo:
+               root = fileInfo.root
         elif os.path.isdir(os.path.join(path, ".svn")) or \
              os.path.isdir(os.path.join(path, "_svn")):
             fileInfo = SVNFileInfo(file);
         vcsFileInfoCache[file] = fileInfo
     if fileInfo:
         file = fileInfo.filename
     # we want forward slashes on win32 paths
-    return file.replace("\\", "/")
+    return (file.replace("\\", "/"), root)
 def GetPlatformSpecificDumper(**kwargs):
     """This function simply returns a instance of a subclass of Dumper
     that is appropriate for the current platform."""
     return {'win32': Dumper_Win32,
             'cygwin': Dumper_Win32,
             'linux2': Dumper_Linux,
             'sunos5': Dumper_Solaris,
             'darwin': Dumper_Mac}[sys.platform](**kwargs)
-def SourceIndex(fileStream, outputPath):
+def SourceIndex(fileStream, outputPath, cvs_root):
     """Takes a list of files, writes info to a data block in a .stream file"""
     # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
     # Create the srcsrv data block that indexes the pdb file
     result = True
     pdbStreamFile = open(outputPath, "w")
-    pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%CVS_WORKINGDIR%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% %var3%\r\nCVS_EXTRACT_TARGET=%targ%\%var2%\%fnbksl%(%var3%)\%fnfile%(%var1%)\r\nCVS_WORKING_DIR=%targ%\%var2%\%fnbksl%(%var3%)\r\nMYSERVER=%CVSROOT%\r\nSRCSRVTRG=%CVS_WORKING_DIR%\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''')
+    pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%CVS_WORKINGDIR%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% %var3%\r\nCVS_EXTRACT_TARGET=%targ%\%var2%\%fnbksl%(%var3%)\%fnfile%(%var1%)\r\nCVS_WORKING_DIR=%targ%\r\nMYSERVER=''')
+    pdbStreamFile.write(cvs_root)
+    pdbStreamFile.write('''\r\nSRCSRVTRG=%targ%\%fnbksl%(%var3%)\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''')
     pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above
     pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n")
     return result
 class Dumper:
     """This class can dump symbols from a file with debug info, and
     store the output in a directory structure that is valid for use as
@@ -339,17 +368,17 @@ class Dumper:
             return ""
     # This is a no-op except on Win32
     def FixFilenameCase(self, file):
         return file
     # This is a no-op except on Win32
-    def SourceServerIndexing(self, debug_file, guid, sourceFileStream):
+    def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root):
         return ""
     # subclasses override this if they want to support this
     def CopyDebug(self, file, debug_file, guid):
     def Process(self, file_or_dir):
         "Process a file or all the (valid) files in a directory."
@@ -372,16 +401,17 @@ class Dumper:
                         result = False
         return result
     def ProcessFile(self, file):
         """Dump symbols from this file into a symbol file, stored
         in the proper directory structure in  |symbol_path|."""
         result = False
         sourceFileStream = ''
+        cvs_root = ''
         for arch in self.archs:
                 cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r")
                 module_line = cmd.next()
                 if module_line.startswith("MODULE"):
                     # MODULE os cpu guid debug_file
                     (guid, debug_file) = (module_line.split())[3:5]
                     # strip off .pdb extensions, and append .sym
@@ -406,17 +436,20 @@ class Dumper:
                             if sys.platform == "sunos5":
                                 start = filename.find(self.srcdir)
                                 if start == -1:
                                     start = 0
                                 filename = filename[start:]
                             filename = self.FixFilenameCase(filename.rstrip())
                             sourcepath = filename
                             if self.vcsinfo:
-                                filename = GetVCSFilename(filename, self.srcdir)
+                                (filename, rootname) = GetVCSFilename(filename, self.srcdir)
+                                # sets cvs_root in case the loop through files were to end on an empty rootname
+                                if rootname:
+                                   cvs_root = rootname
                             # gather up files with cvs for indexing   
                             if filename.startswith("cvs"):
                                 (ver, checkout, source_file, revision) = filename.split(":", 3)
                                 sourceFileStream += sourcepath + "*MYSERVER*" + source_file + '*' + revision + "\r\n"
                             f.write("FILE %s %s\n" % (index, filename))
                             # pass through all other lines unchanged
@@ -424,17 +457,17 @@ class Dumper:
                     # we output relative paths so callers can get a list of what
                     # was generated
                     print rel_path
                     if self.copy_debug:
                         self.CopyDebug(file, debug_file, guid)
                     if self.srcsrv:
                         # Call on SourceServerIndexing
-                        result = self.SourceServerIndexing(debug_file, guid, sourceFileStream)
+                        result = self.SourceServerIndexing(debug_file, guid, sourceFileStream, cvs_root)
                     result = True
             except StopIteration:
                 print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0]
         return result
@@ -481,23 +514,23 @@ class Dumper_Win32(Dumper):
                                 debug_file).replace("\\", "/")
         print rel_path
         full_path = os.path.normpath(os.path.join(self.symbol_path,
         shutil.copyfile(file, full_path)
-    def SourceServerIndexing(self, debug_file, guid, sourceFileStream):
+    def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root):
         # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
         cwd = os.getcwd()
         streamFilename = debug_file + ".stream"
         stream_output_path = os.path.join(cwd, streamFilename)
         # Call SourceIndex to create the .stream file
-        result = SourceIndex(sourceFileStream, stream_output_path)
+        result = SourceIndex(sourceFileStream, stream_output_path, cvs_root)
         if self.copy_debug:
             pdbstr_path = os.environ.get("PDBSTR_PATH")
             pdbstr = os.path.normpath(pdbstr_path)
             pdb_rel_path = os.path.join(debug_file, guid, debug_file)
             pdb_filename = os.path.normpath(os.path.join(self.symbol_path, pdb_rel_path))
             # move to the dir with the stream files to call pdbstr