Bug 777122 - Add telemetry reporting of late writes. r=jlebar.
authorRafael Ávila de Espíndola <respindola@mozilla.org>
Wed, 24 Oct 2012 15:42:54 -0400
changeset 119216 20c0647bdacc237ee8c8f35a96c68cbe8b8b3ec0
parent 119215 fe526c7150ebc848e2cebc8067b8f39812bee1b9
child 119217 d400b4e402f39f661d5ce2ede318fdcb15839797
push id1997
push userakeybl@mozilla.com
push dateMon, 07 Jan 2013 21:25:26 +0000
treeherdermozilla-beta@4baf45cdcf21 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlebar
bugs777122
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 777122 - Add telemetry reporting of late writes. r=jlebar.
mfbt/SHA1.h
xpcom/build/mozPoisonWriteMac.cpp
xpcom/string/public/nsTSubstring.h
--- a/mfbt/SHA1.h
+++ b/mfbt/SHA1.h
@@ -4,16 +4,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 /* Simple class for computing SHA1. */
 
 #ifndef mozilla_SHA1_h_
 #define mozilla_SHA1_h_
 
 #include "mozilla/StandardInteger.h"
+#include "mozilla/Types.h"
 
 #include <stddef.h>
 
 namespace mozilla {
 
 /**
  * This class computes the SHA1 hash of a byte sequence, or of the concatenation
  * of multiple sequences.  For example, computing the SHA1 of two sequences of
@@ -38,23 +39,23 @@ class SHA1Sum
         uint32_t w[16]; /* input buffer */
         uint8_t b[64];
     } u;
     uint64_t size; /* count of hashed bytes. */
     unsigned H[22]; /* 5 state variables, 16 tmp values, 1 extra */
     bool mDone;
 
   public:
-    SHA1Sum();
+    MFBT_API() SHA1Sum();
 
     static const size_t HashSize = 20;
     typedef uint8_t Hash[HashSize];
 
     /* Add len bytes of dataIn to the data sequence being hashed. */
-    void update(const void* dataIn, uint32_t len);
+    MFBT_API(void) update(const void* dataIn, uint32_t len);
 
     /* Compute the final hash of all data into hashOut. */
-    void finish(SHA1Sum::Hash& hashOut);
+    MFBT_API(void) finish(SHA1Sum::Hash& hashOut);
 };
 
 } /* namespace mozilla */
 
 #endif /* mozilla_SHA1_h_ */
--- a/xpcom/build/mozPoisonWriteMac.cpp
+++ b/xpcom/build/mozPoisonWriteMac.cpp
@@ -15,16 +15,17 @@
 #include "nsStackWalk.h"
 #include "nsPrintfCString.h"
 #include "mach_override.h"
 #include "prio.h"
 #include "plstr.h"
 #include "nsCOMPtr.h"
 #include "nsAppDirectoryServiceDefs.h"
 #include "nsDirectoryServiceUtils.h"
+#include "mozilla/SHA1.h"
 #include <sys/stat.h>
 #include <vector>
 #include <algorithm>
 #include <string.h>
 #include <sys/uio.h>
 #include <aio.h>
 #include <dlfcn.h>
 
@@ -34,16 +35,50 @@ using namespace mozilla;
 struct FuncData {
     const char *Name;      // Name of the function for the ones we use dlsym
     const void *Wrapper;   // The function that we will replace 'Function' with
     void *Function;        // The function that will be replaced with 'Wrapper'
     void *Buffer;          // Will point to the jump buffer that lets us call
                            // 'Function' after it has been replaced.
 };
 
+// This a wrapper over a file descriptor that provides a Printf method and
+// computes the sha1 of the data that passes through it.
+class SHA1Stream
+{
+public:
+    SHA1Stream(int aFd) {
+        MozillaRegisterDebugFD(aFd);
+        mFile = fdopen(aFd, "w");
+    }
+    void Printf(const char *aFormat, ...)
+    {
+        MOZ_ASSERT(mFile);
+        va_list list;
+        va_start(list, aFormat);
+        nsAutoCString str;
+        str.AppendPrintf(aFormat, list);
+        va_end(list);
+        mSHA1.update(str.get(), str.Length());
+        fwrite(str.get(), 1, str.Length(), mFile);
+    }
+    void Finish(SHA1Sum::Hash &aHash)
+    {
+        int fd = fileno(mFile);
+        fflush(mFile);
+        MozillaUnRegisterDebugFD(fd);
+        fclose(mFile);
+        mSHA1.finish(aHash);
+        mFile = NULL;
+    }
+private:
+    FILE *mFile;
+    SHA1Sum mSHA1;
+};
+
 void RecordStackWalker(void *aPC, void *aSP, void *aClosure)
 {
     std::vector<uintptr_t> *stack =
         static_cast<std::vector<uintptr_t>*>(aClosure);
     stack->push_back(reinterpret_cast<uintptr_t>(aPC));
 }
 
 char *sProfileDirectory = NULL;
@@ -62,53 +97,61 @@ bool ValidWriteAssert(bool ok)
 
     NS_StackWalk(RecordStackWalker, 0, reinterpret_cast<void*>(&rawStack), 0);
     Telemetry::ProcessedStack stack = Telemetry::GetStackAndModules(rawStack, true);
 
     nsPrintfCString nameAux("%s%s", sProfileDirectory,
                             "/Telemetry.LateWriteTmpXXXXXX");
     char *name;
     nameAux.GetMutableData(&name);
+
+    // We want the sha1 of the entire file, so please don't write to fd
+    // directly; use sha1Stream.
     int fd = mkstemp(name);
-    MozillaRegisterDebugFD(fd);
-    FILE *f = fdopen(fd, "w");
+    SHA1Stream sha1Stream(fd);
+    fd = 0;
 
     size_t numModules = stack.GetNumModules();
-    fprintf(f, "%zu\n", numModules);
+    sha1Stream.Printf("%u\n", (unsigned)numModules);
     for (int i = 0; i < numModules; ++i) {
         Telemetry::ProcessedStack::Module module = stack.GetModule(i);
-        fprintf(f, "%s\n", module.mName.c_str());
+        sha1Stream.Printf("%s\n", module.mName.c_str());
     }
 
     size_t numFrames = stack.GetStackSize();
-    fprintf(f, "%zu\n", numFrames);
+    sha1Stream.Printf("%u\n", (unsigned)numFrames);
     for (size_t i = 0; i < numFrames; ++i) {
         const Telemetry::ProcessedStack::Frame &frame =
             stack.GetFrame(i);
         // NOTE: We write the offsets, while the atos tool expects a value with
         // the virtual address added. For example, running otool -l on the the firefox
         // binary shows
         //      cmd LC_SEGMENT_64
         //      cmdsize 632
         //      segname __TEXT
         //      vmaddr 0x0000000100000000
         // so to print the line matching the offset 123 one has to run
         // atos -o firefox 0x100000123.
-        fprintf(f, "%d %jx\n", frame.mModIndex, frame.mOffset);
+        sha1Stream.Printf("%d %x\n", frame.mModIndex, (unsigned)frame.mOffset);
     }
 
-    fflush(f);
-    MozillaUnRegisterDebugFD(fd);
-    fclose(f);
+    SHA1Sum::Hash sha1;
+    sha1Stream.Finish(sha1);
 
-    // FIXME: For now we just record the last write. We should write the files
-    // to filenames that include the md5. That will provide a simple early
-    // deduplication if the same bug is found in multiple runs.
-    nsPrintfCString finalName("%s%s", sProfileDirectory,
-                              "/Telemetry.LateWriteFinal-last");
+    // Note: These files should be deleted by telemetry once it reads them. If
+    // there were no telemery runs by the time we shut down, we just add files
+    // to the existing ones instead of replacing them. Given that each of these
+    // files is a bug to be fixed, that is probably the right thing to do.
+
+    // We append the sha1 of the contents to the file name. This provides a simple
+    // client side deduplication.
+    nsPrintfCString finalName("%s%s", sProfileDirectory, "/Telemetry.LateWriteFinal-");
+    for (int i = 0; i < 20; ++i) {
+        finalName.AppendPrintf("%02x", sha1[i]);
+    }
     PR_Delete(finalName.get());
     PR_Rename(name, finalName.get());
     return false;
 }
 
 // Wrap aio_write. We have not seen it before, so just assert/report it.
 typedef ssize_t (*aio_write_t)(struct aiocb *aiocbp);
 ssize_t wrap_aio_write(struct aiocb *aiocbp);
--- a/xpcom/string/public/nsTSubstring.h
+++ b/xpcom/string/public/nsTSubstring.h
@@ -402,16 +402,17 @@ class nsTSubstring_CharT
 
       void AppendASCII( const char* data, size_type length = size_type(-1) )                     { ReplaceASCII(mLength, 0, data, length); }
 
       /**
        * Append a formatted string to the current string. Uses the format
        * codes documented in prprf.h
        */
       void AppendPrintf( const char* format, ... );
+      void AppendPrintf( const char* format, va_list ap );
       void AppendInt( int32_t aInteger )
                  { AppendPrintf( "%d", aInteger ); }
       void AppendInt( int32_t aInteger, int aRadix )
         {
           const char *fmt = aRadix == 10 ? "%d" : aRadix == 8 ? "%o" : "%x";
           AppendPrintf( fmt, aInteger );
         }
       void AppendInt( uint32_t aInteger )
@@ -761,17 +762,16 @@ class nsTSubstring_CharT
          */
       void SetDataFlags(uint32_t dataFlags)
         {
           NS_ASSERTION((dataFlags & 0xFFFF0000) == 0, "bad flags");
           mFlags = dataFlags | (mFlags & 0xFFFF0000);
         }
 
       static int AppendFunc( void* arg, const char* s, uint32_t len);
-      void AppendPrintf( const char* format, va_list ap );
 
     public:
 
       // mFlags is a bitwise combination of the following flags.  the meaning
       // and interpretation of these flags is an implementation detail.
       // 
       // NOTE: these flags are declared public _only_ for convenience inside
       // the string implementation.