Bug 698326 - Add memory reporter for the url-classifier. r=njn,jlebar
authorGian-Carlo Pascutto <gpascutto@mozilla.com>
Sun, 13 Nov 2011 11:25:48 +0100
changeset 80231 45e2baac1fcaae4713abc121d4dd8d6baf6a9054
parent 80230 1b823c46084b5a40d78fb23dfddef9d87f20f22f
child 80232 736d3db54c41b27649855d856d904918f500f922
push id323
push userrcampbell@mozilla.com
push dateTue, 15 Nov 2011 21:58:36 +0000
treeherderfx-team@3ea216303184 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnjn, jlebar
bugs698326
milestone11.0a1
Bug 698326 - Add memory reporter for the url-classifier. r=njn,jlebar
toolkit/components/build/nsToolkitCompsCID.h
toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h
toolkit/components/url-classifier/tests/unit/test_prefixset.js
--- a/toolkit/components/build/nsToolkitCompsCID.h
+++ b/toolkit/components/build/nsToolkitCompsCID.h
@@ -155,19 +155,19 @@
 // {59648a91-5a60-4122-8ff2-54b839c84aed}
 #define NS_PARENTALCONTROLSSERVICE_CID \
 { 0x580530e5, 0x118c, 0x4bc7, { 0xab, 0x88, 0xbc, 0x2c, 0xd2, 0xb9, 0x72, 0x23 } }
 
 // {e7f70966-9a37-48d7-8aeb-35998f31090e}
 #define NS_TYPEAHEADFIND_CID \
 { 0xe7f70966, 0x9a37, 0x48d7, { 0x8a, 0xeb, 0x35, 0x99, 0x8f, 0x31, 0x09, 0x0e} }
 
-// {42ef1d52-3351-4973-98f8-d18f089bccfa}
+// {5edc87c2-6960-44e5-8431-bdfbb56f6aff}
 #define NS_URLCLASSIFIERPREFIXSET_CID \
-{ 0x42ef1d52, 0x3351, 0x4973, { 0x98, 0xf8, 0xd1, 0x8f, 0x08, 0x9b, 0xcc, 0xfa} }
+{ 0x5edc87c2, 0x6960, 0x44e5, { 0x84, 0x31, 0xbd, 0xfb, 0xb5, 0x6f, 0x6a, 0xff} }
 
 // {5eb7c3c1-ec1f-4007-87cc-eefb37d68ce6}
 #define NS_URLCLASSIFIERDBSERVICE_CID \
 { 0x5eb7c3c1, 0xec1f, 0x4007, { 0x87, 0xcc, 0xee, 0xfb, 0x37, 0xd6, 0x8c, 0xe6} }
 
 // {c2be6dc0-ef1e-4abd-86a2-4f864ddc57f6}
 #define NS_URLCLASSIFIERSTREAMUPDATER_CID \
 { 0xc2be6dc0, 0xef1e, 0x4abd, { 0x86, 0xa2, 0x4f, 0x86, 0x4d, 0xdc, 0x57, 0xf6} }
--- a/toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl
+++ b/toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl
@@ -36,24 +36,24 @@
  *
  * ***** END LICENSE BLOCK ***** */
 
 #include "nsISupports.idl"
 #include "nsIFile.idl"
 
 interface nsIArray;
 
-[scriptable, uuid(42ef1d52-3351-4973-98f8-d18f089bccfa)]
+[scriptable, uuid(5edc87c2-6960-44e5-8431-bdfbb56f6aff)]
 interface nsIUrlClassifierPrefixSet : nsISupports
 {
   void setPrefixes([const, array, size_is(aLength)] in unsigned long aPrefixes,
                    in unsigned long aLength);
   void addPrefixes([const, array, size_is(aLength)] in unsigned long aPrefixes,
                    in unsigned long aLength);
   boolean contains(in unsigned long aPrefix);
   boolean probe(in unsigned long aPrefix, in unsigned long aKey,
                 inout boolean aReady);
-  PRUint32 estimateSize();
+  PRUint32 sizeOfIncludingThis(in boolean aCountMe);
   PRUint32 getKey();
   boolean isEmpty();
   void loadFromFile(in nsIFile aFile);
   void storeToFile(in nsIFile aFile);
 };
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -3648,17 +3648,17 @@ nsUrlClassifierDBServiceWorker::LoadPref
   }
   if (!exists || NS_FAILED(rv)) {
     LOG(("no (usable) stored PrefixSet found, constructing from store"));
     ConstructPrefixSet();
   }
 
 #ifdef DEBUG
   PRUint32 size = 0;
-  rv = mPrefixSet->EstimateSize(&size);
+  rv = mPrefixSet->SizeOfIncludingThis(true, &size);
   LOG(("SB tree done, size = %d bytes\n", size));
   NS_ENSURE_SUCCESS(rv, rv);
 #endif
 #if defined(PR_LOGGING)
   if (LOG_ENABLED()) {
     PRIntervalTime clockEnd = PR_IntervalNow();
     LOG(("Loading took %dms\n",
          PR_IntervalToMilliseconds(clockEnd - clockStart)));
--- a/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
@@ -37,16 +37,17 @@
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
 #include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
 #include "nsDebug.h"
 #include "nsTArray.h"
+#include "nsString.h"
 #include "nsUrlClassifierPrefixSet.h"
 #include "nsIUrlClassifierPrefixSet.h"
 #include "nsIRandomGenerator.h"
 #include "nsIFile.h"
 #include "nsILocalFile.h"
 #include "nsToolkitCompsCID.h"
 #include "nsTArray.h"
 #include "nsThreadUtils.h"
@@ -62,16 +63,88 @@ using namespace mozilla;
 static const PRLogModuleInfo *gUrlClassifierPrefixSetLog = nsnull;
 #define LOG(args) PR_LOG(gUrlClassifierPrefixSetLog, PR_LOG_DEBUG, args)
 #define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierPrefixSetLog, 4)
 #else
 #define LOG(args)
 #define LOG_ENABLED() (false)
 #endif
 
+class nsPrefixSetReporter : public nsIMemoryReporter
+{
+public:
+  nsPrefixSetReporter(nsUrlClassifierPrefixSet * aParent, const nsACString & aName);
+  virtual ~nsPrefixSetReporter() {};
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMEMORYREPORTER
+
+private:
+  nsCString mPath;
+  nsUrlClassifierPrefixSet * mParent;
+};
+
+NS_IMPL_THREADSAFE_ISUPPORTS1(nsPrefixSetReporter, nsIMemoryReporter)
+
+nsPrefixSetReporter::nsPrefixSetReporter(nsUrlClassifierPrefixSet * aParent,
+                                         const nsACString & aName)
+: mParent(aParent)
+{
+  mPath.Assign(NS_LITERAL_CSTRING("explicit/storage/prefixset"));
+  if (!aName.IsEmpty()) {
+    mPath.Append("/");
+    mPath.Append(aName);
+  }
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetProcess(nsACString & aProcess)
+{
+  aProcess.Truncate();
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetPath(nsACString & aPath)
+{
+  aPath.Assign(mPath);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetKind(PRInt32 * aKind)
+{
+  *aKind = nsIMemoryReporter::KIND_HEAP;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetUnits(PRInt32 * aUnits)
+{
+  *aUnits = nsIMemoryReporter::UNITS_BYTES;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetAmount(PRInt64 * aAmount)
+{
+  PRUint32 size;
+  nsresult rv = mParent->SizeOfIncludingThis(true, &size);
+  *aAmount = size;
+  return rv;
+}
+
+NS_IMETHODIMP
+nsPrefixSetReporter::GetDescription(nsACString & aDescription)
+{
+  aDescription.Assign(NS_LITERAL_CSTRING("Memory used by a PrefixSet for "
+                                         "UrlClassifier, in bytes."));
+  return NS_OK;
+}
+
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsUrlClassifierPrefixSet, nsIUrlClassifierPrefixSet)
 
 nsUrlClassifierPrefixSet::nsUrlClassifierPrefixSet()
   : mPrefixSetLock("mPrefixSetLock"),
     mSetIsReady(mPrefixSetLock, "mSetIsReady"),
     mHasPrefixes(false),
     mRandomKey(0)
 {
@@ -79,16 +152,24 @@ nsUrlClassifierPrefixSet::nsUrlClassifie
   if (!gUrlClassifierPrefixSetLog)
     gUrlClassifierPrefixSetLog = PR_NewLogModule("UrlClassifierPrefixSet");
 #endif
 
   nsresult rv = InitKey();
   if (NS_FAILED(rv)) {
     LOG(("Failed to initialize PrefixSet"));
   }
+
+  mReporter = new nsPrefixSetReporter(this, NS_LITERAL_CSTRING("all"));
+  NS_RegisterMemoryReporter(mReporter);
+}
+
+nsUrlClassifierPrefixSet::~nsUrlClassifierPrefixSet()
+{
+  NS_UnregisterMemoryReporter(mReporter);
 }
 
 nsresult
 nsUrlClassifierPrefixSet::InitKey()
 {
   nsCOMPtr<nsIRandomGenerator> rg =
     do_GetService("@mozilla.org/security/random-generator;1");
   NS_ENSURE_STATE(rg);
@@ -194,16 +275,18 @@ PRUint32 nsUrlClassifierPrefixSet::BinSe
     }
   }
   return end;
 }
 
 NS_IMETHODIMP
 nsUrlClassifierPrefixSet::Contains(PRUint32 aPrefix, bool * aFound)
 {
+  mPrefixSetLock.AssertCurrentThreadOwns();
+
   *aFound = false;
 
   if (!mHasPrefixes) {
     return NS_OK;
   }
 
   PRUint32 target = aPrefix;
 
@@ -239,25 +322,28 @@ nsUrlClassifierPrefixSet::Contains(PRUin
   if (diff == 0) {
     *aFound = true;
   }
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsUrlClassifierPrefixSet::EstimateSize(PRUint32 * aSize)
+nsUrlClassifierPrefixSet::SizeOfIncludingThis(bool aCountMe, PRUint32 * aSize)
 {
   MutexAutoLock lock(mPrefixSetLock);
-  *aSize = sizeof(bool);
-  if (mHasPrefixes) {
-    *aSize += sizeof(PRUint16) * mDeltas.Length();
-    *aSize += sizeof(PRUint32) * mIndexPrefixes.Length();
-    *aSize += sizeof(PRUint32) * mIndexStarts.Length();
+  if (aCountMe) {
+    size_t usable = moz_malloc_usable_size(this);
+    *aSize = (PRUint32)(usable ? usable : sizeof(*this));
+  } else {
+    *aSize = 0;
   }
+  *aSize += mDeltas.SizeOf();
+  *aSize += mIndexPrefixes.SizeOf();
+  *aSize += mIndexStarts.SizeOf();
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsUrlClassifierPrefixSet::IsEmpty(bool * aEmpty)
 {
   MutexAutoLock lock(mPrefixSetLock);
   *aEmpty = !mHasPrefixes;
--- a/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h
+++ b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h
@@ -40,56 +40,60 @@
 
 #ifndef nsUrlClassifierPrefixSet_h_
 #define nsUrlClassifierPrefixSet_h_
 
 #include "nsISupportsUtils.h"
 #include "nsID.h"
 #include "nsIFile.h"
 #include "nsIUrlClassifierPrefixSet.h"
+#include "nsIMemoryReporter.h"
 #include "nsToolkitCompsCID.h"
 #include "mozilla/Mutex.h"
 #include "mozilla/CondVar.h"
 #include "mozilla/FileUtils.h"
 
+class nsPrefixSetReporter;
+
 class nsUrlClassifierPrefixSet : public nsIUrlClassifierPrefixSet
 {
 public:
   nsUrlClassifierPrefixSet();
-  virtual ~nsUrlClassifierPrefixSet() {};
+  virtual ~nsUrlClassifierPrefixSet();
 
   // Can send an empty Array to clean the tree
   NS_IMETHOD SetPrefixes(const PRUint32* aArray, PRUint32 aLength);
   // Given prefixes must be in sorted order and bigger than
   // anything currently in the Prefix Set
   NS_IMETHOD AddPrefixes(const PRUint32* aArray, PRUint32 aLength);
   // Does the PrefixSet contain this prefix? not thread-safe
   NS_IMETHOD Contains(PRUint32 aPrefix, bool* aFound);
   // Do a lookup in the PrefixSet
   // if aReady is set, we will block until there are any entries
   // if not set, we will return in aReady whether we were ready or not
   NS_IMETHOD Probe(PRUint32 aPrefix, PRUint32 aKey, bool* aReady, bool* aFound);
   // Return the estimated size of the set on disk and in memory,
   // in bytes
-  NS_IMETHOD EstimateSize(PRUint32* aSize);
+  NS_IMETHOD SizeOfIncludingThis(bool aCountMe, PRUint32* aSize);
   NS_IMETHOD IsEmpty(bool * aEmpty);
   NS_IMETHOD LoadFromFile(nsIFile* aFile);
   NS_IMETHOD StoreToFile(nsIFile* aFile);
   // Return a key that is used to randomize the collisions in the prefixes
   NS_IMETHOD GetKey(PRUint32* aKey);
 
   NS_DECL_ISUPPORTS
 
 protected:
   static const PRUint32 DELTAS_LIMIT = 100;
   static const PRUint32 MAX_INDEX_DIFF = (1 << 16);
   static const PRUint32 PREFIXSET_VERSION_MAGIC = 1;
 
   mozilla::Mutex mPrefixSetLock;
   mozilla::CondVar mSetIsReady;
+  nsRefPtr<nsPrefixSetReporter> mReporter;
 
   PRUint32 BinSearch(PRUint32 start, PRUint32 end, PRUint32 target);
   nsresult LoadFromFd(mozilla::AutoFDClose & fileFd);
   nsresult StoreToFd(mozilla::AutoFDClose & fileFd);
   nsresult InitKey();
 
   // boolean indicating whether |setPrefixes| has been
   // called with a non-empty array.
@@ -98,11 +102,12 @@ protected:
   PRUint32 mRandomKey;
   // the prefix for each index.
   nsTArray<PRUint32> mIndexPrefixes;
   // the value corresponds to the beginning of the run
   // (an index in |_deltas|) for the index
   nsTArray<PRUint32> mIndexStarts;
   // array containing deltas from indices.
   nsTArray<PRUint16> mDeltas;
+
 };
 
 #endif
--- a/toolkit/components/url-classifier/tests/unit/test_prefixset.js
+++ b/toolkit/components/url-classifier/tests/unit/test_prefixset.js
@@ -23,70 +23,76 @@ function arrContains(arr, target) {
       break;
   }
   if (start == end)
     i = start;
 
   return (!(i < 0 || i >= arr.length) && arr[i] == target);
 }
 
+function wrappedProbe(pset, prefix) {
+  let key = pset.getKey();
+  let dummy = {};
+  return pset.probe(prefix, key, dummy);
+};
+
 // doRandomLookups: we use this to test for false membership with random input
 // over the range of prefixes (unsigned 32-bits integers).
 //    pset: a nsIUrlClassifierPrefixSet to test.
 //    prefixes: an array of prefixes supposed to make up the prefix set.
 //    N: number of random lookups to make.
 function doRandomLookups(pset, prefixes, N) {
   for (let i = 0; i < N; i++) {
     let randInt = prefixes[0];
     while (arrContains(prefixes, randInt))
       randInt = Math.floor(Math.random() * Math.pow(2, 32));
 
-    do_check_false(pset.contains(randInt));
+    do_check_false(wrappedProbe(pset, randInt));
   }
 }
 
 // doExpectedLookups: we use this to test expected membership.
 //    pset: a nsIUrlClassifierPrefixSet to test.
 //    prefixes:
 function doExpectedLookups(pset, prefixes, N) {
   for (let i = 0; i < N; i++) {
     prefixes.forEach(function (x) {
       dump("Checking " + x + "\n");
-      do_check_true(pset.contains(x));
+      do_check_true(wrappedProbe(pset, x));
     });
   }
 }
 
 // testBasicPset: A very basic test of the prefix set to make sure that it
 // exists and to give a basic example of its use.
 function testBasicPset() {
   let pset = Cc["@mozilla.org/url-classifier/prefixset;1"]
                .createInstance(Ci.nsIUrlClassifierPrefixSet);
   let prefixes = [2,100,50,2000,78000,1593203];
   pset.setPrefixes(prefixes, prefixes.length);
 
-  do_check_true(pset.contains(100));
-  do_check_false(pset.contains(100000));
-  do_check_true(pset.contains(1593203));
-  do_check_false(pset.contains(999));
-  do_check_false(pset.contains(0));
+  do_check_true(wrappedProbe(pset, 100));
+  do_check_false(wrappedProbe(pset, 100000));
+  do_check_true(wrappedProbe(pset, 1593203));
+  do_check_false(wrappedProbe(pset, 999));
+  do_check_false(wrappedProbe(pset, 0));
 }
 
 function testDuplicates() {
   let pset = Cc["@mozilla.org/url-classifier/prefixset;1"]
                .createInstance(Ci.nsIUrlClassifierPrefixSet);
   let prefixes = [1,1,2,2,2,3,3,3,3,3,3,5,6,6,7,7,9,9,9];
   pset.setPrefixes(prefixes, prefixes.length);
 
-  do_check_true(pset.contains(1));
-  do_check_true(pset.contains(2));
-  do_check_true(pset.contains(5));
-  do_check_true(pset.contains(9));
-  do_check_false(pset.contains(4));
-  do_check_false(pset.contains(8));
+  do_check_true(wrappedProbe(pset, 1));
+  do_check_true(wrappedProbe(pset, 2));
+  do_check_true(wrappedProbe(pset, 5));
+  do_check_true(wrappedProbe(pset, 9));
+  do_check_false(wrappedProbe(pset, 4));
+  do_check_false(wrappedProbe(pset, 8));
 }
 
 function testSimplePset() {
   let pset = newPset();
   let prefixes = [1,2,100,400,123456789];
   pset.setPrefixes(prefixes, prefixes.length);
 
   doRandomLookups(pset, prefixes, 100);
@@ -109,17 +115,17 @@ function testReSetPrefixes() {
 
   doExpectedLookups(pset, prefixes, 1);
 
   let secondPrefixes = [12, 50, 300, 2000, 5000, 200000];
   pset.setPrefixes(secondPrefixes, secondPrefixes.length);
 
   doExpectedLookups(pset, secondPrefixes, 1);
   for (let i = 0; i < prefixes.length; i++) {
-    do_check_false(pset.contains(prefixes[i]));
+    do_check_false(wrappedProbe(pset, prefixes[i]));
   }
 }
 
 function testLargeSet() {
   let N = 1000;
   let arr = [];
 
   for (let i = 0; i < N; i++) {
@@ -137,22 +143,22 @@ function testLargeSet() {
 }
 
 function testTinySet() {
   let pset = Cc["@mozilla.org/url-classifier/prefixset;1"]
                .createInstance(Ci.nsIUrlClassifierPrefixSet);
   let prefixes = [1];
   pset.setPrefixes(prefixes, prefixes.length);
 
-  do_check_true(pset.contains(1));
-  do_check_false(pset.contains(100000));
+  do_check_true(wrappedProbe(pset, 1));
+  do_check_false(wrappedProbe(pset, 100000));
 
   prefixes = [];
   pset.setPrefixes(prefixes, prefixes.length);
-  do_check_false(pset.contains(1));
+  do_check_false(wrappedProbe(pset, 1));
 }
 
 let tests = [testBasicPset,
              testSimplePset,
              testUnsortedPset,
              testReSetPrefixes,
              testLargeSet,
              testDuplicates,