Bug 1181258 - Limit URI length the predictor keeps. r=honzab
authorNicholas Hurley <hurley@todesschaf.org>
Mon, 06 Jul 2015 15:36:03 -0700
changeset 287691 6c98d3927a163d54d618c8264b4bea7a5411971d
parent 287690 7aeee9f7969d87dafd3ebdec94a0af914c90fee6
child 287692 39c4a62c20a1f44219132e31ef8c452c0b6a13a4
push id934
push userraliiev@mozilla.com
push dateMon, 26 Oct 2015 12:58:05 +0000
treeherdermozilla-release@05704e35c1d0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershonzab
bugs1181258
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1181258 - Limit URI length the predictor keeps. r=honzab
modules/libpref/init/all.js
netwerk/base/Predictor.cpp
netwerk/base/Predictor.h
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1692,16 +1692,17 @@ pref("network.predictor.subresource-degr
 pref("network.predictor.subresource-degradation.week", 10);
 pref("network.predictor.subresource-degradation.month", 25);
 pref("network.predictor.subresource-degradation.year", 50);
 pref("network.predictor.subresource-degradation.max", 100);
 pref("network.predictor.preconnect-min-confidence", 90);
 pref("network.predictor.preresolve-min-confidence", 60);
 pref("network.predictor.redirect-likely-confidence", 75);
 pref("network.predictor.max-resources-per-entry", 100);
+pref("network.predictor.max-uri-length", 500);
 pref("network.predictor.cleaned-up", false);
 
 // The following prefs pertain to the negotiate-auth extension (see bug 17578),
 // which provides transparent Kerberos or NTLM authentication using the SPNEGO
 // protocol.  Each pref is a comma-separated list of keys, where each key has
 // the format:
 //   [scheme "://"] [host [":" port]]
 // For example, "foo.com" would match "http://www.foo.com/bar", etc.
--- a/netwerk/base/Predictor.cpp
+++ b/netwerk/base/Predictor.cpp
@@ -112,16 +112,22 @@ const int32_t PRERESOLVE_MIN_DEFAULT = 6
 const char PREDICTOR_REDIRECT_LIKELY_PREF[] =
   "network.predictor.redirect-likely-confidence";
 const int32_t REDIRECT_LIKELY_DEFAULT = 75;
 
 const char PREDICTOR_MAX_RESOURCES_PREF[] =
   "network.predictor.max-resources-per-entry";
 const uint32_t PREDICTOR_MAX_RESOURCES_DEFAULT = 100;
 
+// This is selected in concert with max-resources-per-entry to keep memory usage
+// low-ish. The default of the combo of the two is ~50k
+const char PREDICTOR_MAX_URI_LENGTH_PREF[] =
+  "network.predictor.max-uri-length";
+const uint32_t PREDICTOR_MAX_URI_LENGTH_DEFAULT = 500;
+
 const char PREDICTOR_CLEANED_UP_PREF[] = "network.predictor.cleaned-up";
 
 // All these time values are in sec
 const uint32_t ONE_DAY = 86400U;
 const uint32_t ONE_WEEK = 7U * ONE_DAY;
 const uint32_t ONE_MONTH = 30U * ONE_DAY;
 const uint32_t ONE_YEAR = 365U * ONE_DAY;
 
@@ -309,16 +315,17 @@ Predictor::Predictor()
   ,mSubresourceDegradationMonth(PREDICTOR_SUB_DELTA_MONTH_DEFAULT)
   ,mSubresourceDegradationYear(PREDICTOR_SUB_DELTA_YEAR_DEFAULT)
   ,mSubresourceDegradationMax(PREDICTOR_SUB_DELTA_MAX_DEFAULT)
   ,mPreconnectMinConfidence(PRECONNECT_MIN_DEFAULT)
   ,mPreresolveMinConfidence(PRERESOLVE_MIN_DEFAULT)
   ,mRedirectLikelyConfidence(REDIRECT_LIKELY_DEFAULT)
   ,mMaxResourcesPerEntry(PREDICTOR_MAX_RESOURCES_DEFAULT)
   ,mStartupCount(1)
+  ,mMaxURILength(PREDICTOR_MAX_URI_LENGTH_DEFAULT)
 {
   gPredictorLog = PR_NewLogModule("NetworkPredictor");
 
   MOZ_ASSERT(!sSelf, "multiple Predictor instances!");
   sSelf = this;
 }
 
 Predictor::~Predictor()
@@ -397,16 +404,19 @@ Predictor::InstallObserver()
                               REDIRECT_LIKELY_DEFAULT);
 
   Preferences::AddIntVarCache(&mMaxResourcesPerEntry,
                               PREDICTOR_MAX_RESOURCES_PREF,
                               PREDICTOR_MAX_RESOURCES_DEFAULT);
 
   Preferences::AddBoolVarCache(&mCleanedUp, PREDICTOR_CLEANED_UP_PREF, false);
 
+  Preferences::AddUintVarCache(&mMaxURILength, PREDICTOR_MAX_URI_LENGTH_PREF,
+                               PREDICTOR_MAX_URI_LENGTH_DEFAULT);
+
   if (!mCleanedUp) {
     mCleanupTimer = do_CreateInstance("@mozilla.org/timer;1");
     mCleanupTimer->Init(this, 60 * 1000, nsITimer::TYPE_ONE_SHOT);
   }
 
   return rv;
 }
 
@@ -506,25 +516,34 @@ class NuwaMarkPredictorThreadRunner : pu
   }
 };
 } // namespace
 #endif
 
 // Predictor::nsICacheEntryMetaDataVisitor
 
 #define SEEN_META_DATA "predictor::seen"
+#define RESOURCE_META_DATA "predictor::resource-count"
 #define META_DATA_PREFIX "predictor::"
+
+static bool
+IsURIMetadataElement(const char *key)
+{
+  return StringBeginsWith(nsDependentCString(key),
+                          NS_LITERAL_CSTRING(META_DATA_PREFIX)) &&
+         !NS_LITERAL_CSTRING(SEEN_META_DATA).Equals(key) &&
+         !NS_LITERAL_CSTRING(RESOURCE_META_DATA).Equals(key);
+}
+
 nsresult
 Predictor::OnMetaDataElement(const char *asciiKey, const char *asciiValue)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (!StringBeginsWith(nsDependentCString(asciiKey),
-                        NS_LITERAL_CSTRING(META_DATA_PREFIX)) ||
-      NS_LITERAL_CSTRING(SEEN_META_DATA).Equals(asciiKey)) {
+  if (!IsURIMetadataElement(asciiKey)) {
     // This isn't a bit of metadata we care about
     return NS_OK;
   }
 
   nsCString key, value;
   key.AssignASCII(asciiKey);
   value.AssignASCII(asciiValue);
   mKeysToOperateOn.AppendElement(key);
@@ -1427,17 +1446,21 @@ Predictor::LearnInternal(PredictorLearnR
   PREDICTOR_LOG(("Predictor::LearnInternal"));
 
   nsCString junk;
   if (!fullUri && reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL &&
       NS_FAILED(entry->GetMetaDataElement(SEEN_META_DATA, getter_Copies(junk)))) {
     // This is an origin-only entry that we haven't seen before. Let's mark it
     // as seen.
     PREDICTOR_LOG(("    marking new origin entry as seen"));
-    entry->SetMetaDataElement(SEEN_META_DATA, "1");
+    nsresult rv = entry->SetMetaDataElement(SEEN_META_DATA, "1");
+    if (NS_FAILED(rv)) {
+      PREDICTOR_LOG(("    failed to mark origin entry seen"));
+      return;
+    }
 
     // Need to ensure someone else can get to the entry if necessary
     entry->MetaDataReady();
     return;
   }
 
   switch (reason) {
     case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL:
@@ -1465,43 +1488,68 @@ Predictor::LearnInternal(PredictorLearnR
 
 NS_IMPL_ISUPPORTS(Predictor::SpaceCleaner, nsICacheEntryMetaDataVisitor)
 
 NS_IMETHODIMP
 Predictor::SpaceCleaner::OnMetaDataElement(const char *key, const char *value)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (!StringBeginsWith(nsDependentCString(key),
-                        NS_LITERAL_CSTRING(META_DATA_PREFIX)) ||
-      NS_LITERAL_CSTRING(SEEN_META_DATA).Equals(key)) {
+  if (!IsURIMetadataElement(key)) {
     // This isn't a bit of metadata we care about
     return NS_OK;
   }
 
-  nsCOMPtr<nsIURI> sanityCheck;
+  nsCOMPtr<nsIURI> parsedURI;
   uint32_t hitCount, lastHit, flags;
   bool ok = mPredictor->ParseMetaDataEntry(key, value,
-                                           getter_AddRefs(sanityCheck),
+                                           getter_AddRefs(parsedURI),
                                            hitCount, lastHit, flags);
 
-  if (!ok || !mKeyToDelete || lastHit < mLRUStamp) {
-    mKeyToDelete = key;
+  if (!ok) {
+    // Couldn't parse this one, just get rid of it
+    nsCString nsKey;
+    nsKey.AssignASCII(key);
+    mLongKeysToDelete.AppendElement(nsKey);
+    return NS_OK;
+  }
+
+  nsCString uri;
+  nsresult rv = parsedURI->GetAsciiSpec(uri);
+  uint32_t uriLength = uri.Length();
+  if (ok && NS_SUCCEEDED(rv) &&
+      uriLength > mPredictor->mMaxURILength) {
+    // Default to getting rid of URIs that are too long and were put in before
+    // we had our limit on URI length, in order to free up some space.
+    nsCString nsKey;
+    nsKey.AssignASCII(key);
+    mLongKeysToDelete.AppendElement(nsKey);
+    return NS_OK;
+  }
+
+  if (!ok || !mLRUKeyToDelete || lastHit < mLRUStamp) {
+    mLRUKeyToDelete = key;
     mLRUStamp = ok ? lastHit : 0;
   }
 
   return NS_OK;
 }
 
 void
 Predictor::SpaceCleaner::Finalize(nsICacheEntry *entry)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
-  entry->SetMetaDataElement(mKeyToDelete, nullptr);
+  if (mLRUKeyToDelete) {
+    entry->SetMetaDataElement(mLRUKeyToDelete, nullptr);
+  }
+
+  for (size_t i = 0; i < mLongKeysToDelete.Length(); ++i) {
+    entry->SetMetaDataElement(mLongKeysToDelete[i].BeginReading(), nullptr);
+  }
 }
 
 // Called when a subresource has been hit from a top-level load. Uses the two
 // helper functions above to update the database appropriately.
 void
 Predictor::LearnForSubresource(nsICacheEntry *entry, nsIURI *targetURI)
 {
   MOZ_ASSERT(NS_IsMainThread());
@@ -1516,65 +1564,85 @@ Predictor::LearnForSubresource(nsICacheE
   rv = entry->GetFetchCount(&loadCount);
   RETURN_IF_FAILED(rv);
 
   nsCString key;
   key.AssignLiteral(META_DATA_PREFIX);
   nsCString uri;
   targetURI->GetAsciiSpec(uri);
   key.Append(uri);
+  if (uri.Length() > mMaxURILength) {
+    // We do this to conserve space/prevent OOMs
+    PREDICTOR_LOG(("    uri too long!"));
+    entry->SetMetaDataElement(key.BeginReading(), nullptr);
+    return;
+  }
 
   nsCString value;
   rv = entry->GetMetaDataElement(key.BeginReading(), getter_Copies(value));
 
   uint32_t hitCount, lastHit, flags;
   bool isNewResource = (NS_FAILED(rv) ||
                         !ParseMetaDataEntry(nullptr, value.BeginReading(),
                                             nullptr, hitCount, lastHit, flags));
 
+  int32_t resourceCount = 0;
   if (isNewResource) {
     // This is a new addition
     PREDICTOR_LOG(("    new resource"));
-    int32_t resourceCount;
     nsCString s;
-    rv = entry->GetMetaDataElement("predictor::resource-count",
-                                   getter_Copies(s));
-    if (NS_FAILED(rv)) {
-      resourceCount = 0;
-    } else {
+    rv = entry->GetMetaDataElement(RESOURCE_META_DATA, getter_Copies(s));
+    if (NS_SUCCEEDED(rv)) {
       resourceCount = atoi(s.BeginReading());
     }
     if (resourceCount >= mMaxResourcesPerEntry) {
       nsRefPtr<Predictor::SpaceCleaner> cleaner =
         new Predictor::SpaceCleaner(this);
       entry->VisitMetaData(cleaner);
       cleaner->Finalize(entry);
     } else {
       ++resourceCount;
     }
     nsAutoCString count;
     count.AppendInt(resourceCount);
-    entry->SetMetaDataElement("predictor::resource-count", count.BeginReading());
+    rv = entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading());
+    if (NS_FAILED(rv)) {
+      PREDICTOR_LOG(("    failed to update resource count"));
+      return;
+    }
     hitCount = 1;
   } else {
     PREDICTOR_LOG(("    existing resource"));
     hitCount = std::min(hitCount + 1, static_cast<uint32_t>(loadCount));
   }
 
   nsCString newValue;
   newValue.AppendInt(METADATA_VERSION);
   newValue.AppendLiteral(",");
   newValue.AppendInt(hitCount);
   newValue.AppendLiteral(",");
   newValue.AppendInt(lastLoad);
   // These are for flags, that will be used for prefetch and possibly other
   // things later on
   newValue.AppendLiteral(",");
   newValue.AppendInt(0);
-  entry->SetMetaDataElement(key.BeginReading(), newValue.BeginReading());
+  rv = entry->SetMetaDataElement(key.BeginReading(), newValue.BeginReading());
+  PREDICTOR_LOG(("    SetMetaDataElement -> 0x%08X", rv));
+  if (NS_FAILED(rv) && isNewResource) {
+    // Roll back the increment to the resource count we made above.
+    PREDICTOR_LOG(("    rolling back resource count update"));
+    --resourceCount;
+    if (resourceCount == 0) {
+      entry->SetMetaDataElement(RESOURCE_META_DATA, nullptr);
+    } else {
+      nsAutoCString count;
+      count.AppendInt(resourceCount);
+      entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading());
+    }
+  }
 }
 
 // This is called when a top-level loaded ended up redirecting to a different
 // URI so we can keep track of that fact.
 void
 Predictor::LearnForRedirect(nsICacheEntry *entry, nsIURI *targetURI)
 {
   MOZ_ASSERT(NS_IsMainThread());
--- a/netwerk/base/Predictor.h
+++ b/netwerk/base/Predictor.h
@@ -141,26 +141,27 @@ private:
   class SpaceCleaner : public nsICacheEntryMetaDataVisitor
   {
   public:
     NS_DECL_ISUPPORTS
     NS_DECL_NSICACHEENTRYMETADATAVISITOR
 
     explicit SpaceCleaner(Predictor *predictor)
       :mLRUStamp(0)
-      ,mKeyToDelete(nullptr)
+      ,mLRUKeyToDelete(nullptr)
       ,mPredictor(predictor)
     { }
 
     void Finalize(nsICacheEntry *entry);
 
   private:
     virtual ~SpaceCleaner() { }
     uint32_t mLRUStamp;
-    const char *mKeyToDelete;
+    const char *mLRUKeyToDelete;
+    nsTArray<nsCString> mLongKeysToDelete;
     nsRefPtr<Predictor> mPredictor;
   };
 
   // Observer-related stuff
   nsresult InstallObserver();
   void RemoveObserver();
 
   // Service startup utilities
@@ -349,16 +350,18 @@ private:
   nsCOMPtr<nsIIOService> mIOService;
   nsCOMPtr<nsISpeculativeConnect> mSpeculativeService;
 
   nsCOMPtr<nsIURI> mStartupURI;
   uint32_t mStartupTime;
   uint32_t mLastStartupTime;
   int32_t mStartupCount;
 
+  uint32_t mMaxURILength;
+
   nsCOMPtr<nsIDNSService> mDnsService;
 
   nsRefPtr<DNSListener> mDNSListener;
 
   nsTArray<nsCOMPtr<nsIURI>> mPreconnects;
   nsTArray<nsCOMPtr<nsIURI>> mPreresolves;
 
   static Predictor *sSelf;