--- a/layout/style/FontFaceSet.cpp
+++ b/layout/style/FontFaceSet.cpp
@@ -347,32 +347,24 @@ FontFaceSet::Length()
 static PLDHashOperator DestroyIterator(nsPtrHashKey<nsFontFaceLoader>* aKey,
                                        void* aUserArg)
 {
   aKey->GetKey()->Cancel();
   return PL_DHASH_REMOVE;
 }
 
 void
-FontFaceSet::DisconnectFromRule(FontFace* aFontFace)
-{
-  nsCSSFontFaceRule* rule = aFontFace->GetRule();
-  aFontFace->DisconnectFromRule();
-  mRuleFaceMap.Remove(rule);
-}
-
-void
 FontFaceSet::DestroyUserFontSet()
 {
   Disconnect();
   mDocument = nullptr;
   mPresContext = nullptr;
   mLoaders.EnumerateEntries(DestroyIterator, nullptr);
   for (size_t i = 0; i < mRuleFaces.Length(); i++) {
-    DisconnectFromRule(mRuleFaces[i].mFontFace);
+    mRuleFaces[i].mFontFace->DisconnectFromRule();
     mRuleFaces[i].mFontFace->SetUserFontEntry(nullptr);
   }
   for (size_t i = 0; i < mNonRuleFaces.Length(); i++) {
     mNonRuleFaces[i]->SetUserFontEntry(nullptr);
   }
   for (size_t i = 0; i < mUnavailableFaces.Length(); i++) {
     mUnavailableFaces[i]->SetUserFontEntry(nullptr);
   }
@@ -587,17 +579,17 @@ FontFaceSet::UpdateRules(const nsTArray<
         }
       }
 
       // Any left over FontFace objects should also cease being rule backed.
       MOZ_ASSERT(!mUnavailableFaces.Contains(f),
                  "FontFace should not occur in mUnavailableFaces twice");
 
       mUnavailableFaces.AppendElement(f);
-      DisconnectFromRule(f);
+      f->DisconnectFromRule();
     }
   }
 
   if (modified) {
     IncrementGeneration(true);
     mHasLoadingFontFacesIsDirty = true;
     CheckLoadingStarted();
     CheckLoadingFinished();
@@ -983,17 +975,17 @@ FontFaceSet::FindRuleForUserFontEntry(gf
     }
   }
   return nullptr;
 }
 
 gfxUserFontEntry*
 FontFaceSet::FindUserFontEntryForRule(nsCSSFontFaceRule* aRule)
 {
-  FontFace* f = mRuleFaceMap.Get(aRule);
+  FontFace* f = aRule->GetFontFace();
   if (f) {
     return f->GetUserFontEntry();
   }
   return nullptr;
 }
 
 nsresult
 FontFaceSet::LogMessage(gfxUserFontEntry* aUserFontEntry,
@@ -1268,33 +1260,27 @@ FontFaceSet::DoRebuildUserFontSet()
   }
 
   mPresContext->RebuildUserFontSet();
 }
 
 FontFace*
 FontFaceSet::FontFaceForRule(nsCSSFontFaceRule* aRule)
 {
-  MOZ_ASSERT(aRule);
-
-  FontFace* f = mRuleFaceMap.Get(aRule);
+  FontFace* f = aRule->GetFontFace();
   if (f) {
-    MOZ_ASSERT(f->GetFontFaceSet() == this,
-               "existing FontFace is from another FontFaceSet?");
     return f;
   }
 
   // We might be creating a FontFace object for an @font-face rule that we are
   // just about to create a user font entry for, so entry might be null.
   gfxUserFontEntry* entry = FindUserFontEntryForRule(aRule);
   nsRefPtr<FontFace> newFontFace =
     FontFace::CreateForRule(GetParentObject(), mPresContext, aRule, entry);
-  MOZ_ASSERT(newFontFace->GetFontFaceSet() == this,
-             "new FontFace is from another FontFaceSet?");
-  mRuleFaceMap.Put(aRule, newFontFace);
+  aRule->SetFontFace(newFontFace);
   return newFontFace;
 }
 
 void
 FontFaceSet::AddUnavailableFontFace(FontFace* aFontFace)
 {
   MOZ_ASSERT(!aFontFace->HasRule());
   MOZ_ASSERT(!aFontFace->IsInFontFaceSet());

--- a/layout/style/FontFaceSet.h
+++ b/layout/style/FontFaceSet.h
@@ -192,22 +192,16 @@ private:
   bool HasAvailableFontFace(FontFace* aFontFace);
 
   /**
    * Removes any listeners and observers.
    */
   void Disconnect();
 
   /**
-   * Calls DisconnectFromRule on the given FontFace and removes its entry from
-   * mRuleFaceMap.
-   */
-  void DisconnectFromRule(FontFace* aFontFace);
-
-  /**
    * Returns whether there might be any pending font loads, which should cause
    * the mReady Promise not to be resolved yet.
    */
   bool MightHavePendingFontLoads();
 
   /**
    * Checks to see whether it is time to replace mReady and dispatch a
    * "loading" event.
@@ -306,23 +300,16 @@ private:
   // The non rule backed FontFace objects that have been added to this
   // FontFaceSet and their corresponding user font entries.
   nsTArray<nsRefPtr<FontFace>> mNonRuleFaces;
 
   // The non rule backed FontFace objects that have not been added to
   // this FontFaceSet.
   nsTArray<FontFace*> mUnavailableFaces;
 
-  // Map of nsCSSFontFaceRule objects to FontFace objects.  We hold a weak
-  // reference to both; for actively used FontFaces, mRuleFaces will hold
-  // a strong reference to the FontFace and the FontFace will hold on to
-  // the nsCSSFontFaceRule.  FontFaceSet::DisconnectFromRule will ensure its
-  // entry in this array will be removed.
-  nsDataHashtable<nsPtrHashKey<nsCSSFontFaceRule>, FontFace*> mRuleFaceMap;
-
   // The overall status of the loading or loaded fonts in the FontFaceSet.
   mozilla::dom::FontFaceSetLoadStatus mStatus;
 
   // Whether mNonRuleFaces has changed since last time UpdateRules ran.
   bool mNonRuleFacesDirty;
 
   // Whether we have called MaybeResolve() on mReady.
   bool mReadyIsResolved;