--- a/toolkit/devtools/server/actors/script.js
+++ b/toolkit/devtools/server/actors/script.js
@@ -15,16 +15,18 @@ let TYPED_ARRAY_CLASSES = ["Uint8Array",
 let OBJECT_PREVIEW_MAX_ITEMS = 10;
 
 /**
  * BreakpointStore objects keep track of all breakpoints that get set so that we
  * can reset them when the same script is introduced to the thread again (such
  * as after a refresh).
  */
 function BreakpointStore() {
+  this._size = 0;
+
   // If we have a whole-line breakpoint set at LINE in URL, then
   //
   //   this._wholeLineBreakpoints[URL][LINE]
   //
   // is an object
   //
   //   { url, line[, actor] }
   //
@@ -39,16 +41,18 @@ function BreakpointStore() {
   //
   //   { url, line[, actor] }
   //
   // where the `actor` property is optional.
   this._breakpoints = Object.create(null);
 }
 
 BreakpointStore.prototype = {
+  _size: null,
+  get size() { return this._size; },
 
   /**
    * Add a breakpoint to the breakpoint store.
    *
    * @param Object aBreakpoint
    *        The breakpoint to be added (not copied). It is an object with the
    *        following properties:
    *          - url
@@ -70,49 +74,57 @@ BreakpointStore.prototype = {
       this._breakpoints[url][line][column] = aBreakpoint;
     } else {
       // Add a breakpoint that breaks on the whole line.
       if (!this._wholeLineBreakpoints[url]) {
         this._wholeLineBreakpoints[url] = [];
       }
       this._wholeLineBreakpoints[url][line] = aBreakpoint;
     }
+
+    this._size++;
   },
 
   /**
    * Remove a breakpoint from the breakpoint store.
    *
    * @param Object aBreakpoint
    *        The breakpoint to be removed. It is an object with the following
    *        properties:
    *          - url
    *          - line
    *          - column (optional)
    */
   removeBreakpoint: function ({ url, line, column }) {
     if (column != null) {
       if (this._breakpoints[url]) {
         if (this._breakpoints[url][line]) {
-          delete this._breakpoints[url][line][column];
-
-          // If this was the last breakpoint on this line, delete the line from
-          // `this._breakpoints[url]` as well. Otherwise `_iterLines` will yield
-          // this line even though we no longer have breakpoints on
-          // it. Furthermore, we use Object.keys() instead of just checking
-          // `this._breakpoints[url].length` directly, because deleting
-          // properties from sparse arrays doesn't update the `length` property
-          // like adding them does.
-          if (Object.keys(this._breakpoints[url][line]).length === 0) {
-            delete this._breakpoints[url][line];
+          if (this._breakpoints[url][line][column]) {
+            delete this._breakpoints[url][line][column];
+            this._size--;
+
+            // If this was the last breakpoint on this line, delete the line from
+            // `this._breakpoints[url]` as well. Otherwise `_iterLines` will yield
+            // this line even though we no longer have breakpoints on
+            // it. Furthermore, we use Object.keys() instead of just checking
+            // `this._breakpoints[url].length` directly, because deleting
+            // properties from sparse arrays doesn't update the `length` property
+            // like adding them does.
+            if (Object.keys(this._breakpoints[url][line]).length === 0) {
+              delete this._breakpoints[url][line];
+            }
           }
         }
       }
     } else {
       if (this._wholeLineBreakpoints[url]) {
-        delete this._wholeLineBreakpoints[url][line];
+        if (this._wholeLineBreakpoints[url][line]) {
+          delete this._wholeLineBreakpoints[url][line];
+          this._size--;
+        }
       }
     }
   },
 
   /**
    * Get a breakpoint from the breakpoint store. Will throw an error if the
    * breakpoint is not found.
    *
@@ -2215,16 +2227,20 @@ ThreadActor.prototype = {
     }
     return true;
   },
 
   /**
    * Restore any pre-existing breakpoints to the scripts that we have access to.
    */
   _restoreBreakpoints: function () {
+    if (this.breakpointStore.size === 0) {
+      return;
+    }
+
     for (let s of this.dbg.findScripts()) {
       this._addScript(s);
     }
   },
 
   /**
    * Add the provided script to the server cache.
    *