Bug 551677 - Gloda could provide ability to force indexing of all messages in a folder, even non-dirty indexed ones. r=mixedpuppy.
authorAndrew Sutherland <asutherland@asutherland.org>
Tue, 16 Mar 2010 15:36:36 -0700
changeset 5188 6417f114db07461d35db188054a2f41342f08cfe
parent 5187 6e94b4293dd2b6dd40b388a1283018417cbc9ba9
child 5189 defb1451fce4f30728c3d4160a2e7191cdb35e36
push idunknown
push userunknown
push dateunknown
reviewersmixedpuppy
bugs551677
Bug 551677 - Gloda could provide ability to force indexing of all messages in a folder, even non-dirty indexed ones. r=mixedpuppy.
mailnews/db/gloda/modules/index_msg.js
mailnews/db/gloda/modules/indexer.js
mailnews/db/gloda/test/unit/base_index_messages.js
mailnews/db/gloda/test/unit/resources/asyncCallbackHandler.js
mailnews/db/gloda/test/unit/resources/glodaTestHelper.js
mailnews/db/gloda/test/unit/test_index_sweep_folder.js
mailnews/test/resources/asyncTestUtils.js
--- a/mailnews/db/gloda/modules/index_msg.js
+++ b/mailnews/db/gloda/modules/index_msg.js
@@ -1369,23 +1369,29 @@ var GlodaMsgIndexer = {
       }
       // Commit the filthy status changes to the message database.
       this._indexingDatabase.Commit(Ci.nsMsgDBCommitType.kLargeCommit);
 
       // this will automatically persist to the database
       glodaFolder._downgradeDirtyStatus(glodaFolder.kFolderDirty);
     }
 
+    // Figure out whether we're supposed to index _everything_ or just what
+    //  has not yet been indexed.
+    let force = ("force" in aJob) && aJob.force;
+    let enumeratorType = force ? this.kEnumAllMsgs : this.kEnumMsgsToIndex;
+
     // Pass 1: count the number of messages to index.
     //  We do this in order to be able to report to the user what we're doing.
     // TODO: give up after reaching a certain number of messages in folders
     //  with ridiculous numbers of messages and make the interface just say
     //  something like "over N messages to go."
 
-    this._indexerGetEnumerator(this.kEnumMsgsToIndex);
+    this._indexerGetEnumerator(enumeratorType);
+
     let numMessagesToIndex = 0;
     let numMessagesOut = {};
     // Keep going until we run out of headers.
     while (this._indexingFolder.msgDatabase.nextMatchingHdrs(
              this._indexingEnumerator,
              HEADER_CHECK_SYNC_BLOCK_SIZE * 8, // this way is faster, do more
              0, // moot, we don't return headers
              null, // don't return headers, we just want the count
@@ -1394,17 +1400,17 @@ var GlodaMsgIndexer = {
       yield this.kWorkSync;
     }
     numMessagesToIndex += numMessagesOut.value;
 
     aJob.goal = numMessagesToIndex;
 
     if (numMessagesToIndex > 0) {
       // We used up the iterator, get a new one.
-      this._indexerGetEnumerator(this.kEnumMsgsToIndex);
+      this._indexerGetEnumerator(enumeratorType);
 
       // Pass 2: index the messages.
       let count = 0;
       for (let msgHdr in fixIterator(this._indexingEnumerator, nsIMsgDBHdr)) {
         // per above, we want to periodically release control while doing all
         // this header traversal/investigation.
         if (++count % HEADER_CHECK_SYNC_BLOCK_SIZE == 0)
           yield this.kWorkSync;
@@ -1420,19 +1426,20 @@ var GlodaMsgIndexer = {
         //  msgsClassified.
         if (this._indexingFolder.getProcessingFlags(msgHdr.messageKey) &
             NOT_YET_REPORTED_PROCESSING_FLAGS)
           continue;
 
         // Because the gloda id could be in-flight, we need to double-check the
         //  enumerator here since it can't know about our in-memory stuff.
         let [glodaId, glodaDirty] = PendingCommitTracker.getGlodaState(msgHdr);
-        // if the message seems valid, skip it.  (that means good gloda id
-        //  and not dirty)
-        if (glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+        // if the message seems valid and we are not forcing indexing, skip it.
+        //  (that means good gloda id and not dirty)
+        if (!force &&
+            glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
             glodaDirty == this.kMessageClean)
           continue;
 
         if (logDebug)
           this._log.debug(">>>  _indexMessage");
         yield aCallbackHandle.pushAndGo(
           this._indexMessage(msgHdr, aCallbackHandle),
           {what: "indexMessage", msgHdr: msgHdr});
@@ -1447,16 +1454,18 @@ var GlodaMsgIndexer = {
     //  will hit the disk but the gloda-id properties on the headers will not
     //  get set.  This should ideally be resolved by detecting a non-clean
     //  shutdown and marking all folders as dirty.
     glodaFolder._downgradeDirtyStatus(glodaFolder.kFolderClean);
 
     // by definition, it's not likely we'll visit this folder again anytime soon
     this._indexerLeaveFolder();
 
+    aJob.safelyInvokeCallback(true);
+
     yield this.kWorkDone;
   },
 
   /**
    * Invoked when a "message" job is scheduled so that we can clear
    *  _pendingAddJob if that is the job.  We do this so that work items are not
    *  added to _pendingAddJob while it is being processed.
    */
@@ -1596,16 +1605,17 @@ var GlodaMsgIndexer = {
     return false;
   },
 
   /**
    * Cleanup after an aborted "folder" or "message" job.
    */
   _cleanup_indexing: function gloda_index_cleanup_indexing(aJob) {
     this._indexerLeaveFolder();
+    aJob.safelyInvokeCallback(false);
   },
 
   /**
    * Maximum number of deleted messages to process at a time.  Arbitrary; there
    *  are no real known performance constraints at this point.
    */
   DELETED_MESSAGE_BLOCK_SIZE: 32,
 
@@ -1709,26 +1719,41 @@ var GlodaMsgIndexer = {
     }
     else {
       this._log.info("Skipping Account, root folder not nsIMsgFolder");
     }
   },
 
   /**
    * Queue a single folder for indexing given an nsIMsgFolder.
+   *
+   * @param [aOptions.callback] A callback to invoke when the folder finishes
+   *     indexing.  First argument is true if the task ran to completion
+   *     successfully, false if we had to abort for some reason.
+   * @param [aOptions.force=false] Should we force the indexing of all messages
+   *     in the folder (true) or just index what hasn't been indexed (false).
+   * @return true if we are going to index the folder, false if not.
    */
-  indexFolder: function glodaIndexFolder(aMsgFolder) {
+  indexFolder: function glodaIndexFolder(aMsgFolder, aOptions) {
     let glodaFolder = GlodaDatastore._mapFolder(aMsgFolder);
     // stay out of compacting/compacted folders
     if (glodaFolder.compacting || glodaFolder.compacted)
-      return;
+      return false;
 
     this._log.info("Queue-ing folder for indexing: " +
                    aMsgFolder.prettiestName);
-    GlodaIndexer.indexJob(new IndexingJob("folder", glodaFolder.id));
+    let job = new IndexingJob("folder", glodaFolder.id);
+    if (aOptions) {
+      if ("callback" in aOptions)
+        job.callback = aOptions.callback;
+      if ("force" in aOptions)
+        job.force = true;
+    }
+    GlodaIndexer.indexJob(job);
+    return true;
   },
 
   /**
    * Queue a list of messages for indexing.
    *
    * @param aFoldersAndMessages List of [nsIMsgFolder, message key] tuples.
    */
   indexMessages: function gloda_index_indexMessages(aFoldersAndMessages) {
--- a/mailnews/db/gloda/modules/indexer.js
+++ b/mailnews/db/gloda/modules/indexer.js
@@ -86,18 +86,31 @@ Cu.import("resource://app/modules/gloda/
  *     with the goal.
  */
 function IndexingJob(aJobType, aID, aItems) {
   this.jobType = aJobType;
   this.id = aID;
   this.items = (aItems != null) ? aItems : [];
   this.offset = 0;
   this.goal = null;
+  this.callback = null;
+  this.callbackThis = null;
 }
 IndexingJob.prototype = {
+  safelyInvokeCallback: function() {
+    if (!this.callback)
+      return;
+    try {
+      this.callback.apply(this.callbackThis, arguments);
+    }
+    catch(ex) {
+      GlodaIndexer._log.warning("job callback invocation problem: " +
+                                ex.fileName + ":" + ex.lineNumber + ": " + ex);
+    }
+  },
   toString: function IndexingJob_toString() {
     return "[job:" + this.jobType +
       " id:" + this.id + " items:" + (this.items ? this.items.length : "no") +
       " offset:" + this.offset + " goal:" + this.goal + "]";
   }
 };
 
 /**
--- a/mailnews/db/gloda/test/unit/base_index_messages.js
+++ b/mailnews/db/gloda/test/unit/base_index_messages.js
@@ -205,18 +205,30 @@ function test_indexing_sweep() {
   //  test_index_sweep_folder.js.
   mark_sub_test_start("filthy folder indexing");
   let glodaFolderC = Gloda.getFolderForFolder(
                        get_real_injection_folder(folderC));
   glodaFolderC._dirtyStatus = glodaFolderC.kFolderFilthy;
   mark_action("actual", "marked gloda folder dirty", [glodaFolderC]);
   GlodaMsgIndexer.indexingSweepNeeded = true;
   yield wait_for_gloda_indexer([setC1, setC2]);
+
+  // -- Forced folder indexing.
+  var callbackInvoked = false;
+  mark_sub_test_start("forced folder indexing");
+  GlodaMsgIndexer.indexFolder(get_real_injection_folder(folderA), {
+    force: true,
+    callback: function() {
+      callbackInvoked = true;
+    }});
+  yield wait_for_gloda_indexer([setA1, setA2]);
+  do_check_true(callbackInvoked);
 }
 
+
 /**
  * We used to screw up and downgrade filthy folders to dirty if we saw an event
  *  happen in the folder before we got to the folder; this tests that we no
  *  longer do that.
  */
 function test_event_driven_indexing_does_not_mess_with_filthy_folders() {
   // add a folder with a message.
   let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
new file mode 100644
--- /dev/null
+++ b/mailnews/db/gloda/test/unit/resources/asyncCallbackHandler.js
@@ -0,0 +1,71 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ *   Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Thunderbird Global Database.
+ *
+ * The Initial Developer of the Original Code is the Mozilla Foundation.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Andrew Sutherland <asutherland@asutherland.org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/**
+ * Implements GlodaIndexer._callbackHandle's interface adapted to our async
+ *  test driver.  This allows us to run indexing workers directly in tests
+ *  or support code.
+ *
+ * We do not do anything with the context stack or recovery.  Use the actual
+ *  indexer callback handler for that!
+ *
+ * Actually, we do very little at all right now.  This will fill out as needs
+ *  arise.
+ */
+let asyncCallbackHandle = {
+  pushAndGo: function asyncCallbackHandle_push(aIterator, aContext) {
+    asyncGeneratorStack.push([
+      _asyncCallbackHandle_glodaWorkerAdapter(aIterator),
+      "callbackHandler pushAndGo"]);
+    return async_driver();
+  }
+};
+
+function _asyncCallbackHandle_glodaWorkerAdapter(aIter) {
+  while(true) {
+    switch(aIter.next()) {
+      case GlodaIndexer.kWorkSync:
+        yield true;
+        break;
+      case GlodaIndexer.kWorkDone:
+      case GlodaIndexer.kWorkDoneWithResult:
+        return;
+      default:
+        yield false;
+        break;
+    }
+  }
+}
--- a/mailnews/db/gloda/test/unit/resources/glodaTestHelper.js
+++ b/mailnews/db/gloda/test/unit/resources/glodaTestHelper.js
@@ -8,18 +8,17 @@
  *
  * Software distributed under the License is distributed on an "AS IS" basis,
  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  * for the specific language governing rights and limitations under the
  * License.
  *
  * The Original Code is Thunderbird Global Database.
  *
- * The Initial Developer of the Original Code is
- * Mozilla Messaging, Inc.
+ * The Initial Developer of the Original Code is the Mozilla Foundation.
  * Portions created by the Initial Developer are Copyright (C) 2008
  * the Initial Developer. All Rights Reserved.
  *
  * Contributor(s):
  *   Andrew Sutherland <asutherland@asutherland.org>
  *   Siddharth Agarwal <sid.bugzilla@gmail.com>
  *
  * Alternatively, the contents of this file may be used under the terms of
--- a/mailnews/db/gloda/test/unit/test_index_sweep_folder.js
+++ b/mailnews/db/gloda/test/unit/test_index_sweep_folder.js
@@ -8,19 +8,18 @@
  *
  * Software distributed under the License is distributed on an "AS IS" basis,
  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  * for the specific language governing rights and limitations under the
  * License.
  *
  * The Original Code is Thunderbird Global Database.
  *
- * The Initial Developer of the Original Code is
- * Mozilla Messaging, Inc.
- * Portions created by the Initial Developer are Copyright (C) 2009
+ * The Initial Developer of the Original Code is the Mozilla Foundation.
+ * Portions created by the Initial Developer are Copyright (C) 2010
  * the Initial Developer. All Rights Reserved.
  *
  * Contributor(s):
  *   Andrew Sutherland <asutherland@asutherland.org>
  *
  * Alternatively, the contents of this file may be used under the terms of
  * either the GNU General Public License Version 2 or later (the "GPL"), or
  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
@@ -30,95 +29,165 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
-/* This file tests the folder indexing logic of Gloda._worker_folderIndex in
- *  the greater context of the sweep indexing mechanism.
+/*
+ * This file tests the folder indexing logic of Gloda._worker_folderIndex in
+ *  the greater context of the sweep indexing mechanism in a whitebox fashion.
  *
  * Automated indexing is suppressed for the duration of this file.
  *
- * This is all white-box testing where we know and depend on how the mechanism
- *  is supposed to work/works.  In order to test the phases of the logic we
- *  inject failures into GlodaIndexer._indexerGetEnumerator with a wrapper to
- *  control how far indexing gets.  We also clobber or wrap other functions as
- *  needed.
+ * In order to test the phases of the logic we inject failures into
+ *  GlodaIndexer._indexerGetEnumerator with a wrapper to control how far
+ *  indexing gets.  We also clobber or wrap other functions as needed.
  */
 
 load("resources/glodaTestHelper.js");
-
-/**
- * We do not index news or RSS.  Make sure we stay out of those folders.
- */
-function test_ignore_folders_we_should_not_index() {
-  // make sure it ignores news
-  // make sure it ignores RSS
-}
+load("resources/asyncCallbackHandler.js");
 
 /**
- * When we enter a filthy folder we should be marking all the messages as filthy
- *  and committing.
- *
+ * How many more enumerations before we should throw; 0 means don't throw.
+ */
+var explode_enumeration_after = 0;
+GlodaMsgIndexer._original_indexerGetEnumerator =
+  GlodaMsgIndexer._indexerGetEnumerator;
+/**
+ * Wrapper for GlodaMsgIndexer._indexerGetEnumerator to cause explosions.
  */
-function test_propagate_filthy_from_folder_to_messages() {
-  // mark the folder as filthy
+GlodaMsgIndexer._indexerGetEnumerator = function() {
+  if (explode_enumeration_after && !(--explode_enumeration_after))
+    throw asyncExpectedEarlyAbort;
 
-  // index the folder, aborting at the second get enumerator request
-
-  // all those messages better be filthy now!
-}
+  return GlodaMsgIndexer._original_indexerGetEnumerator.apply(
+    GlodaMsgIndexer, arguments);
+};
 
 /**
- * Create a folder indexing job for the given injection folder handle.  We
- *
+ * Create a folder indexing job for the given injection folder handle and
+ * run it until completion.
  */
-function spin_folder_indexer(aFolderHandle) {
+function spin_folder_indexer() {
+  return async_run({func: _spin_folder_indexer_gen,
+                    args: arguments});
+}
+function _spin_folder_indexer_gen(aFolderHandle, aExpectedJobGoal) {
   let msgFolder = get_real_injection_folder(aFolderHandle);
 
   // cheat and use indexFolder to build the job for us
   GlodaMsgIndexer.indexFolder(msgFolder);
   // steal that job...
   let job = GlodaIndexer._indexQueue.pop();
   GlodaIndexer._indexingJobGoal--;
 
   // create the worker
-  let worker = GlodaIndexer._worker_folderIndex(job);
+  let worker = GlodaMsgIndexer._worker_folderIndex(job, asyncCallbackHandle);
+  try {
+    yield asyncCallbackHandle.pushAndGo(worker);
+  }
+  catch(ex) {
+    // it's okay if this is from our exploding enumerator wrapper.
+    if (ex != expectedReaper)
+      do_throw(ex);
+  }
+
+  if (aExpectedJobGoal !== undefined)
+    do_check_eq(job.goal, aExpectedJobGoal);
+}
+
+/**
+ * The value itself does not matter; it just needs to be present and be in a
+ *  certain range for our logic testing.
+ */
+const arbitraryGlodaId = 4096;
+
+/**
+ * When we enter a filthy folder we should be marking all the messages as filthy
+ *  that have gloda-id's and committing.
+ */
+function test_propagate_filthy_from_folder_to_messages() {
+  // mark the folder as filthy
+  let [folder, msgSet] = make_folder_with_sets([{count: 3}]);
+  let glodaFolder = Gloda.getFolderForFolder(folder);
+  glodaFolder._dirtyStatus = glodaFolder.kFolderFilthy;
 
+  // mark each header with a gloda-id so they can get marked filthy
+  for each (let msgHdr in msgSet.msgHdrs) {
+    msgHdr.setUint32Property("gloda-id", arbitraryGlodaId);
+  }
 
+  // force the database to see it as filthy so we can verify it changes
+  glodaFolder._datastore.updateFolderDirtyStatus(glodaFolder);
+  yield sqlExpectCount(1,
+    "SELECT COUNT(*) FROM folderLocations WHERE id = ? " +
+      "AND dirtyStatus = ?", glodaFolder.id, glodaFolder.kFolderFilthy);
+
+  // index the folder, aborting at the second get enumerator request
+  explode_enumeration_after = 2;
+  yield spin_folder_indexer(folder);
+
+  // the folder should only be dirty
+  do_check_eq(glodaFolder.dirtyStatus, glodaFolder.kFolderDirty);
+  // make sure the database sees it as dirty
+  yield sqlExpectCount(1,
+    "SELECT COUNT(*) FROM folderLocations WHERE id = ? " +
+      "AND dirtyStatus = ?", glodaFolder.id, glodaFolder.kFolderDirty);
+
+  // The messages should be filthy per the headers (we force a commit of the
+  //  database.)
+  for each (let msgHdr in msgSet.msgHdrs) {
+    do_check_eq(msgHdr.getUint32Property("gloda-dirty"),
+                GlodaMsgIndexer.kMessageFilthy);
+  }
 }
 
+
 /**
  * Make sure our counting pass and our indexing passes gets it right.  We test
  *  with 0,1,2 messages matching.
  */
-function test_count_and_index_messages() {
-  let [folder, msgSet] = make_folder_with_sets([{count: 3}]);
+function test_count_pass() {
+  let [folder, msgSet] = make_folder_with_sets([{count: 2}]);
   yield wait_for_message_injection();
 
-  let hdrs = msgSet.msgHdrs;
+  let hdrs = msgSet.msgHdrList;
+
+  // - (clean) messages with gloda-id's do not get indexed
+  // nothing is indexed at this point, so all 2.
+  explode_enumeration_after = 2;
+  yield spin_folder_indexer(folder, 2);
 
-  // - messages with no gloda-id need to get indexed!
+  // pretend the first is indexed, leaving a count of 1.
+  hdrs[0].setUint32Property("gloda-id", arbitraryGlodaId);
+  explode_enumeration_after = 2;
+  yield spin_folder_indexer(folder, 1);
 
-  // - messages with gloda-id's do not get indexed
+  // pretend both are indexed, count of 0
+  hdrs[1].setUint32Property("gloda-id", arbitraryGlodaId);
+  // (No explosion should happen since we should never get to the second
+  //  enumerator.)
+  yield spin_folder_indexer(folder, 0);
 
   // - dirty messages get indexed
+  hdrs[0].setUint32Property("gloda-dirty", GlodaMsgIndexer.kMessageDirty);
+  explode_enumeration_after = 2;
+  yield spin_folder_indexer(folder, 1);
+
+  hdrs[1].setUint32Property("gloda-dirty", GlodaMsgIndexer.kMessageDirty);
+  explode_enumeration_after = 2;
+  yield spin_folder_indexer(folder, 2);
 }
 
-/**
- * Make sure we try and index the right messages.  This is basically the
- */
-
 let tests = [
-  test_ignore_folders_we_should_not_index,
   test_propagate_filthy_from_folder_to_messages,
-  test_count_and_index_messages,
+  test_count_pass,
 ];
 
 function run_test() {
   configure_message_injection({mode: "local"});
   // we do not want the event-driven indexer crimping our style
   configure_gloda_indexing({event: false});
   glodaHelperRunTests(tests);
 }
--- a/mailnews/test/resources/asyncTestUtils.js
+++ b/mailnews/test/resources/asyncTestUtils.js
@@ -128,32 +128,41 @@ function async_run(aArgs) {
  *
  * Note: This function actually schedules the real driver to run after a
  *  timeout. This is to ensure that if you call us from a notification event
  *  that all the other things getting notified get a chance to do their work
  *  before we actually continue execution.  It also keeps our stack traces
  *  cleaner.
  */
 function async_driver() {
-  do_timeout_function(0, _async_driver);
+  do_execute_soon(_async_driver);
   return false;
 }
 
+/**
+ * Sentinel object that test helpers can throw to cause a generator stack to
+ *  abort without triggering a test failure.  You would use this when you wrap
+ *  a utility function to inject faults into the calling function so that you
+ *  can control how far the logic gets.  This allows testing of multi-step logic
+ *  after each step.
+ */
+var asyncExpectedEarlyAbort = "REAP!";
+
 // the real driver!
 function _async_driver() {
   let curGenerator;
   while (asyncGeneratorStack.length) {
     curGenerator = asyncGeneratorStack[asyncGeneratorStack.length-1][0];
     try {
       while (curGenerator.next()) {
       }
       return false;
     }
     catch (ex) {
-      if (ex != StopIteration) {
+      if (ex != StopIteration && ex != asyncExpectedEarlyAbort) {
         let asyncStack = [];
         dump("*******************************************\n");
         dump("Generator explosion!\n");
         dump("Unhappiness at: " + ex.fileName + ":" + ex.lineNumber + "\n");
         dump("Because: " + ex + "\n");
         dump("Stack:\n  " + ex.stack.replace("\n", "\n  ", "g") + "\n");
         dump("**** Async Generator Stack source functions:\n");
         for (let i = asyncGeneratorStack.length - 1; i >= 0; i--) {