Bug 1437828 - xpcshell script used to convert files, coming from bug 1353542 draft
authorAlexandre Poirot <poirot.alex@gmail.com>
Tue, 13 Feb 2018 03:56:23 -0800
changeset 754800 bece3cedb89fa11e36a22ce0eb28089cf7640ce3
parent 754799 371dcd98fc452fc26fbe9eb60ad3e822216bc58e
push id99001
push userbmo:poirot.alex@gmail.com
push dateWed, 14 Feb 2018 10:14:19 +0000
bugs1437828, 1353542
milestone60.0a1
Bug 1437828 - xpcshell script used to convert files, coming from bug 1353542 MozReview-Commit-ID: KWgrEEcwHX1
xpc
new file mode 100644
--- /dev/null
+++ b/xpc
@@ -0,0 +1,679 @@
+var { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+
+Components.utils.import("resource://gre/modules/osfile.jsm");
+Components.utils.import("resource://gre/modules/Task.jsm");
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+const init = Components.classes["@mozilla.org/jsreflect;1"].createInstance();
+init();
+
+var done = false;
+
+const kIgnorePaths = [
+  "obj",
+  "mozilla", // for comm-central
+
+  // Exclude files where it looks like m-c isn't the upstream
+  "devtools/client/inspector/markup/test",
+  "devtools/client/debugger/new/test/mochitest/examples/sourcemaps2",
+  "devtools/client/jsonview/lib",
+  "devtools/client/shared/vendor",
+  "devtools/client/sourceeditor/codemirror",
+  "dom/tests/mochitest/ajax",
+  "dom/tests/mochitest/dom-level",
+  "dom/canvas/test/webgl-conf/checkout",
+  "js/src",
+  "mobile/android/tests/browser/chrome/tp5",
+  "python",
+  "testing/mochitest/MochiKit",
+  "testing/mochitest/tests/MochiKit",
+  "testing/talos/talos/tests",
+  "testing/web-platform/tests",
+  "testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py/docs/_build/html/_static/jquery.js",
+  "testing/marionette/atom.js",
+  "browser/extensions/pocket/content/panels/js/vendor/jquery-2.1.1.min.js",
+  "addon-sdk/source/examples/actor-repl/data/codemirror-compressed.js",
+  "servo/tests/html/webvr/js/third-party",
+  
+  "toolkit/modules/Task.jsm",
+  // This file tests the Task.jsm module.
+  "toolkit/modules/tests/xpcshell/test_task.js",
+  "toolkit/modules/tests/xpcshell/test_Promise.js",
+
+  // This whole test folder uses .next on the runTests generator of each test file.
+  "toolkit/components/thumbnails/test",
+
+  ".hg",
+  "tools/lint/eslint"
+];
+
+let generatorWhitelist = [];
+let generatorWhitelistPrefixes = [];
+
+let replaceGenerators = arguments.includes("--replace-generators");
+let showGeneratorsWithoutYield = arguments.includes("--show-without-yield");
+let showGenerators = arguments.includes("--show-generators");
+let gPaths = arguments.filter(a => !a.startsWith("-"));
+
+if (!gPaths.length) {
+  gPaths = ["toolkit", "browser"];
+  generatorWhitelist = [
+    "browser/base/content/test/static/head.js:111",
+//    "toolkit/modules/tests/xpcshell/test_Promise.js:695",
+//    "toolkit/modules/tests/xpcshell/test_Promise.js:782",
+    "toolkit/modules/InlineSpellChecker.jsm:511",
+    "toolkit/modules/FinderIterator.jsm:535",
+    "toolkit/mozapps/extensions/internal/XPIProviderUtils.js:232",
+//    "toolkit/modules/Task.jsm:120",
+    "toolkit/modules/tests/xpcshell/test_FinderIterator.js:6",
+    "toolkit/mozapps/extensions/test/mochitest/test_bug687194.html:47",
+    "toolkit/components/extensions/test/mochitest/chrome_cleanup_script.js:17",
+    "toolkit/components/extensions/Schemas.jsm:2425",
+    "toolkit/components/extensions/MessageChannel.jsm:177",
+    "toolkit/components/extensions/ExtensionUtils.jsm:987",
+    "toolkit/components/extensions/ExtensionContent.jsm:730",
+    "toolkit/components/extensions/extension-process-script.js:515",
+    "toolkit/components/extensions/ext-cookies.js:131",
+    "toolkit/components/places/tests/unit/test_isURIVisited.js:33",
+    "toolkit/components/places/PlacesUtils.jsm:499",
+    "toolkit/components/places/nsTaggingService.js:635",
+    "toolkit/content/tests/chrome/RegisterUnregisterChrome.js:29",
+    "toolkit/modules/addons/WebNavigationFrames.jsm:41",
+    "browser/base/content/test/social/head.js:135",
+    "browser/base/content/browser.js:271",
+    "browser/base/content/browser-gestureSupport.js:258",
+    "browser/components/customizableui/CustomizableUI.jsm:2893",
+    "browser/components/migration/MSMigrationUtils.jsm:506",
+    "browser/components/migration/ESEDBReader.jsm:360",
+    "browser/components/newtab/tests/browser/browser_PreviewProvider.js:52",
+    "browser/components/sessionstore/test/head.js:312",
+    "browser/components/distribution.js:479",
+    "browser/modules/SocialService.jsm:40",
+  ];
+
+  generatorWhitelistPrefixes = [
+    "toolkit/components/contentprefs/tests/unit_cps2",
+    "toolkit/components/contentprefs/ContentPrefStore.jsm",
+    "toolkit/components/extensions/ExtensionTabs.jsm",
+    "toolkit/components/thumbnails/test/browser_thumbnails",
+    "toolkit/content/tests/chrome/test_mousescroll.xul",
+    "toolkit/content/tests/chrome/test_autocomplete_placehold_last_complete.xul",
+    "toolkit/content/tests/chrome/test_arrowpanel.xul",
+    "toolkit/modules/addons/WebRequestUpload.jsm",
+    "toolkit/modules/subprocess/subprocess_",
+    "browser/components/downloads/DownloadsCommon.jsm",
+    "browser/components/extensions/ext-utils.js",
+    "browser/modules/test/unit/social/test_SocialService",
+    "toolkit/components/satchel/test/unit/test_",
+  ];
+}
+
+generatorWhitelist = new Set(generatorWhitelist);
+
+
+function isFileRelevant(file) {
+  return file.includes("Task.") || file.includes("yield");
+}
+
+function deepEqualExceptLocation(obj1, obj2) {
+  if ((typeof obj1) != (typeof obj2))
+    return false;
+
+  if (typeof obj1 == "string")
+    return obj1 == obj2;
+
+  if (obj1 === obj2)
+    return true;
+
+  for (let prop in obj1) {
+    if (prop == "loc")
+      continue;
+    if (!deepEqualExceptLocation(obj1[prop], obj2[prop]))
+      return false;
+  }
+  for (let prop in obj2) {
+    if (prop == "loc")
+      continue;
+    if (!deepEqualExceptLocation(obj1[prop], obj2[prop]))
+      return false;
+  }
+
+  return true;
+}
+
+let globalCounts = {functions: 0, generators: 0, converted: 0, alreadyConverted: 0};
+let noYield = 0;
+
+function processScript(file, relativePath = "", startLine = 1) {
+  file = file.replace(/^#/gm, "//XPCShell-preprocessor-");
+  let lines = file.split("\n");
+  let removals = [];
+
+  let getLineForLoc = loc => lines[loc.line - startLine];
+
+  let replacedLocationSet = new Set();
+
+  function convertToNonGeneratorFunction(fun) {
+    // 'function*' may not be included in fun.loc
+    let fromLoc = fun.loc.start;
+    let start = getLineForLoc(fromLoc).slice(0, fromLoc.column);
+    let match = /function\s*\*? ?$/.exec(start);
+    if (match)
+      fromLoc = {line: fromLoc.line, column: fromLoc.column - match[0].length};
+    let toLoc = fun.loc.start;
+
+    if (fun.id) {
+      let prefix = "function ";
+      // Handle the '*methodName() {' -> 'methodName() {' case.
+      if (!match && (match = /\* ?$/.exec(start))) {
+        fromLoc = {line: fromLoc.line, column: fromLoc.column - match[0].length};
+        prefix = "";
+      }
+
+      removals.push({from_line: fromLoc.line - startLine,
+                     from_column: fromLoc.column,
+                     to_line: toLoc.line - startLine,
+                     to_column: toLoc.column,
+                     insert: prefix
+                    });
+    } else {
+      let end = getLineForLoc(toLoc).slice(toLoc.column);
+      toLoc = {line: toLoc.line, column: toLoc.column + end.indexOf("(")};
+      removals.push({from_line: fromLoc.line - startLine,
+                     from_column: fromLoc.column,
+                     to_line: toLoc.line - startLine,
+                     to_column: toLoc.column,
+                     insert: "function"
+                    });
+    }
+  }
+  
+  function replaceGenerator(fun, loc) {
+    if (!loc)
+      loc = fun.loc;
+    replacedLocationSet.add(fun.loc.start.line + ":" + fun.loc.start.column);
+
+    // Go through the AST recursively and replace each 'yield' with 'await'
+    let walk = obj => {
+      // Avoid replacing yield keywords in subfunctions; they may be actual generators!
+      if (obj.type == "FunctionExpression" || obj.type == "FunctionDeclaration")
+        return;
+
+      if (obj.type == "YieldExpression") {
+        let fromLoc = obj.loc.start;
+        // obj.argument.loc.start won't be correct if obj.argument starts with '('.
+        let match = /yield\*?/.exec(getLineForLoc(fromLoc).slice(fromLoc.column));
+        let toLoc = {line: fromLoc.line, column: fromLoc.column + match[0].length};
+        removals.push({from_line: fromLoc.line - startLine,
+                       from_column: fromLoc.column,
+                       to_line: toLoc.line - startLine,
+                       to_column: toLoc.column,
+                       insert: "await"
+                      });
+      }
+
+      for (let prop in obj) {
+        if (obj[prop] && (typeof obj[prop]) == "object")
+          walk(obj[prop]);
+      }
+    };
+    if (fun.body)
+      walk(fun.body);
+
+    // loc may either be the call expression's location, or the function expression location.
+    // In that latter case, 'function*' may not be included.
+    let fromLoc = loc.start;
+    let start = getLineForLoc(fromLoc).slice(0, fromLoc.column);
+    let match = /function\s*\*? ?$/.exec(start);
+    if (match)
+      fromLoc = {line: fromLoc.line, column: fromLoc.column - match[0].length};
+    let toLoc = fun.loc.start;
+
+    if (fun.id) {
+      let prefix = "async function ";
+      // Handle the '*methodName() {' -> 'async methodName() {' case.
+      if (!match && (match = /\* ?$/.exec(start))) {
+        fromLoc = {line: fromLoc.line, column: fromLoc.column - match[0].length};
+        prefix = "async ";
+      }
+
+      removals.push({from_line: fromLoc.line - startLine,
+                     from_column: fromLoc.column,
+                     to_line: toLoc.line - startLine,
+                     to_column: toLoc.column,
+                     insert: prefix
+                    });
+    } else {
+      // If loc is for the whole call expression, we may be in the
+      // 'methodName: Task.async(function*()' case, let's grab the name from it.
+      let match = /(\w+): ?$/.exec(start);
+      let name = "function";
+      if (match) {
+        name = match[1];
+        fromLoc = {line: fromLoc.line, column: fromLoc.column - match[0].length};
+      }
+      let end = getLineForLoc(toLoc).slice(toLoc.column);
+      toLoc = {line: toLoc.line, column: toLoc.column + end.indexOf("(")};
+      removals.push({from_line: fromLoc.line - startLine,
+                     from_column: fromLoc.column,
+                     to_line: toLoc.line - startLine,
+                     to_column: toLoc.column,
+                     insert: "async " + name
+                    });
+    }
+  }
+
+  
+  try {
+    Reflect.parse(file, {
+      source: relativePath,
+      line: startLine,
+      builder: {
+        program: function(body, loc) {
+          let counts = {functions: 0, generators: 0, converted: 0, alreadyConverted: 0};
+
+          let yields = [];
+          let walkFun = obj => {
+            // Avoid subfunctions
+            if (obj.type == "FunctionExpression" || obj.type == "FunctionDeclaration")
+              return false;
+
+            if (obj.type == "YieldExpression") {
+              yields.push(getLineForLoc(obj.loc.start));
+
+              if (!obj.argument)
+                return false;
+
+              // yield promiseBlah; or yield blahPromise;
+              if (obj.argument.type == "Identifier" &&
+                  (obj.argument.name.startsWith("promise") ||
+                   obj.argument.name.endsWith("Promise")))
+                return true;
+              // yield new Promise(...)
+              if (obj.argument.type == "NewExpression" &&
+                  obj.argument.callee.type == "Identifier" &&
+                  obj.argument.callee.name == "Promise")
+                return true;
+              if (obj.argument.type == "CallExpression") {
+                let callee = obj.argument.callee;
+                if (callee.type == "MemberExpression" &&
+                    callee.object.type == "Identifier" &&
+                    ["BrowserTestUtils", "ContentTask", "Promise", "Task",
+                     "PlacesUtils", "PlacesTestUtils", "NormandyApi"].includes(callee.object.name)) {
+                  return true;
+                }
+                // yield OS.File...
+                if (callee.type == "MemberExpression" &&
+                    callee.object.type == "MemberExpression" &&
+                    callee.object.object.type == "Identifier" &&
+                    callee.object.object.name == "OS" &&
+                    callee.object.property.type == "Identifier" &&
+                    callee.object.property.name == "File") {
+                  return true;
+                }
+                // yield promiseBlah(...)
+                if (callee.type == "Identifier" && (callee.name.startsWith("promise") ||
+                                                    callee.name.startsWith("waitFor")))
+                  return true;
+              }
+              // yield blah.promise; eg. deferred.promise
+              if (obj.argument.type == "MemberExpression" &&
+                  obj.argument.property.type == "Identifier" &&
+                  obj.argument.property.name == "promise")
+                return true;
+            }
+
+            for (let prop in obj) {
+              if (obj[prop] && (typeof obj[prop]) == "object" && walkFun(obj[prop]))
+                return true;
+            }
+            return false;
+          };
+          
+          // Go through the AST recursively and replace each 'yield' with 'await'
+          let walk = obj => {
+            // Avoid replacing yield keywords in subfunctions; they may be actual generators!
+            if (obj.type == "FunctionExpression" || obj.type == "FunctionDeclaration") {
+              counts.functions++;
+              if (obj.generator) {
+                counts.generators++;
+                if (replacedLocationSet.has(obj.loc.start.line + ":" + obj.loc.start.column)) {
+                  counts.alreadyConverted++;
+                } else {
+                  yields = [];
+                  if (walkFun(obj.body))
+                    replaceGenerator(obj);
+                  else {
+                    if (!yields.length) {
+                      convertToNonGeneratorFunction(obj);
+                      if (showGeneratorsWithoutYield)
+                        dump("generator without yield at: " + relativePath + " " + obj.loc.start.line + "\n");
+                      noYield++;
+                    } else {
+                      let location = relativePath + ":" + obj.loc.start.line;
+                      if (replaceGenerators) {
+                        if (!generatorWhitelistPrefixes.some(p => relativePath.startsWith(p))) {
+                          if (!generatorWhitelist.has(location)) {
+                            for (let entry of generatorWhitelist)
+                              if (entry.startsWith(relativePath))
+                                print("warning: found " + entry + " but replacing a generator at line " + obj.loc.start.line);
+                            replaceGenerator(obj)
+                          } else {
+                            generatorWhitelist.delete(location);
+                          }
+                        }
+                      } else if (showGenerators) {
+                        dump(location + "\n" + yields.join("\n") + "\n\n");
+                      }
+                    }
+                  }
+                }
+              }
+            }
+
+            for (let prop in obj) {
+              if (obj[prop] && (typeof obj[prop]) == "object")
+                walk(obj[prop]);
+            }
+          };
+          walk(body);
+          if (counts.generators) {
+            counts.converted = replacedLocationSet.size;
+          }
+          for (let i in counts)
+            globalCounts[i] += counts[i];
+        },
+
+        callExpression: function(callee, args, loc) {
+          let rv = {type: "CallExpression", callee: callee, arguments: args, loc: loc};
+          if (!callee)
+            return rv;
+
+          // Remove XPCOMUtils.defineLazyModuleGetter and Cu.import calls for Task.jsm
+          if (callee.type == "MemberExpression" && callee.property.type == "Identifier" &&
+              ((callee.property.name == "import" && args.length >= 1 &&
+               args[0].type == "Literal" && args[0].value.endsWith("modules/Task.jsm")) ||
+               (callee.property.name == "defineLazyModuleGetter" && args.length >= 3 &&
+                args[2].type == "Literal" && args[2].value.endsWith("modules/Task.jsm")))) {
+
+            // Preserve the import for files that use Task.Debugging
+            if (![
+              "toolkit/components/asyncshutdown/AsyncShutdown.jsm",
+              "toolkit/components/osfile/modules/osfile_async_front.jsm",
+              "toolkit/modules/Log.jsm",
+              "testing/mochitest/browser-test.js",
+              "testing/xpcshell/head.js",
+            ].includes(relativePath)) {
+              let fromLoc = loc.start;
+              let toLoc = loc.end;
+              removals.push({from_line: fromLoc.line - startLine,
+                             from_column: 0,
+                             to_line: toLoc.line - startLine + 1,
+                             to_column: 0,
+                            });
+            }
+          }
+
+          // ContentTask.spawn's third argument is a task.
+          if (callee.type == "MemberExpression" &&
+              callee.property.type == "Identifier" && callee.property.name == "spawn" &&
+              callee.object.type == "Identifier" && callee.object.name == "ContentTask" &&
+              args.length == 3 && args[2].type == "FunctionExpression" && args[2].generator)
+            replaceGenerator(args[2]);
+
+          // add_task
+          if (callee.type == "Identifier" && callee.name == "add_task" &&
+              args[0].type == "FunctionExpression" && args[0].generator) {
+            replaceGenerator(args[0]);
+          }
+
+          // Task.spawn
+          if (callee.type == "MemberExpression" &&
+              callee.property.type == "Identifier" && callee.property.name == "spawn" &&
+              callee.object.type == "Identifier" && callee.object.name == "Task" &&
+              args.length == 1) {
+
+            
+            let fromLoc = callee.loc.start;
+            let toLoc = callee.loc.end;
+            removals.push({from_line: fromLoc.line - startLine,
+                           from_column: fromLoc.column,
+                           to_line: toLoc.line - startLine,
+                           to_column: toLoc.column,
+                          });
+
+            if (args[0].type == "FunctionExpression" && args[0].generator)
+              replaceGenerator(args[0]);
+            else if (args[0].type == "CallExpression" &&
+                     args[0].callee.type == "MemberExpression" &&
+                     args[0].callee.property.type == "Identifier" &&
+                     args[0].callee.property.name == "bind" &&
+                     args[0].callee.object.type == "FunctionExpression" &&
+                     args[0].callee.object.generator)
+              replaceGenerator(args[0].callee.object);
+            else
+              dump(relativePath + " " + callee.loc.start.line + " needs review\n");
+
+            toLoc = fromLoc = args[0].loc.end;
+            removals.push({from_line: fromLoc.line - startLine,
+                           from_column: fromLoc.column,
+                           to_line: toLoc.line - startLine,
+                           to_column: toLoc.column,
+                           insert: ")("
+                        });
+          }
+
+          // return if we are not in a Task.async call.
+          if (callee.type != "MemberExpression" ||
+              callee.property.type != "Identifier" ||
+              callee.property.name != "async" ||
+              callee.object.type != "Identifier" || callee.object.name != "Task" ||
+              args.length != 1)
+            return rv;
+
+          let fun;
+          if (args[0].type == "FunctionExpression" && args[0].generator) {
+            fun = args[0];
+          } else if (args[0].type == "CallExpression" &&
+                     args[0].callee.type == "MemberExpression" &&
+                     args[0].callee.property.type == "Identifier" &&
+                     args[0].callee.property.name == "bind" &&
+                     args[0].callee.object.type == "FunctionExpression" &&
+                     args[0].callee.object.generator) {
+            fun = args[0].callee.object;
+          } else {
+            return rv;
+          }
+          replaceGenerator(fun, loc);
+
+          let fromLoc = args[0].loc.end;
+          let toLoc = loc.end;
+          removals.push({from_line: fromLoc.line - startLine,
+                         from_column: fromLoc.column,
+                         to_line: toLoc.line - startLine,
+                         to_column: toLoc.column,
+                        });
+
+          return rv;
+        }
+      }
+    });
+
+    if (!removals.length)
+      return null;
+
+    removals.sort((a, b) => {
+      return (a.from_line - b.from_line) || (a.from_column - b.from_column);
+    });
+
+    let removal;
+    while ((removal = removals.pop())) {
+      let line = lines[removal.from_line].slice(0, removal.from_column) +
+                 (removal.insert || "") +
+                 lines[removal.to_line].slice(removal.to_column);
+      lines[removal.from_line] = line;
+      lines.splice(removal.from_line + 1, removal.to_line - removal.from_line);
+    }
+
+    return lines.join("\n").replace(/^\/\/XPCShell-preprocessor-/gm, "#");
+  } catch (ex) {
+    dump("Error reading " + relativePath + ":" + ex.lineNumber + " " + ex + "\n");
+    return null;
+  }
+}
+
+Task.spawn(function *() {
+  let currentDirectory = yield OS.File.getCurrentDirectory();
+  dump("current:"+currentDirectory+"\n");
+  let paths = gPaths.map(a => currentDirectory + "/" + a);
+  
+  let decoder = new TextDecoder();
+
+  while (paths.length) {
+    let iterator;
+    try {
+      iterator = new OS.File.DirectoryIterator(paths.pop());
+
+      let items = [];
+      while(true) {
+        let i = iterator.next();
+        if (!i) break;
+        i = yield i;
+        if (!i || i.done) break;
+        i = i.value;
+        if (!i) break;
+        items.push(i);
+      }
+      for (let child of items) {
+        if (typeof child == "string") {
+          continue;
+        }
+        let path = child.path;
+        let relativePath = path.slice(currentDirectory.length + 1);
+        if (kIgnorePaths.some(p => relativePath.startsWith(p)))
+          continue;
+
+        if (child.isDir) {
+          paths.push(path);
+          continue;
+        }
+
+        if (!path.endsWith(".js") && !path.endsWith(".jsm") &&
+            !path.endsWith(".xml") && !path.endsWith(".xhtml") &&
+            !path.endsWith(".html") && !path.endsWith("xul"))
+          continue;
+        
+        let file = decoder.decode(yield OS.File.read(path));
+        if (!isFileRelevant(file))
+          continue;
+
+        if (path.endsWith(".js") || path.endsWith(".jsm")) {
+          file = processScript(file, relativePath);
+        } else if (path.endsWith(".html")) {
+          // Look for <script> tags.
+          let lines = file.split("\n");
+          let startLine, endLine;
+          let madeChanges = false;
+          for (startLine = 0; startLine < lines.length; ++startLine) {
+            if (/<script.*>/.test(lines[startLine]) &&
+                !lines[startLine].includes("</script>")) {
+              let prefix = lines[startLine].replace(/.*<script[^>]*>/, "");
+              for (endLine = ++startLine; endLine < lines.length; ++endLine) {
+                if (lines[endLine].includes("</script>"))
+                  break;
+              }
+              if (endLine == lines.length)
+                break;
+              let suffix = lines[endLine].replace(/<\/script>.*/, "")
+                                         .replace(/\/\/(.*)/, "/*$1*/");
+              let f = lines.slice(startLine, endLine).join("\n");
+              if (!isFileRelevant(f)) {
+                startLine = endLine;
+                continue;
+              }
+              f = processScript("function f() {" + prefix + "\n"+f+"\n" + suffix + "}",
+                                relativePath, startLine);
+              if (!f) {
+                startLine = endLine;
+                continue;
+              }
+              let modifiedLines = f.split("\n");
+              modifiedLines.shift();
+              modifiedLines.pop();
+              lines.splice(startLine, endLine - startLine, ...modifiedLines);
+              startLine += modifiedLines.length;
+              madeChanges = true;
+            }
+          }
+          file = madeChanges ? lines.join("\n") : null;
+        } else {
+          // Handle XBL bindings and xhtml files by looking for CDATA blocks
+          if (path.endsWith(".xml") &&
+              !file.includes("xmlns=\"http://www.mozilla.org/xbl\""))
+            continue;
+
+          let lines = file.split("\n");
+          let startLine, endLine;
+          let madeChanges = false;
+          for (startLine = 0; startLine < lines.length; ++startLine) {
+            if (lines[startLine].includes("<![CDATA[")) {
+              let prefix = lines[startLine].replace(/.*<!\[CDATA\[/, "");
+              for (endLine = ++startLine; endLine < lines.length; ++endLine) {
+                if (lines[endLine].includes("]]>"))
+                  break;
+              }
+              let suffix = lines[endLine].replace(/\]\]>.*/, "")
+                                         .replace(/\/\/(.*)/, "/*$1*/");
+              let f = lines.slice(startLine, endLine).join("\n");
+              if (!isFileRelevant(f))
+                continue;
+              f = processScript("function f() {" + prefix + "\n"+f+"\n" + suffix + "}",
+                                relativePath, startLine);
+              if (!f)
+                continue;
+              let modifiedLines = f.split("\n");
+              modifiedLines.shift();
+              modifiedLines.pop();
+              lines.splice(startLine, endLine - startLine, ...modifiedLines);
+              startLine += modifiedLines.length;
+              madeChanges = true;
+            }
+          }
+          file = madeChanges ? lines.join("\n") : null;
+        }
+
+        if (file)
+          yield OS.File.writeAtomic(path, (new TextEncoder()).encode(file));
+      }
+    } catch (ex) {
+      // Ignore StopIteration to prevent exiting the loop.
+      //if (ex != StopIteration) {
+        throw ex;
+      //}
+    }
+    iterator.close();
+  }
+  done = true;
+
+  dump("global counts: " + globalCounts.toSource() + "\n");
+  dump((globalCounts.converted / globalCounts.generators * 100) + "% converted\n");
+  dump("generators without yield: " + noYield + "\n");
+  
+  if (replaceGenerators) {
+    for (let whitelistEntry of generatorWhitelist)
+      print("Error: unused whitelist entry: " + whitelistEntry);
+  }
+
+});
+
+// Spin an event loop.
+(() => {
+  var thread = Components.classes["@mozilla.org/thread-manager;1"]
+                         .getService().currentThread;
+  while (!done)
+    thread.processNextEvent(true);
+
+  // get rid of any pending event
+  while (thread.hasPendingEvents())
+    thread.processNextEvent(true);
+})();