simpler product-loading tool
authorMike Shaver <shaver@mozilla.org>
Tue, 01 Feb 2011 15:56:37 +0000
changeset 30 1d783f4418e5
parent 29 9c999d1d3d81
child 31 ad67b2adaa87
push id23
push usershaver@mozilla.com
push dateTue, 01 Feb 2011 15:57:55 +0000
simpler product-loading tool
load/load-product.js
new file mode 100644
--- /dev/null
+++ b/load/load-product.js
@@ -0,0 +1,119 @@
+var http = require('http');
+var https = require('https');
+var util = require('util');
+var redis = require('redis');
+var indexer = require('../lib/indexer');
+
+var redisClient = redis.createClient(6380);
+
+var product = process.argv[2];
+
+redisClient.on("error", function (err) {
+          console.log("Redis connection error to " + redisClient.host + ":" + redisClient.port + " - " + err);
+          redisClient.quit();
+});
+
+function componentUrlForProduct(product)
+{
+    return '/latest/count?product=' + product + 
+        '&status=UNCONFIRMED&status=NEW&status=ASSIGNED&status=REOPENED' + '&x_axis_field=component';
+}
+
+function bzapiGet(url, cb)
+{
+    var req = https.get({path: url, host: 'api-dev.bugzilla.mozilla.org'}, cb);
+    req.on('error', function (err) { util.log("ERROR: " + url + ": " + err); });
+}
+
+function onComponentListResponse(response)
+{
+    var data = '';
+    response.on('data', function(chunk) { data += chunk; });
+    response.on('end', function() { loadComponents(JSON.parse(data).x_labels);});
+}
+var componentListRequest = bzapiGet(componentUrlForProduct(product), onComponentListResponse);
+
+/*
+ * Load bugs for components and index them.
+ */
+
+/* Loading */
+function bugDataUrlForComponent(component)
+{
+    var STATUS_OPEN = ["NEW", "ASSIGNED", "UNCONFIRMED", "REOPENED"];
+    var qsData = { status: STATUS_OPEN, include_fields: 'id,summary,product,component,whiteboard',
+                   component: component, product: product};
+    return '/latest/bug?' + require("querystring").stringify(qsData, '&', '=', false); 
+}
+
+var loaded = 0, indexed = 0; // component processing counters
+var totalBugs = 0;
+var components = [];
+
+function loadComponents(cs) {
+    /* global */ components = cs;
+    util.log(product + ": found " + cs.length + " components");
+    loadOneComponent(); // start the loading chain
+}
+    
+function loadOneComponent()
+{
+    var component = components[loaded];
+    printUpdate("loading: " + component);
+    
+    var loadStart = Date.now();
+
+    function onBugDataResponse(response)
+    {
+        var data = '';
+        response.on('data', function(chunk) { data += chunk; });
+        response.on('end', function() { 
+            loaded++;
+            printUpdate("load of " + component + ": " + (Date.now() - loadStart) + "ms, " + data.length + " bytes");
+            if (loaded != components.length) {
+                loadOneComponent(); // start next load
+            } else {
+                printUpdate("all loads complete");
+            }
+            process.nextTick(function() { indexJSON(data, component); }); // index in parallel
+        });
+    }
+    var bugDataReq = bzapiGet(bugDataUrlForComponent(component), onBugDataResponse);
+}
+
+/* indexing */
+
+function indexJSON(data, component)
+{
+    var terms = 0;
+
+    function indexBug(bug)
+    {
+        var keywords = indexer.keywordsForSummary(bug.summary);
+        keywords.forEach(function (w) { redisClient.sadd("word:" + w, bug.id); });
+        terms += keywords.length;
+        redisClient.set("bug:" + bug.id, bug.summary, function() { });
+    }
+
+    var indexStart = Date.now();
+    var bugList = JSON.parse(data).bugs;
+    printUpdate("indexing: " + component + "; " + bugList.length + " bugs");
+    bugList.forEach(indexBug);
+    redisClient.ping(function indexingCompleted() // wait for pipeline to flush
+                {
+                    totalBugs += bugList.length;
+                    ++indexed; // global count
+                    printUpdate("indexed " + component + ": " + (Date.now() - indexStart) + "ms, " +
+                                bugList.length + " bugs, " + terms + " terms");
+                    if (indexed == components.length) {
+                        util.log("All done!");
+                        redisClient.quit();
+                    }
+                });
+}
+
+function printUpdate(detail)
+{
+    util.log(product + ": " + loaded + ":" + indexed + " of " + components.length + " (" +
+             detail + ")");
+}