robustify, index component/product, rearch of static files
authorMike Shaver <shaver@mozilla.org>
Sun, 06 Feb 2011 05:20:21 +0000
changeset 34 7d87f021cc6423e904c71b9ee3de804398f7fd56
parent 33 a7b6a81983f785a5e3ac5f4bf14e46b4e55c1c5a
child 35 a32498bf2edbc56c31dcd5a8a3aad1c6472ab9b2
push id25
push usershaver@mozilla.com
push dateSun, 06 Feb 2011 05:21:47 +0000
robustify, index component/product, rearch of static files
load/load-product.js
server/static/index.html
server/static/main.css
server/static/search.js
server/websocket.js
--- a/load/load-product.js
+++ b/load/load-product.js
@@ -1,17 +1,24 @@
 var http = require('http');
 var https = require('https');
 var util = require('util');
 var redis = require('redis');
 var indexer = require('../lib/indexer');
 
+var loaded = 0, indexed = 0; // component processing counters
+var totalBugs = 0;
+var components = [];
+
 var redisClient = redis.createClient(6380);
 
+var globalStart = Date.now();
+
 var product = process.argv[2];
+redisClient.sadd("products", product.toLowerCase().replace(/ /g, '+'));
 
 redisClient.on("error", function (err) {
           console.log("Redis connection error to " + redisClient.host + ":" + redisClient.port + " - " + err);
           redisClient.quit();
 });
 
 function componentUrlForProduct(product)
 {
@@ -36,38 +43,35 @@ var componentListRequest = bzapiGet(comp
 /*
  * Load bugs for components and index them.
  */
 
 /* Loading */
 function bugDataUrlForComponent(component)
 {
     var STATUS_OPEN = ["NEW", "ASSIGNED", "UNCONFIRMED", "REOPENED"];
-    var qsData = { status: STATUS_OPEN, include_fields: 'id,summary,product,component,whiteboard',
+    var qsData = { status: STATUS_OPEN, include_fields: 'id,summary',
                    component: component, product: product};
     return '/latest/bug?' + require("querystring").stringify(qsData, '&', '=', false); 
 }
 
-var loaded = 0, indexed = 0; // component processing counters
-var totalBugs = 0;
-var components = [];
-
 function loadComponents(cs) {
     /* global */ components = cs;
-    util.log(product + ": found " + cs.length + " components");
+    printUpdate("found " + cs.length + " components");
     loadOneComponent(); // start the loading chain
 }
     
 function loadOneComponent()
 {
     var component = components[loaded];
     printUpdate("loading: " + component);
     
     var loadStart = Date.now();
 
+    redisClient.sadd("components", component.toLowerCase().replace(/ /g, '+'));
     function onBugDataResponse(response)
     {
         var data = '';
         response.on('data', function(chunk) { data += chunk; });
         response.on('end', function() { 
             loaded++;
             printUpdate("load of " + component + ": " + (Date.now() - loadStart) + "ms, " + data.length + " bytes");
             if (loaded != components.length) {
@@ -88,30 +92,32 @@ function indexJSON(data, component)
     var terms = 0;
 
     function indexBug(bug)
     {
         var keywords = indexer.keywordsForSummary(bug.summary);
         keywords.forEach(function (w) { redisClient.sadd("word:" + w, bug.id); });
         terms += keywords.length;
         redisClient.set("bug:" + bug.id, bug.summary, function() { });
+        redisClient.sadd("component:" + component.toLowerCase().replace(/ /g, '+'), bug.id);
+        redisClient.sadd("product:" + product.toLowerCase().replace(/ /g, '+'), bug.id);
     }
 
     var indexStart = Date.now();
     var bugList = JSON.parse(data).bugs;
     printUpdate("indexing: " + component + "; " + bugList.length + " bugs");
     bugList.forEach(indexBug);
     redisClient.ping(function indexingCompleted() // wait for pipeline to flush
                 {
                     totalBugs += bugList.length;
                     ++indexed; // global count
                     printUpdate("indexed " + component + ": " + (Date.now() - indexStart) + "ms, " +
                                 bugList.length + " bugs, " + terms + " terms");
                     if (indexed == components.length) {
-                        util.log("All done!");
+                        printUpdate("All done! " + (Date.now() - globalStart) + "ms");
                         redisClient.quit();
                     }
                 });
 }
 
 function printUpdate(detail)
 {
     util.log(product + ": " + loaded + ":" + indexed + " of " + components.length + " (" +
--- a/server/static/index.html
+++ b/server/static/index.html
@@ -1,73 +1,15 @@
 <!doctype html>
 <html>
   <head>
     <title>Bugzilla fastness</title>
-<style>
-#matchingbugs {
-  font-family: helvetica, arial, sans-serif;
-/*  font-size: .8em; */
-  padding-bottom: 40px;
-}
-</style>
-<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js" type="text/javascript" charset="utf-8"></script>
+    <link href="static/main.css" type="text/css" rel="stylesheet">
+    <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js" type="text/javascript" charset="utf-8"></script>
+    <script src="static/search.js" type="text/javascript"></script>
   </head>
   <body>
-<div id="controls">
-<hr />
-<input type="text" id="keywords"/> <button id="submitton">Search</button> &larr; type 4-letter-or-longer words here to search bugzilla (space or enter ends words, click button for mobile)
-</div>
-<div id="matchingbugs"> </div>
-    <script type="text/javascript">
-var results;
-$(function(){
-  $("#keywords").focus();
-
-  $(window).bind("focus", function(){
-    $("#keywords").focus();
-  });
-
-  function sendmsg(){
-    var terms = $("#keywords").val();
-    location.hash = "#" + encodeURIComponent(terms);
-    var xhr = new XMLHttpRequest();
-    xhr.open("GET", "/json?" + encodeURI(terms), true);
-    xhr.onreadystatechange = function() {
-        if (xhr.readyState != 4 || xhr.status != 200)
-            return;
-        var results = JSON.parse(xhr.responseText);
-        var div = $("#matchingbugs");
-        div.html(results.bugs.length + " results for '" + $("#keywords").val() + "' in " +  (Date.now() - xhr.sendTime) + " (total)/" +
-                 results.times.server + " (server)/" + results.times.redis + " (redis) ms<br>");
-
-        results.bugs.forEach(function (v) { div.append("<a href='https://bugzilla.mozilla.org/show_bug.cgi?id=" + v.id + "'>" + v.id + "</a>: ");
-        div.append(document.createTextNode(v.summary)); div.append("<br>"); });
-        if (results.hasMore)
-            div.append("<br>(Showing " + results.bugs.length + " of " + results.totalResults + " results)<br>");
-        };
-    xhr.sendTime = Date.now();
-    xhr.send(null);
-  };
-
-  $("#submitton").bind("click", sendmsg);
-
-  $("#keywords").bind("keydown", function(e){
-
- if(e.keyCode == 32 || e.keyCode == 13 ||
-   (e.keyCode == 8 && $("#keywords").val().substr(-2, 1) == " "))
-{
-      sendmsg();
-  }});
-
-  if (location.hash) {
-      $("#keywords").val(decodeURIComponent(location.hash.substring(1)));
-      sendmsg();
-  }
-
-  function log(data){
-    $("#matchingbugs").html(data);
-  };
-
-});
-    </script>
+    <div id="controls">
+        <input type="text" id="keywords"/> <button id="submitton">Search</button> &larr; type 4-letter-or-longer words here to search bugzilla (space or enter ends words, click button for mobile)
+    </div>
+    <div id="matchingbugs"> </div>
   </body>
 </html>
new file mode 100644
--- /dev/null
+++ b/server/static/main.css
@@ -0,0 +1,4 @@
+#matchingbugs {
+  font-family: helvetica, arial, sans-serif;
+  padding-bottom: 40px;
+}
new file mode 100644
--- /dev/null
+++ b/server/static/search.js
@@ -0,0 +1,50 @@
+var results;
+$(function(){
+  $("#keywords").focus();
+
+  $(window).bind("focus", function(){
+    $("#keywords").focus();
+  });
+
+  function sendmsg(){
+    var terms = $("#keywords").val();
+    location.hash = "#" + encodeURIComponent(terms);
+    var xhr = new XMLHttpRequest();
+    xhr.open("GET", "/json?" + encodeURI(terms), true);
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState != 4 || xhr.status != 200)
+            return;
+        var results = JSON.parse(xhr.responseText);
+        var div = $("#matchingbugs");
+        div.html(results.bugs.length + " results for '" + $("#keywords").val() + "' in " +  (Date.now() - xhr.sendTime) + " (total)/" +
+                 results.times.server + " (server)/" + results.times.redis + " (redis) ms<br>");
+
+        results.bugs.forEach(function (v) { div.append("<a href='https://bugzilla.mozilla.org/show_bug.cgi?id=" + v.id + "'>" + v.id + "</a>: ");
+        div.append(document.createTextNode(v.summary)); div.append("<br>"); });
+        if (results.hasMore)
+            div.append("<br>(Showing " + results.bugs.length + " of " + results.totalResults + " results)<br>");
+        };
+    xhr.sendTime = Date.now();
+    xhr.send(null);
+  };
+
+  $("#submitton").bind("click", sendmsg);
+
+  $("#keywords").bind("keydown", function(e){
+
+ if(e.keyCode == 32 || e.keyCode == 13 ||
+   (e.keyCode == 8 && $("#keywords").val().substr(-2, 1) == " "))
+{
+      sendmsg();
+  }});
+
+  if (location.hash) {
+      $("#keywords").val(decodeURIComponent(location.hash.substring(1)));
+      sendmsg();
+  }
+
+  function log(data){
+    $("#matchingbugs").html(data);
+  };
+
+});
--- a/server/websocket.js
+++ b/server/websocket.js
@@ -107,19 +107,20 @@ function handleRequest(req, res)
                 res.write("404 Not found\n");
                 res.end();
                 return;
             }
             
             res.writeHead(200, {'Content-Type': typeForPathname(pathname)});
             fs.createReadStream(pathname.substr(1), { flags: 'r', bufferSize: 4*1024 }).
                 addListener('data', function(chunk) { res.write(chunk); }).
-                addListener('end', res.end.bind(res));
+                addListener('end', res.end.bind(res)).
+                addListener('error', util.log);
+            return;
         });
-        return;
     }
     
     if (req.method == "GET" && pathname == "/json") {
         terms = decodeURIComponent(urlx.query);
         res.writeHead(200, {'Content-Type': 'text/plain'});
         runSearch(terms, function (data) { res.write(data); res.end(); });
         return;
     }
@@ -130,19 +131,19 @@ function handleRequest(req, res)
 var httpServer = http.createServer(handleRequest);
 
 var clientCount = 0;
 
 var socket = io.listen(httpServer);
 socket.on("clientConnect",
           function(client) {
               util.log("connect (" + ++clientCount + ")");
-              client.on("message", function(message) { runSearch(message, 
-                                          function(data) { client.send(data) }); });
+              client.on("message", 
+                  function(message) { runSearch(message, function(data) { client.send(data); }); });
               client.on("error", function(err) { util.log("ERROR " + err); });
-              client.on("disconnect", function() { util.log ("disconnect (" + --clientCount + ")")});
+              client.on("disconnect", function() { util.log ("disconnect (" + --clientCount + ")");});
           }); 
 
 httpServer.on("close", function(conn) { util.log("connection closed"); });
 httpServer.on("error", function (conn, err) { util.log("server error: " + err); });
 
 httpServer.listen(parseInt(process.argv[2]) || 8000);