Bug 1025522 - Split log files to prevent them from growing too large. r=aleth, florian a=aleth CLOSED TREE
authorNihanth Subramanya <nhnt11@gmail.com>
Fri, 27 Jun 2014 04:46:45 +0530
changeset 30258 afc53c36ca1f5a08aa37b9320b3e5bbacae764e4
parent 30257 578db539d403a3d08ce628b31d2a2963ddf7b766
child 30259 09d2d89fd95befba5154e4d6069afe1b99b4d9ab
push id2480
push useriann_cvs@blueyonder.co.uk
push dateSun, 26 Jul 2015 18:04:43 +0000
treeherdertry-comm-central@fc221c9a961a [default view] [failures only]
reviewersaleth, florian, aleth
bugs1025522
Bug 1025522 - Split log files to prevent them from growing too large. r=aleth, florian a=aleth CLOSED TREE
chat/components/public/imILogger.idl
chat/components/src/logger.js
chat/components/src/test/test_logger.js
mail/components/im/modules/index_im.js
--- a/chat/components/public/imILogger.idl
+++ b/chat/components/public/imILogger.idl
@@ -54,24 +54,24 @@ interface imILog: nsISupports {
 [scriptable, function, uuid(2ab5f8ac-4b89-4954-9a4a-7c167f1e3b0d)]
 interface imIProcessLogsCallback: nsISupports {
   // The callback can return a promise. If it does, then it will not be called
   // on the next log until this promise resolves. If it throws (or rejects),
   // iteration will stop.
   jsval processLog(in AUTF8String aLogPath);
 };
 
-[scriptable, uuid(f8ac75ed-e9b5-432e-989b-f01fed2e5a3f)]
+[scriptable, uuid(7e2476dc-8199-4454-9661-b78ee73fa49e)]
 interface imILogger: nsISupports {
   // Returns a promise that resolves to an imILog instance.
   jsval getLogFromFile(in AUTF8String aFilePath, [optional] in boolean aGroupByDay);
-  // Returns a promise that resolves to the log file path if a log writer
+  // Returns a promise that resolves to the log file paths if a log writer
   // exists for the conversation, or null otherwise. The promise resolves
-  // after any pending I/O operations on the file complete.
-  jsval getLogPathForConversation(in prplIConversation aConversation);
+  // after any pending I/O operations on the files complete.
+  jsval getLogPathsForConversation(in prplIConversation aConversation);
 
   // Below methods return promises that resolve to nsISimpleEnumerator instances.
 
   // Get logs for a username that may not be in the contact list.
   jsval getLogsForAccountAndName(in imIAccount aAccount,
                                  in AUTF8String aNormalizedName,
                                  [optional] in boolean aGroupByDay);
 
--- a/chat/components/src/logger.js
+++ b/chat/components/src/logger.js
@@ -109,27 +109,29 @@ function encodeName(aName) {
 }
 
 function getLogFolderPathForAccount(aAccount) {
   return OS.Path.join(OS.Constants.Path.profileDir,
                       "logs", aAccount.protocol.normalizedName,
                       encodeName(aAccount.normalizedName));
 }
 
-function getLogFilePathForConversation(aConv, aFormat) {
+function getLogFilePathForConversation(aConv, aFormat, aStartTime) {
+  if (!aStartTime)
+    aStartTime = aConv.startDate / 1000;
   let path = getLogFolderPathForAccount(aConv.account);
   let name = aConv.normalizedName;
   if (convIsRealMUC(aConv))
     name += ".chat";
   return OS.Path.join(path, encodeName(name),
-                      getNewLogFileName(aFormat, aConv.startDate));
+                      getNewLogFileName(aFormat, aStartTime));
 }
 
-function getNewLogFileName(aFormat, aDate) {
-  let date = aDate ? new Date(aDate / 1000) : new Date();
+function getNewLogFileName(aFormat, aStartTime) {
+  let date = aStartTime ? new Date(aStartTime) : new Date();
   let dateTime = date.toLocaleFormat("%Y-%m-%d.%H%M%S");
   let offset = date.getTimezoneOffset();
   if (offset < 0) {
     dateTime += "+";
     offset *= -1;
   }
   else
     dateTime += "-";
@@ -144,47 +146,71 @@ function getNewLogFileName(aFormat, aDat
 
 
 // One of these is maintained for every conversation being logged. It initializes
 // a log file and appends to it as required.
 function LogWriter(aConversation) {
   this._conv = aConversation;
   if (Services.prefs.getCharPref("purple.logging.format") == "json")
     this.format = "json";
-  this.path = getLogFilePathForConversation(aConversation, this.format);
-  this._initialized =
-    appendToFile(this.path, this.encoder.encode(this._getHeader()), true);
-  // Catch the error separately so that _initialized will stay rejected if
-  // writing the header failed.
-  this._initialized.catch(aError =>
-                          Cu.reportError("Failed to initialize log file:\n" + aError));
+  this.paths = [];
+  this.startNewFile(this._conv.startDate / 1000);
 }
 LogWriter.prototype = {
-  path: null,
+  // All log file paths used by this LogWriter.
+  paths: [],
+  // Path of the log file that is currently being written to.
+  get currentPath() this.paths[this.paths.length - 1],
   // Constructor sets this to a promise that will resolve when the log header
   // has been written.
   _initialized: null,
+  _startTime: null,
+  _lastMessageTime: null,
+  _messageCount: 0,
   format: "txt",
   encoder: new TextEncoder(),
-  _getHeader: function cl_getHeader() {
+  startNewFile: function lw_startNewFile(aStartTime, aContinuedSession) {
+    // We start a new log file every 1000 messages. The start time of this new
+    // log file is the time of the next message. Since message times are in seconds,
+    // if we receive 1000 messages within a second after starting the new file,
+    // we will create another file, using the same start time - and so the same
+    // file name. To avoid this, ensure the new start time is at least one second
+    // greater than the current one. This is ugly, but should rarely be needed.
+    aStartTime = Math.max(aStartTime, this._startTime + 1000);
+    this._startTime = this._lastMessageTime = aStartTime;
+    this._messageCount = 0;
+    this.paths.push(getLogFilePathForConversation(this._conv, this.format, aStartTime));
     let account = this._conv.account;
+    let header;
     if (this.format == "json") {
-      return JSON.stringify({date: new Date(this._conv.startDate / 1000),
-                             name: this._conv.name,
-                             title: this._conv.title,
-                             account: account.normalizedName,
-                             protocol: account.protocol.normalizedName,
-                             isChat: this._conv.isChat,
-                             normalizedName: this._conv.normalizedName
-                            }) + "\n";
+      header = {
+        date: new Date(this._startTime),
+        name: this._conv.name,
+        title: this._conv.title,
+        account: account.normalizedName,
+        protocol: account.protocol.normalizedName,
+        isChat: this._conv.isChat,
+        normalizedName: this._conv.normalizedName
+      };
+      if (aContinuedSession)
+        header.continuedSession = true;
+      header = JSON.stringify(header) + "\n";
     }
-    return "Conversation with " + this._conv.name +
-           " at " + (new Date(this._conv.startDate / 1000)).toLocaleString() +
-           " on " + account.name +
-           " (" + account.protocol.normalizedName + ")" + kLineBreak;
+    else {
+      header = "Conversation with " + this._conv.name +
+               " at " + (new Date(this._conv.startDate / 1000)).toLocaleString() +
+               " on " + account.name +
+               " (" + account.protocol.normalizedName + ")" + kLineBreak;
+    }
+    this._initialized =
+      appendToFile(this.currentPath, this.encoder.encode(header), true);
+    // Catch the error separately so that _initialized will stay rejected if
+    // writing the header failed.
+    this._initialized.catch(aError =>
+                            Cu.reportError("Failed to initialize log file:\n" + aError));
   },
   _serialize: function cl_serialize(aString) {
     // TODO cleanup once bug 102699 is fixed
     let doc = getHiddenHTMLWindow().document;
     let div = doc.createElementNS("http://www.w3.org/1999/xhtml", "div");
     div.innerHTML = aString.replace(/\r?\n/g, "<br/>").replace(/<br>/gi, "<br/>");
     const type = "text/plain";
     let encoder =
@@ -198,36 +224,62 @@ LogWriter.prototype = {
         let content = aNode.textContent;
         if (url != content)
           aNode.textContent = content + " (" + url + ")";
       }
       return null;
     }});
     return encoder.encodeToString();
   },
+  // We start a new log file in the following cases:
+  // - If it has been 30 minutes since the last message.
+  kInactivityLimit: 30 * 60 * 1000,
+  // - If at midnight, it's been longer than 3 hours since we started the file.
+  kDayOverlapLimit: 3 * 60 * 60 * 1000,
+  // - After every 1000 messages.
+  kMessageCountLimit: 1000,
   logMessage: function cl_logMessage(aMessage) {
+    // aMessage.time is in seconds, we need it in milliseconds.
+    let messageTime = aMessage.time * 1000;
+    let messageMidnight = new Date(messageTime).setHours(0, 0, 0, 0);
+
+    let inactivityLimitExceeded =
+      !aMessage.delayed && messageTime - this._lastMessageTime > this.kInactivityLimit;
+    let dayOverlapLimitExceeded =
+      !aMessage.delayed && messageMidnight - this._startTime > this.kDayOverlapLimit;
+
+    if (inactivityLimitExceeded || dayOverlapLimitExceeded ||
+        this._messageCount == this.kMessageCountLimit) {
+      // We start a new session if the inactivity limit was exceeded.
+      this.startNewFile(messageTime, !inactivityLimitExceeded);
+    }
+    ++this._messageCount;
+
+    if (!aMessage.delayed)
+      this._lastMessageTime = messageTime;
+
     let lineToWrite;
     if (this.format == "json") {
       let msg = {
-        date: new Date(aMessage.time * 1000),
+        date: new Date(messageTime),
         who: aMessage.who,
         text: aMessage.displayMessage,
         flags: ["outgoing", "incoming", "system", "autoResponse",
                 "containsNick", "error", "delayed",
                 "noFormat", "containsImages", "notification",
                 "noLinkification"].filter(function(f) aMessage[f])
       };
       let alias = aMessage.alias;
       if (alias && alias != msg.who)
         msg.alias = alias;
       lineToWrite = JSON.stringify(msg) + "\n";
     }
     else {
       // Text log.
-      let date = new Date(aMessage.time * 1000);
+      let date = new Date(messageTime);
       let line = "(" + date.toLocaleTimeString() + ") ";
       let msg = this._serialize(aMessage.displayMessage);
       if (aMessage.system)
         line += msg;
       else {
         let sender = aMessage.alias || aMessage.who;
         if (aMessage.autoResponse)
           line += sender + " <AUTO-REPLY>: " + msg;
@@ -237,24 +289,25 @@ LogWriter.prototype = {
           else
             line += sender + ": " + msg;
         }
       }
       lineToWrite = line + kLineBreak;
     }
     lineToWrite = this.encoder.encode(lineToWrite);
     this._initialized.then(() => {
-      appendToFile(this.path, lineToWrite)
+      appendToFile(this.currentPath, lineToWrite)
         .catch(aError => Cu.reportError("Failed to log message:\n" + aError));
     });
   }
 };
 
 const dummyLogWriter = {
-  path: null,
+  paths: null,
+  currentPath: null,
   logMessage: function() {}
 };
 
 
 let gLogWritersById = new Map();
 function getLogWriter(aConversation) {
   let id = aConversation.id;
   if (!gLogWritersById.has(id)) {
@@ -490,34 +543,39 @@ Log.prototype = {
         let contents = yield queueFileOperation(path, () => OS.File.read(path));
         lines = decoder.decode(contents).split("\n");
       } catch (aError) {
         Cu.reportError("Error reading log file \"" + path + "\":\n" + aError);
         continue;
       }
       let nextLine = lines.shift();
       let filename = OS.Path.basename(path);
-      let sessionMsg = {
-        who: "sessionstart",
-        date: getDateFromFilename(filename)[0],
-        text: "",
-        flags: ["noLog", "notification"]
-      };
 
       let data;
       try {
         // This will fail if either nextLine is undefined, or not valid JSON.
         data = JSON.parse(nextLine);
       } catch (aError) {
-        sessionMsg.text = _("badLogFile", filename);
-        sessionMsg.flags.push("error", "system");
-        messages.push(sessionMsg);
+        messages.push({
+          who: "sessionstart",
+          date: getDateFromFilename(filename)[0],
+          text: _("badLogFile", filename),
+          flags: ["noLog", "notification", "error", "system"]
+        });
         continue;
       }
-      messages.push(sessionMsg);
+
+      if (firstFile || !data.continuedSession) {
+        messages.push({
+          who: "sessionstart",
+          date: getDateFromFilename(filename)[0],
+          text: "",
+          flags: ["noLog", "notification"]
+        });
+      }
 
       if (firstFile) {
         properties.startDate = new Date(data.date) * 1000;
         properties.name = data.name;
         properties.title = data.title;
         properties._accountName = data.account;
         properties._protocolName = data.protocol;
         properties._isChat = data.isChat;
@@ -682,28 +740,29 @@ Logger.prototype = {
     });
   },
   // Creates and returns the appropriate LogEnumerator for the given log array
   // depending on aGroupByDay, or an EmptyEnumerator if the input array is empty.
   _getEnumerator: function logger__getEnumerator(aLogArray, aGroupByDay) {
     let enumerator = aGroupByDay ? DailyLogEnumerator : LogEnumerator;
     return aLogArray.length ? new enumerator(aLogArray) : EmptyEnumerator;
   },
-  getLogPathForConversation: function logger_getLogPathForConversation(aConversation) {
+  getLogPathsForConversation: Task.async(function* (aConversation) {
     let writer = gLogWritersById.get(aConversation.id);
     // Resolve to null if we haven't created a LogWriter yet for this conv, or
-    // if logging is disabled (path will be null).
-    if (!writer || !writer.path)
-      return Promise.resolve(null);
-    let path = writer.path;
-    // Wait for any pending file operations to finish, then resolve to the path
+    // if logging is disabled (paths will be null).
+    if (!writer || !writer.paths)
+      return null;
+    let paths = writer.paths;
+    // Wait for any pending file operations to finish, then resolve to the paths
     // regardless of whether these operations succeeded.
-    return (gFilePromises.get(path) || Promise.resolve()).then(
-      () => path, () => path);
-  },
+    for (let path of paths)
+      yield gFilePromises.get(path);
+    return paths;
+  }),
   getLogsForAccountAndName: function logger_getLogsForAccountAndName(aAccount,
                                        aNormalizedName, aGroupByDay) {
     return this._getLogArray(aAccount, aNormalizedName)
                .then(aEntries => this._getEnumerator(aEntries, aGroupByDay));
   },
   getLogsForAccountBuddy: function logger_getLogsForAccountBuddy(aAccountBuddy,
                                                                  aGroupByDay) {
     return this.getLogsForAccountAndName(aAccountBuddy.account,
--- a/chat/components/src/test/test_logger.js
+++ b/chat/components/src/test/test_logger.js
@@ -221,40 +221,41 @@ let test_getLogFolderPathForAccount = fu
 // Tests the global function getLogFilePathForConversation in logger.js.
 let test_getLogFilePathForConversation = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyConv, "format");
   let expectedPath = OS.Path.join(logDirPath, dummyAccount.protocol.normalizedName,
                                   gLogger.encodeName(dummyAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyConv.normalizedName));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyConv.startDate));
+    expectedPath, gLogger.getNewLogFileName("format", dummyConv.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_getLogFilePathForMUC = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyMUC, "format");
   let expectedPath = OS.Path.join(logDirPath, dummyAccount.protocol.normalizedName,
                                   gLogger.encodeName(dummyAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyMUC.normalizedName + ".chat"));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyMUC.startDate));
+    expectedPath, gLogger.getNewLogFileName("format", dummyMUC.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_getLogFilePathForTwitterConv = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyTwitterConv, "format");
   let expectedPath =
     OS.Path.join(logDirPath, dummyTwitterAccount.protocol.normalizedName,
                  gLogger.encodeName(dummyTwitterAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyTwitterConv.normalizedName));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyTwitterConv.startDate));
+    expectedPath, gLogger.getNewLogFileName("format",
+                                            dummyTwitterConv.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_appendToFile = function* () {
   const kStringToWrite = "Hello, world!";
   let path = OS.Path.join(OS.Constants.Path.profileDir, "testFile.txt");
   let encoder = new TextEncoder();
   let encodedString = encoder.encode(kStringToWrite);
@@ -263,28 +264,29 @@ let test_appendToFile = function* () {
   gLogger.appendToFile(path, encodedString);
   let text = (new TextDecoder()).decode(
     yield gLogger.queueFileOperation(path, () => OS.File.read(path)));
   // The read text should be equal to kStringToWrite repeated twice.
   equal(text, kStringToWrite + kStringToWrite);
   yield OS.File.remove(path);
 }
 
-// Tests the getLogPathForConversation API defined in the imILogger interface.
-let test_getLogPathForConversation = function* () {
+// Tests the getLogPathsForConversation API defined in the imILogger interface.
+let test_getLogPathsForConversation = function* () {
   let logger = new gLogger.Logger();
-  let path = yield logger.getLogPathForConversation(dummyConv);
+  let paths = yield logger.getLogPathsForConversation(dummyConv);
   // The path should be null since a LogWriter hasn't been created yet.
-  equal(path, null);
+  equal(paths, null);
   let logWriter = gLogger.getLogWriter(dummyConv);
-  path = yield logger.getLogPathForConversation(dummyConv);
-  equal(path, logWriter.path);
-  ok(yield OS.File.exists(path));
+  paths = yield logger.getLogPathsForConversation(dummyConv);
+  equal(paths.length, 1);
+  equal(paths[0], logWriter.currentPath);
+  ok(yield OS.File.exists(paths[0]));
   // Ensure this doesn't interfere with future tests.
-  yield OS.File.remove(path);
+  yield OS.File.remove(paths[0]);
   gLogger.closeLogWriter(dummyConv);
 }
 
 let test_logging = function* () {
   let logger = new gLogger.Logger();
   let oneSec = 1000000; // Microseconds.
 
   // Creates a set of dummy messages for a conv (sets appropriate times).
@@ -325,31 +327,32 @@ let test_logging = function* () {
     let logWriter = gLogger.getLogWriter(aConv);
     for (let message of aMessages)
       logWriter.logMessage(message);
     // If we don't wait for the messages to get written, we have no guarantee
     // later in the test that the log files were created, and getConversation
     // will return an EmptyEnumerator. Logging the messages is queued on the
     // _initialized promise, so we need to yield on that first.
     yield logWriter._initialized;
-    yield gLogger.gFilePromises.get(logWriter.path);
+    yield gLogger.gFilePromises.get(logWriter.currentPath);
     // Ensure two different files for the different dates.
     gLogger.closeLogWriter(aConv);
   });
   yield logMessagesForConv(dummyConv, firstDayMsgs);
   yield logMessagesForConv(dummyConv2, secondDayMsgs);
 
   // Write a zero-length file and a file with incorrect JSON for each day
   // to ensure they are handled correctly.
-  let logDir = OS.Path.dirname(yield gLogger.getLogFilePathForConversation(dummyConv, "json"));
+  let logDir = OS.Path.dirname(
+    gLogger.getLogFilePathForConversation(dummyConv, "json"));
   let createBadFiles = Task.async(function* (aConv) {
     let blankFile = OS.Path.join(logDir,
-      gLogger.getNewLogFileName("json", aConv.startDate + oneSec));
+      gLogger.getNewLogFileName("json", (aConv.startDate + oneSec) / 1000));
     let invalidJSONFile = OS.Path.join(logDir,
-      gLogger.getNewLogFileName("json", aConv.startDate + (2 * oneSec)));
+      gLogger.getNewLogFileName("json", (aConv.startDate + (2 * oneSec)) / 1000));
     let file = yield OS.File.open(blankFile, {truncate: true});
     yield file.close();
     yield OS.File.writeAtomic(invalidJSONFile,
                               new TextEncoder().encode("This isn't JSON!"));
   });
   yield createBadFiles(dummyConv);
   yield createBadFiles(dummyConv2);
 
@@ -415,16 +418,107 @@ let test_logging = function* () {
       yield OS.File.remove(aLog);
     })
   });
   let logFolder = OS.Path.dirname(gLogger.getLogFilePathForConversation(dummyConv));
   // The folder should now be empty - this will throw if it isn't.
   yield OS.File.removeEmptyDir(logFolder, {ignoreAbsent: false});
 }
 
+let test_logFileSplitting = function* () {
+  // Start clean, remove the log directory.
+  let logFolderPath = OS.Path.join(OS.Constants.Path.profileDir, "logs");
+  yield OS.File.removeDir(logFolderPath, {ignoreAbsent: true});
+  let logWriter = gLogger.getLogWriter(dummyConv);
+  let startTime = logWriter._startTime / 1000; // Message times are in seconds.
+  let oldPath = logWriter.currentPath;
+  let message = {
+    time: startTime,
+    who: "John Doe",
+    originalMessage: "Hello, world!",
+    outgoing: true
+  };
+
+  let logMessage = Task.async(function* (aMessage) {
+    logWriter.logMessage(aMessage);
+    yield logWriter._initialized;
+    yield gLogger.gFilePromises.get(logWriter.currentPath);
+  });
+
+  yield logMessage(message);
+  message.time += (logWriter.kInactivityLimit / 1000) + 1;
+  // This should go in a new log file.
+  yield logMessage(message);
+  notEqual(logWriter.currentPath, oldPath);
+  // The log writer's new start time should be the time of the message.
+  equal(message.time * 1000, logWriter._startTime);
+
+  let getCurrentHeader = Task.async(function* () {
+    return JSON.parse(new TextDecoder()
+                      .decode(yield OS.File.read(logWriter.currentPath))
+                      .split("\n")[0]);
+  });
+
+  // The header of the new log file should not have the continuedSession flag set.
+  ok(!(yield getCurrentHeader()).continuedSession);
+
+  // Set the start time sufficiently before midnight, and the last message time
+  // to just before midnight. A new log file should be created at midnight.
+  logWriter._startTime = new Date(logWriter._startTime)
+    .setHours(24, 0, 0, -(logWriter.kDayOverlapLimit + 1));
+  let nearlyMidnight = new Date(logWriter._startTime).setHours(24, 0, 0, -1);
+  oldPath = logWriter.currentPath;
+  logWriter._lastMessageTime = nearlyMidnight;
+  message.time = new Date(nearlyMidnight).setHours(24, 0, 0, 1) / 1000;
+  yield logMessage(message);
+  // The message should have gone in a new file.
+  notEqual(oldPath, logWriter.currentPath);
+  // The header should have the continuedSession flag set this time.
+  ok((yield getCurrentHeader()).continuedSession);
+
+  // Ensure a new file is created every kMessageCountLimit messages.
+  oldPath = logWriter.currentPath;
+  let messageCountLimit = logWriter.kMessageCountLimit;
+  for (let i = 0; i < messageCountLimit; ++i)
+    logMessage(message);
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  // The header should have the continuedSession flag set this time too.
+  ok((yield getCurrentHeader()).continuedSession);
+  // Again, to make sure it still works correctly after splitting it once already.
+  oldPath = logWriter.currentPath;
+  // We already logged one message to ensure it went into a new file, so i = 1.
+  for (let i = 1; i < messageCountLimit; ++i)
+    logMessage(message);
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  ok((yield getCurrentHeader()).continuedSession);
+
+  // The new start time is the time of the message. If we log sufficiently more
+  // messages with the same time property, ensure that the start time of the next
+  // log file is greater than the previous one, and that a new path is being used.
+  let oldStartTime = logWriter._startTime;
+  oldPath = logWriter.currentPath;
+  logWriter._messageCount = messageCountLimit;
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  ok(logWriter._startTime > oldStartTime);
+
+  // Do it again with the same message.
+  oldStartTime = logWriter._startTime;
+  oldPath = logWriter.currentPath;
+  logWriter._messageCount = messageCountLimit;
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  ok(logWriter._startTime > oldStartTime);
+
+  // Clean up.
+  yield OS.File.removeDir(logFolderPath);
+}
+
 function run_test() {
   // Test encodeName().
   for (let i = 0; i < encodeName_input.length; ++i)
     equal(gLogger.encodeName(encodeName_input[i]), encodeName_output[i]);
 
   // Test convIsRealMUC().
   ok(!gLogger.convIsRealMUC(dummyConv));
   ok(!gLogger.convIsRealMUC(dummyTwitterConv));
@@ -437,14 +531,16 @@ function run_test() {
   add_task(test_getLogFilePathForMUC);
 
   add_task(test_getLogFilePathForTwitterConv);
 
   add_task(test_queueFileOperation);
 
   add_task(test_appendToFile);
 
-  add_task(test_getLogPathForConversation);
+  add_task(test_getLogPathsForConversation);
 
   add_task(test_logging);
 
+  add_task(test_logFileSplitting);
+
   run_next_test();
 }
--- a/mail/components/im/modules/index_im.js
+++ b/mail/components/im/modules/index_im.js
@@ -348,17 +348,17 @@ var GlodaIMIndexer = {
     let convId = aConversation.id;
 
     // If we've already scheduled this conversation to be indexed, let's
     // not repeat.
     if (!(convId in this._knownConversations)) {
       this._knownConversations[convId] = {
         id: convId,
         scheduledIndex: null,
-        logFile: null,
+        logFileCount: null,
         convObj: {}
       };
     }
 
     if (!this._knownConversations[convId].scheduledIndex) {
       // Ok, let's schedule the job.
       this._knownConversations[convId].scheduledIndex = setTimeout(
         this._beginIndexingJob.bind(this, aConversation),
@@ -372,75 +372,87 @@ var GlodaIMIndexer = {
     // In the event that we're triggering this indexing job manually, without
     // bothering to schedule it (for example, when a conversation is closed),
     // we give the conversation an entry in _knownConversations, which would
     // normally have been done in _scheduleIndexingJob.
     if (!(convId in this._knownConversations)) {
       this._knownConversations[convId] = {
         id: convId,
         scheduledIndex: null,
-        logFile: null,
+        logFileCount: null,
         convObj: {}
       };
     }
 
     let conv = this._knownConversations[convId];
     Task.spawn(function* () {
-      if (!conv.logFile) {
-        let logFile =
-          yield Services.logs.getLogPathForConversation(aConversation);
-        if (!logFile) {
-          // Log file doesn't exist yet, nothing to do!
-          return;
-        }
+      // We need to get the log files every time, because a new log file might
+      // have been started since we last got them.
+      let logFiles =
+        yield Services.logs.getLogPathsForConversation(aConversation);
+      if (!logFiles.length) {
+        // No log files exist yet, nothing to do!
+        return;
+      }
 
-        // We initialize the _knownFiles tree path for the current file below in
+      if (conv.logFileCount == undefined) {
+        // We initialize the _knownFiles tree path for the current files below in
         // case it doesn't already exist.
-        let folder = OS.Path.dirname(logFile);
+        let folder = OS.Path.dirname(logFiles[0]);
         let convName = OS.Path.basename(folder);
         folder = OS.Path.dirname(folder);
         let accountName = OS.Path.basename(folder);
         folder = OS.Path.dirname(folder);
         let protoName = OS.Path.basename(folder);
         if (!Object.prototype.hasOwnProperty.call(this._knownFiles, protoName))
           this._knownFiles[protoName] = {};
         let protoObj = this._knownFiles[protoName];
         if (!Object.prototype.hasOwnProperty.call(protoObj, accountName))
           protoObj[accountName] = {};
         let accountObj = protoObj[accountName];
         if (!Object.prototype.hasOwnProperty.call(accountObj, convName))
           accountObj[convName] = {};
 
-        conv.logFile = logFile;
         // convObj is the penultimate level of the tree,
         // maps file name -> last modified time
         conv.convObj = accountObj[convName];
-      }
-
-      let logPath = conv.logFile;
-      let fileName = OS.Path.basename(logPath);
-      let lastModifiedTime =
-        (yield OS.File.stat(logPath)).lastModificationDate.valueOf();
-      if (Object.prototype.hasOwnProperty.call(conv.convObj, fileName) &&
-          conv.convObj[fileName] == lastModifiedTime) {
-        // The file hasn't changed since we last indexed it, so we're done.
-        return;
+        conv.logFileCount = 0;
       }
 
-      if (this._indexingJobPromise)
-        yield this._indexingJobPromise;
-      this._indexingJobPromise = new Promise(aResolve => {
-        this._indexingJobCallbacks.set(convId, aResolve);
-      });
+      // The last log file in the array is the one currently being written to.
+      // When new log files are started, we want to finish indexing the previous
+      // one as well as index the new ones. The index of the previous one is
+      // conv.logFiles.length - 1, so we slice from there. This gives us all new
+      // log files even if there are multiple new ones.
+      let currentLogFiles = conv.logFileCount > 1 ?
+                            logFiles.slice(conv.logFileCount - 1) :
+                            logFiles;
+      for (let logFile of currentLogFiles) {
+        let fileName = OS.Path.basename(logFile);
+        let lastModifiedTime =
+          (yield OS.File.stat(logFile)).lastModificationDate.valueOf();
+        if (Object.prototype.hasOwnProperty.call(conv.convObj, fileName) &&
+            conv.convObj[fileName] == lastModifiedTime) {
+          // The file hasn't changed since we last indexed it, so we're done.
+          continue;
+        }
 
-      let job = new IndexingJob("indexIMConversation", null);
-      job.conversation = conv;
-      job.path = logPath;
-      job.lastModifiedTime = lastModifiedTime;
-      GlodaIndexer.indexJob(job);
+        if (this._indexingJobPromise)
+          yield this._indexingJobPromise;
+        this._indexingJobPromise = new Promise(aResolve => {
+          this._indexingJobCallbacks.set(convId, aResolve);
+        });
+
+        let job = new IndexingJob("indexIMConversation", null);
+        job.conversation = conv;
+        job.path = logFile;
+        job.lastModifiedTime = lastModifiedTime;
+        GlodaIndexer.indexJob(job);
+      }
+      conv.logFileCount = logFiles.length;
     }.bind(this)).catch(Components.utils.reportError);
 
     // Now clear the job, so we can index in the future.
     this._knownConversations[convId].scheduledIndex = null;
   },
 
   observe: function logger_observe(aSubject, aTopic, aData) {
     if (aTopic == "new-ui-conversation") {
@@ -525,16 +537,22 @@ var GlodaIMIndexer = {
         },
         handleError: aError =>
           Cu.reportError("Error finding gloda id from path:\n" + aError),
         handleCompletion: () => {resolve(id);}
       });
     });
   },
 
+  // Get the path of a log file relative to the logs directory - the last 4
+  // components of the path.
+  _getRelativePath: function(aLogPath) {
+    return OS.Path.split(aLogPath).components.slice(-4).join("/");
+  },
+
   /* aGlodaConv is an optional inout param that lets the caller save and reuse
    * the GlodaIMConversation instance created when the conversation is indexed
    * the first time. After a conversation is indexed for the first time,
    * the GlodaIMConversation instance has its id property set to the row id of
    * the conversation in the database. This id is required to later update the
    * conversation in the database, so the caller dealing with ongoing
    * conversation has to provide the aGlodaConv parameter, while the caller
    * dealing with old conversations doesn't care.
@@ -565,19 +583,17 @@ var GlodaIMIndexer = {
                          })
                          .join("\n\n");
     let glodaConv;
     if (aGlodaConv && aGlodaConv.value) {
       glodaConv = aGlodaConv.value;
       glodaConv._content = content;
     }
     else {
-      // Get the path of the file relative to the logs directory - the last 4
-      // components of the path.
-      let relativePath = OS.Path.split(aLogPath).components.slice(-4).join("/");
+      let relativePath = this._getRelativePath(aLogPath);
       glodaConv = new GlodaIMConversation(logConv.title, log.time, relativePath, content);
       // If we've indexed this file before, we need the id of the existing
       // gloda conversation so that the existing entry gets updated. This can
       // happen if the log sweep detects that the last messages in an open
       // chat were not in fact indexed before that session was shut down.
       let id = yield this._getIdFromPath(relativePath);
       if (id)
         glodaConv.id = id;
@@ -596,17 +612,19 @@ var GlodaIMIndexer = {
     aCache[fileName] = aLastModifiedTime || 1;
     this._scheduleCacheSave();
 
     return rv;
   }),
 
   _worker_indexIMConversation: function(aJob, aCallbackHandle) {
     let glodaConv = {};
-    if (aJob.conversation.glodaConv)
+    let existingGlodaConv = aJob.conversation.glodaConv;
+    if (existingGlodaConv &&
+        existingGlodaConv.path == this._getRelativePath(aJob.path))
       glodaConv.value = aJob.conversation.glodaConv;
 
     // indexIMConversation may initiate an async grokNounItem sub-job.
     this.indexIMConversation(aCallbackHandle, aJob.path, aJob.lastModifiedTime,
                              aJob.conversation.convObj, glodaConv)
         .then(() => GlodaIndexer.callbackDriver());
     // Tell the Indexer that we're doing async indexing. We'll be left alone
     // until callbackDriver() is called above.