Bug 1025522 - Split log files to prevent them from growing too large. r=aleth, florian draft
authorNihanth Subramanya <nhnt11@gmail.com>
Fri, 27 Jun 2014 04:46:45 +0530
changeset 25825 278eceeeb7e3fb311b0d281124b3cf991fe6fe59
parent 25819 17422c1f7444d216320f2b9bd91a9cd7245bf526
child 25826 261491094d19d8019648f76c813dbe60720c766e
push id1711
push usernhnt11@gmail.com
push dateThu, 11 Sep 2014 17:02:55 +0000
treeherdertry-comm-central@67d546c5ad5d [default view] [failures only]
reviewersaleth, florian
bugs1025522
Bug 1025522 - Split log files to prevent them from growing too large. r=aleth, florian
chat/components/public/imILogger.idl
chat/components/src/logger.js
chat/components/src/test/test_logger.js
mail/components/im/modules/index_im.js
--- a/chat/components/public/imILogger.idl
+++ b/chat/components/public/imILogger.idl
@@ -58,20 +58,20 @@ interface imIProcessLogsCallback: nsISup
   // iteration will stop.
   jsval processLog(in AUTF8String aLogPath);
 };
 
 [scriptable, uuid(b9d5701a-df53-4e0e-99b7-706e0118e075)]
 interface imILogger: nsISupports {
   // Returns a promise that resolves to an imILog instance.
   jsval getLogFromFile(in AUTF8String aFilePath, [optional] in boolean aGroupByDay);
-  // Returns a promise that resolves to the log file path if a log writer
+  // Returns a promise that resolves to the log file paths if a log writer
   // exists for the conversation, or null otherwise. The promise resolves
-  // after any pending I/O operations on the file complete.
-  jsval getLogPathForConversation(in prplIConversation aConversation);
+  // after any pending I/O operations on the files complete.
+  jsval getLogPathsForConversation(in prplIConversation aConversation);
 
   // Below methods return promises that resolve to nsISimpleEnumerator instances.
 
   // Get logs for a username that may not be in the contact list.
   jsval getLogsForAccountAndName(in imIAccount aAccount,
                                  in AUTF8String aNormalizedName,
                                  [optional] in boolean aGroupByDay);
 
--- a/chat/components/src/logger.js
+++ b/chat/components/src/logger.js
@@ -109,27 +109,29 @@ function encodeName(aName) {
 }
 
 function getLogFolderPathForAccount(aAccount) {
   return OS.Path.join(OS.Constants.Path.profileDir,
                       "logs", aAccount.protocol.normalizedName,
                       encodeName(aAccount.normalizedName));
 }
 
-function getLogFilePathForConversation(aConv, aFormat) {
+function getLogFilePathForConversation(aConv, aFormat, aStartTime) {
+  if (!aStartTime)
+    aStartTime = aConv.startDate / 1000;
   let path = getLogFolderPathForAccount(aConv.account);
   let name = aConv.normalizedName;
   if (convIsRealMUC(aConv))
     name += ".chat";
   return OS.Path.join(path, encodeName(name),
-                      getNewLogFileName(aFormat, aConv.startDate));
+                      getNewLogFileName(aFormat, aStartTime));
 }
 
-function getNewLogFileName(aFormat, aDate) {
-  let date = aDate ? new Date(aDate / 1000) : new Date();
+function getNewLogFileName(aFormat, aStartTime) {
+  let date = aStartTime ? new Date(aStartTime) : new Date();
   let dateTime = date.toLocaleFormat("%Y-%m-%d.%H%M%S");
   let offset = date.getTimezoneOffset();
   if (offset < 0) {
     dateTime += "+";
     offset *= -1;
   }
   else
     dateTime += "-";
@@ -144,47 +146,71 @@ function getNewLogFileName(aFormat, aDat
 
 
 // One of these is maintained for every conversation being logged. It initializes
 // a log file and appends to it as required.
 function LogWriter(aConversation) {
   this._conv = aConversation;
   if (Services.prefs.getCharPref("purple.logging.format") == "json")
     this.format = "json";
-  this.path = getLogFilePathForConversation(aConversation, this.format);
-  this._initialized =
-    appendToFile(this.path, this.encoder.encode(this._getHeader()), true);
-  // Catch the error separately so that _initialized will stay rejected if
-  // writing the header failed.
-  this._initialized.catch(aError =>
-                          Cu.reportError("Failed to initialize log file:\n" + aError));
+  this.paths = [];
+  this.startNewFile(this._conv.startDate / 1000);
 }
 LogWriter.prototype = {
-  path: null,
+  // All log file paths used by this LogWriter.
+  paths: [],
+  // Path of the log file that is currently being written to.
+  get currentPath() this.paths[this.paths.length - 1],
   // Constructor sets this to a promise that will resolve when the log header
   // has been written.
   _initialized: null,
+  _startTime: null,
+  _lastMessageTime: null,
+  _messageCount: 0,
   format: "txt",
   encoder: new TextEncoder(),
-  _getHeader: function cl_getHeader() {
+  startNewFile: function lw_startNewFile(aStartTime, aContinuedSession) {
+    // We start a new log file every 1000 messages. The start time of this new
+    // log file is the time of the next message. Since message times are in seconds,
+    // if we receive 1000 messages within a second after starting the new file,
+    // we will create another file, using the same start time - and so the same
+    // file name. To avoid this, ensure the new start time is at least one second
+    // greater than the current one. This is ugly, but should rarely be needed.
+    aStartTime = Math.max(aStartTime, this._startTime + 1000);
+    this._startTime = this._lastMessageTime = aStartTime;
+    this._messageCount = 0;
+    this.paths.push(getLogFilePathForConversation(this._conv, this.format, aStartTime));
     let account = this._conv.account;
+    let header;
     if (this.format == "json") {
-      return JSON.stringify({date: new Date(this._conv.startDate / 1000),
-                             name: this._conv.name,
-                             title: this._conv.title,
-                             account: account.normalizedName,
-                             protocol: account.protocol.normalizedName,
-                             isChat: this._conv.isChat,
-                             normalizedName: this._conv.normalizedName
-                            }) + "\n";
+      header = {
+        date: new Date(this._startTime),
+        name: this._conv.name,
+        title: this._conv.title,
+        account: account.normalizedName,
+        protocol: account.protocol.normalizedName,
+        isChat: this._conv.isChat,
+        normalizedName: this._conv.normalizedName
+      };
+      if (aContinuedSession)
+        header.continuedSession = true;
+      header = JSON.stringify(header) + "\n";
     }
-    return "Conversation with " + this._conv.name +
-           " at " + (new Date(this._conv.startDate / 1000)).toLocaleString() +
-           " on " + account.name +
-           " (" + account.protocol.normalizedName + ")" + kLineBreak;
+    else {
+      header = "Conversation with " + this._conv.name +
+               " at " + (new Date(this._conv.startDate / 1000)).toLocaleString() +
+               " on " + account.name +
+               " (" + account.protocol.normalizedName + ")" + kLineBreak;
+    }
+    this._initialized =
+      appendToFile(this.currentPath, this.encoder.encode(header), true);
+    // Catch the error separately so that _initialized will stay rejected if
+    // writing the header failed.
+    this._initialized.catch(aError =>
+                            Cu.reportError("Failed to initialize log file:\n" + aError));
   },
   _serialize: function cl_serialize(aString) {
     // TODO cleanup once bug 102699 is fixed
     let doc = getHiddenHTMLWindow().document;
     let div = doc.createElementNS("http://www.w3.org/1999/xhtml", "div");
     div.innerHTML = aString.replace(/\r?\n/g, "<br/>").replace(/<br>/gi, "<br/>");
     const type = "text/plain";
     let encoder =
@@ -198,36 +224,61 @@ LogWriter.prototype = {
         let content = aNode.textContent;
         if (url != content)
           aNode.textContent = content + " (" + url + ")";
       }
       return null;
     }});
     return encoder.encodeToString();
   },
-  logMessage: function cl_logMessage(aMessage) {
+  // We start a new log file in the following cases:
+  // - If it has been 30 minutes since the last message.
+  kInactivityLimit: 30 * 60 * 1000,
+  // - If at midnight, it's been longer than 3 hours since we started the file.
+  kDayOverlapLimit: 3 * 60 * 60 * 1000,
+  // - After every 1000 messages.
+  kMessageCountLimit: 1000,
+  logMessage: function lw_logMessage(aMessage) {
+    // aMessage.time is in seconds, we need it in milliseconds.
+    let messageTime = aMessage.time * 1000;
+    let messageMidnight = new Date(messageTime).setHours(0, 0, 0, 0);
+
+    let inactivityLimitExceeded =
+      !aMessage.delayed && messageTime - this._lastMessageTime > this.kInactivityLimit;
+    let dayOverlapLimitExceeded =
+      !aMessage.delayed && messageMidnight - this._startTime > this.kDayOverlapLimit;
+
+    if (inactivityLimitExceeded || dayOverlapLimitExceeded ||
+        ++this._messageCount > this.kMessageCountLimit) {
+      // We start a new session if the inactivity limit was exceeded.
+      this.startNewFile(messageTime, !inactivityLimitExceeded);
+    }
+
+    if (!aMessage.delayed)
+      this._lastMessageTime = messageTime;
+
     let lineToWrite;
     if (this.format == "json") {
       let msg = {
-        date: new Date(aMessage.time * 1000),
+        date: new Date(messageTime),
         who: aMessage.who,
         text: aMessage.originalMessage,
         flags: ["outgoing", "incoming", "system", "autoResponse",
                 "containsNick", "error", "delayed",
                 "noFormat", "containsImages", "notification",
                 "noLinkification"].filter(function(f) aMessage[f])
       };
       let alias = aMessage.alias;
       if (alias && alias != msg.who)
         msg.alias = alias;
       lineToWrite = JSON.stringify(msg) + "\n";
     }
     else {
       // Text log.
-      let date = new Date(aMessage.time * 1000);
+      let date = new Date(messageTime);
       let line = "(" + date.toLocaleTimeString() + ") ";
       let msg = this._serialize(aMessage.originalMessage);
       if (aMessage.system)
         line += msg;
       else {
         let sender = aMessage.alias || aMessage.who;
         if (aMessage.autoResponse)
           line += sender + " <AUTO-REPLY>: " + msg;
@@ -237,24 +288,25 @@ LogWriter.prototype = {
           else
             line += sender + ": " + msg;
         }
       }
       lineToWrite = line + kLineBreak;
     }
     lineToWrite = this.encoder.encode(lineToWrite);
     this._initialized.then(() => {
-      appendToFile(this.path, lineToWrite)
+      appendToFile(this.currentPath, lineToWrite)
         .catch(aError => Cu.reportError("Failed to log message:\n" + aError));
     });
   }
 };
 
 const dummyLogWriter = {
-  path: null,
+  paths: null,
+  currentPath: null,
   logMessage: function() {}
 };
 
 
 let gLogWritersById = new Map();
 function getLogWriter(aConversation) {
   let id = aConversation.id;
   if (!gLogWritersById.has(id)) {
@@ -486,34 +538,39 @@ Log.prototype = {
         let contents = yield queueFileOperation(path, () => OS.File.read(path));
         lines = decoder.decode(contents).split("\n");
       } catch (aError) {
         Cu.reportError("Error reading log file \"" + path + "\":\n" + aError);
         continue;
       }
       let nextLine = lines.shift();
       let filename = OS.Path.basename(path);
-      let sessionMsg = {
-        who: "sessionstart",
-        date: getDateFromFilename(filename)[0],
-        text: "",
-        flags: ["noLog", "notification"]
-      };
 
       let data;
       try {
         // This will fail if either nextLine is undefined, or not valid JSON.
         data = JSON.parse(nextLine);
       } catch (aError) {
-        sessionMsg.text = _("badLogFile", filename);
-        sessionMsg.flags.push("error", "system");
-        messages.push(sessionMsg);
+        messages.push({
+          who: "sessionstart",
+          date: getDateFromFilename(filename)[0],
+          text: _("badLogFile", filename),
+          flags: ["noLog", "notification", "error", "system"]
+        });
         continue;
       }
-      messages.push(sessionMsg);
+
+      if (firstFile || !data.continuedSession) {
+        messages.push({
+          who: "sessionstart",
+          date: getDateFromFilename(filename)[0],
+          text: "",
+          flags: ["noLog", "notification"]
+        });
+      }
 
       if (firstFile) {
         properties.startDate = new Date(data.date) * 1000;
         properties.name = data.name;
         properties.title = data.title;
         properties._accountName = data.account;
         properties._protocolName = data.protocol;
         properties._isChat = data.isChat;
@@ -679,28 +736,29 @@ Logger.prototype = {
     });
   },
   // Creates and returns the appropriate LogEnumerator for the given log array
   // depending on aGroupByDay, or an EmptyEnumerator if the input array is empty.
   _getEnumerator: function logger__getEnumerator(aLogArray, aGroupByDay) {
     let enumerator = aGroupByDay ? DailyLogEnumerator : LogEnumerator;
     return aLogArray.length ? new enumerator(aLogArray) : EmptyEnumerator;
   },
-  getLogPathForConversation: function logger_getLogPathForConversation(aConversation) {
+  getLogPathsForConversation: Task.async(function* (aConversation) {
     let writer = gLogWritersById.get(aConversation.id);
     // Resolve to null if we haven't created a LogWriter yet for this conv, or
-    // if logging is disabled (path will be null).
-    if (!writer || !writer.path)
-      return Promise.resolve(null);
-    let path = writer.path;
-    // Wait for any pending file operations to finish, then resolve to the path
+    // if logging is disabled (paths will be null).
+    if (!writer || !writer.paths)
+      return null;
+    let paths = writer.paths;
+    // Wait for any pending file operations to finish, then resolve to the paths
     // regardless of whether these operations succeeded.
-    return (gFilePromises.get(path) || Promise.resolve()).then(
-      () => path, () => path);
-  },
+    for (let path of paths)
+      yield gFilePromises.get(path);
+    return paths;
+  }),
   getLogsForAccountAndName: function logger_getLogsForAccountAndName(aAccount,
                                        aNormalizedName, aGroupByDay) {
     return this._getLogArray(aAccount, aNormalizedName)
                .then(aEntries => this._getEnumerator(aEntries, aGroupByDay));
   },
   getLogsForAccountBuddy: function logger_getLogsForAccountBuddy(aAccountBuddy,
                                                                  aGroupByDay) {
     return this.getLogsForAccountAndName(aAccountBuddy.account,
--- a/chat/components/src/test/test_logger.js
+++ b/chat/components/src/test/test_logger.js
@@ -221,68 +221,70 @@ let test_getLogFolderPathForAccount = fu
 // Tests the global function getLogFilePathForConversation in logger.js.
 let test_getLogFilePathForConversation = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyConv, "format");
   let expectedPath = OS.Path.join(logDirPath, dummyAccount.protocol.normalizedName,
                                   gLogger.encodeName(dummyAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyConv.normalizedName));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyConv.startDate));
+    expectedPath, gLogger.getNewLogFileName("format", dummyConv.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_getLogFilePathForMUC = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyMUC, "format");
   let expectedPath = OS.Path.join(logDirPath, dummyAccount.protocol.normalizedName,
                                   gLogger.encodeName(dummyAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyMUC.normalizedName + ".chat"));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyMUC.startDate));
+    expectedPath, gLogger.getNewLogFileName("format", dummyMUC.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_getLogFilePathForTwitterConv = function* () {
   let path = gLogger.getLogFilePathForConversation(dummyTwitterConv, "format");
   let expectedPath =
     OS.Path.join(logDirPath, dummyTwitterAccount.protocol.normalizedName,
                  gLogger.encodeName(dummyTwitterAccount.normalizedName));
   expectedPath = OS.Path.join(
     expectedPath, gLogger.encodeName(dummyTwitterConv.normalizedName));
   expectedPath = OS.Path.join(
-    expectedPath, gLogger.getNewLogFileName("format", dummyTwitterConv.startDate));
+    expectedPath, gLogger.getNewLogFileName("format",
+                                            dummyTwitterConv.startDate / 1000));
   equal(path, expectedPath);
 }
 
 let test_appendToFile = function* () {
   const kStringToWrite = "Hello, world!";
   let path = OS.Path.join(OS.Constants.Path.profileDir, "testFile.txt");
   let encodedString = (new TextEncoder()).encode(kStringToWrite);
   gLogger.appendToFile(path, encodedString);
   gLogger.appendToFile(path, encodedString);
   let text = (new TextDecoder()).decode(
     yield gLogger.queueFileOperation(path, () => OS.File.read(path)));
   // The read text should be equal to kStringToWrite repeated twice.
   equal(text, kStringToWrite + kStringToWrite);
   yield OS.File.remove(path);
 }
 
-// Tests the getLogPathForConversation API defined in the imILogger interface.
-let test_getLogPathForConversation = function* () {
+// Tests the getLogPathsForConversation API defined in the imILogger interface.
+let test_getLogPathsForConversation = function* () {
   let logger = new gLogger.Logger();
-  let path = yield logger.getLogPathForConversation(dummyConv);
+  let paths = yield logger.getLogPathsForConversation(dummyConv);
   // The path should be null since a LogWriter hasn't been created yet.
-  equal(path, null);
+  equal(paths, null);
   let logWriter = gLogger.getLogWriter(dummyConv);
-  let path = yield logger.getLogPathForConversation(dummyConv);
-  equal(path, logWriter.path);
-  ok(yield OS.File.exists(path));
+  paths = yield logger.getLogPathsForConversation(dummyConv);
+  equal(paths.length, 1);
+  equal(paths[0], logWriter.currentPath);
+  ok(yield OS.File.exists(paths[0]));
   // Ensure this doesn't interfere with future tests.
-  yield OS.File.remove(path);
+  yield OS.File.remove(paths[0]);
   gLogger.closeLogWriter(dummyConv);
 }
 
 let test_logging = function* () {
   let logger = new gLogger.Logger();
   let oneSec = 1000000; // Microseconds.
 
   // Creates a set of dummy messages for a conv (sets appropriate times).
@@ -323,31 +325,32 @@ let test_logging = function* () {
     let logWriter = gLogger.getLogWriter(aConv);
     for (let message of aMessages)
       logWriter.logMessage(message);
     // If we don't wait for the messages to get written, we have no guarantee
     // later in the test that the log files were created, and getConversation
     // will return an EmptyEnumerator. Logging the messages is queued on the
     // _initialized promise, so we need to yield on that first.
     yield logWriter._initialized;
-    yield gLogger.gFilePromises.get(logWriter.path);
+    yield gLogger.gFilePromises.get(logWriter.currentPath);
     // Ensure two different files for the different dates.
     gLogger.closeLogWriter(aConv);
   });
   yield logMessagesForConv(dummyConv, firstDayMsgs);
   yield logMessagesForConv(dummyConv2, secondDayMsgs);
 
   // Write a zero-length file and a file with incorrect JSON for each day
   // to ensure they are handled correctly.
-  let logDir = OS.Path.dirname(yield gLogger.getLogFilePathForConversation(dummyConv, "json"));
+  let logDir = OS.Path.dirname(
+    gLogger.getLogFilePathForConversation(dummyConv, "json"));
   let createBadFiles = Task.async(function* (aConv) {
     let blankFile = OS.Path.join(logDir,
-      gLogger.getNewLogFileName("json", aConv.startDate + oneSec));
+      gLogger.getNewLogFileName("json", (aConv.startDate + oneSec) / 1000));
     let invalidJSONFile = OS.Path.join(logDir,
-      gLogger.getNewLogFileName("json", aConv.startDate + (2 * oneSec)));
+      gLogger.getNewLogFileName("json", (aConv.startDate + (2 * oneSec)) / 1000));
     let file = yield OS.File.open(blankFile, {truncate: true});
     yield file.close();
     yield OS.File.writeAtomic(invalidJSONFile,
                               new TextEncoder().encode("This isn't JSON!"));
   });
   yield createBadFiles(dummyConv);
   yield createBadFiles(dummyConv2);
 
@@ -413,16 +416,101 @@ let test_logging = function* () {
       yield OS.File.remove(aLog);
     })
   });
   let logFolder = OS.Path.dirname(gLogger.getLogFilePathForConversation(dummyConv));
   // The folder should now be empty - this will throw if it isn't.
   yield OS.File.removeEmptyDir(logFolder, {ignoreAbsent: false});
 }
 
+let test_logFileSplitting = function* () {
+  // Start clean, remove the log directory.
+  let logFolderPath = OS.Path.join(OS.Constants.Path.profileDir, "logs");
+  yield OS.File.removeDir(logFolderPath, {ignoreAbsent: true});
+  let logWriter = gLogger.getLogWriter(dummyConv);
+  let startTime = logWriter._startTime / 1000; // Message times are in seconds.
+  let oldPath = logWriter.currentPath;
+  let message = {
+    time: startTime,
+    who: "John Doe",
+    originalMessage: "Hello, world!",
+    outgoing: true
+  };
+
+  let logMessage = Task.async(function* (aMessage) {
+    logWriter.logMessage(aMessage);
+    yield logWriter._initialized;
+    yield gLogger.gFilePromises.get(logWriter.currentPath);
+  });
+
+  yield logMessage(message);
+  message.time += (logWriter.kInactivityLimit / 1000) + 1;
+  // This should go in a new log file.
+  yield logMessage(message);
+  notEqual(logWriter.currentPath, oldPath);
+  // The log writer's new start time should be the time of the message.
+  equal(message.time * 1000, logWriter._startTime);
+  // The header of the new log file should not have the continuedSession flag set.
+  let header = JSON.parse(new TextDecoder()
+                          .decode(yield OS.File.read(logWriter.currentPath))
+                          .split("\n")[0]);
+  ok(!header.continuedSession);
+
+  // Set the start time sufficiently before midnight, and the last message time
+  // to just before midnight. A new log file should be created at midnight.
+  logWriter._startTime = new Date(logWriter._startTime)
+    .setHours(24, 0, 0, -(logWriter.kDayOverlapLimit + 1));
+  let nearlyMidnight = new Date(logWriter._startTime).setHours(24, 0, 0, -1);
+  oldPath = logWriter.currentPath;
+  logWriter._lastMessageTime = nearlyMidnight;
+  message.time = new Date(nearlyMidnight).setHours(24, 0, 0, 1) / 1000;
+  yield logMessage(message);
+  // The message should have gone in a new file.
+  notEqual(oldPath, logWriter.currentPath);
+  // The header should have the continuedSession flag set this time.
+  let header = JSON.parse(new TextDecoder()
+                          .decode(yield OS.File.read(logWriter.currentPath))
+                          .split("\n")[0]);
+  ok(header.continuedSession);
+
+  // Ensure a new file is created after 1000 messages.
+  oldPath = logWriter.currentPath;
+  let messageCountLimit = logWriter.kMessageCountLimit;
+  for (let i = 0; i < messageCountLimit; ++i)
+    logMessage(message);
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  // The header should have the continuedSession flag set this time too.
+  let header = JSON.parse(new TextDecoder()
+                          .decode(yield OS.File.read(logWriter.currentPath))
+                          .split("\n")[0]);
+  ok(header.continuedSession);
+
+  // The new start time is the time of the message. If we log sufficiently more
+  // messages with the same time property, ensure that the start time of the next
+  // log file is greater than the previous one, and that a new path is being used.
+  let oldStartTime = logWriter._startTime;
+  oldPath = logWriter.currentPath;
+  logWriter._messageCount = messageCountLimit;
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  ok(logWriter._startTime > oldStartTime);
+
+  // Do it again with the same message.
+  oldStartTime = logWriter._startTime;
+  oldPath = logWriter.currentPath;
+  logWriter._messageCount = messageCountLimit;
+  yield logMessage(message);
+  notEqual(oldPath, logWriter.currentPath);
+  ok(logWriter._startTime > oldStartTime);
+
+  // Clean up.
+  yield OS.File.removeDir(logFolderPath);
+}
+
 function run_test() {
   // Test encodeName().
   for (let i = 0; i < encodeName_input.length; ++i)
     equal(gLogger.encodeName(encodeName_input[i]), encodeName_output[i]);
 
   // Test convIsRealMUC().
   ok(!gLogger.convIsRealMUC(dummyConv));
   ok(!gLogger.convIsRealMUC(dummyTwitterConv));
@@ -435,14 +523,16 @@ function run_test() {
   add_task(test_getLogFilePathForMUC);
 
   add_task(test_getLogFilePathForTwitterConv);
 
   add_task(test_queueFileOperation);
 
   add_task(test_appendToFile);
 
-  add_task(test_getLogPathForConversation);
+  add_task(test_getLogPathsForConversation);
 
   add_task(test_logging);
 
+  add_task(test_logFileSplitting);
+
   run_next_test();
 }
--- a/mail/components/im/modules/index_im.js
+++ b/mail/components/im/modules/index_im.js
@@ -311,64 +311,66 @@ var GlodaIMIndexer = {
     // In the event that we're triggering this indexing job manually, without
     // bothering to schedule it (for example, when a conversation is closed),
     // we give the conversation an entry in _knownConversations, which would
     // normally have been done in _scheduleIndexingJob.
     if (!(convId in this._knownConversations))
       this._knownConversations[convId] = {id: convId};
 
     Task.spawn(function* () {
-      if (!this._knownConversations[convId].logFile) {
-        let logFile =
-          yield Services.logs.getLogPathForConversation(aConversation);
-        if (!logFile) {
-          // Log file doesn't exist yet, nothing to do!
+      if (!this._knownConversations[convId].logFiles) {
+        let logFiles =
+          yield Services.logs.getLogPathsForConversation(aConversation);
+        if (!logFiles.length) {
+          // Log files doesn't exist yet, nothing to do!
           return;
         }
-        let folder = OS.Path.dirname(logFile);
+
+        let folder = OS.Path.dirname(logFiles[0]);
         let convName = OS.Path.basename(folder);
         folder = OS.Path.dirname(folder);
         let accountName = OS.Path.basename(folder);
         folder = OS.Path.dirname(folder);
         let protoName = OS.Path.basename(folder);
         if (!Object.prototype.hasOwnProperty.call(this._knownFiles, protoName))
           this._knownFiles[protoName] = {};
         let protoObj = this._knownFiles[protoName];
         if (!Object.prototype.hasOwnProperty.call(protoObj, accountName))
           protoObj[accountName] = {};
         let accountObj = protoObj[accountName];
         if (!Object.prototype.hasOwnProperty.call(accountObj, convName))
           accountObj[convName] = {};
-
-        this._knownConversations[convId].logFile = logFile;
+        this._knownConversations[convId].logFiles = logFiles;
         this._knownConversations[convId].convObj = accountObj[convName];
       }
 
       let conv = this._knownConversations[convId];
       let cache = conv.convObj;
-      let fileName = OS.Path.basename(conv.logFile);
-      let fileInfo = yield OS.File.stat(conv.logFile);
-      let lastModifiedTime = fileInfo.lastModificationDate.valueOf();
-      if (Object.prototype.hasOwnProperty.call(cache, fileName) &&
-          cache[fileName] == lastModifiedTime)
-        return;
-      if (this._indexingJobPromise)
-        yield this._indexingJobPromise;
-      this._indexingJobPromise = new Promise(aResolve => {
-        this._indexingJobCallbacks.set(convId, aResolve);
-      });
-      let log = yield Services.logs.getLogFromFile(conv.logFile);
-      let logConv = yield log.getConversation();
-      let job = new IndexingJob("indexIMConversation", null);
-      job.conversation = conv;
-      job.log = log;
-      job.path = conv.logFile;
-      job.logConv = logConv;
-      GlodaIndexer.indexJob(job);
-      cache[fileName] = lastModifiedTime;
+      for (let logFile of conv.logFiles) {
+        let fileName = OS.Path.basename(logFile);
+        let fileInfo = yield OS.File.stat(logFile);
+        let lastModifiedTime = fileInfo.lastModificationDate.valueOf();
+        if (Object.prototype.hasOwnProperty.call(cache, fileName) &&
+            cache[fileName] == lastModifiedTime)
+          return;
+        if (this._indexingJobPromise)
+          yield this._indexingJobPromise;
+        this._indexingJobPromise = new Promise(aResolve => {
+          this._indexingJobCallbacks.set(convId, aResolve);
+        });
+        let log = yield Services.logs.getLogFromFile(logFile);
+        let logConv = yield log.getConversation();
+        let job = new IndexingJob("indexIMConversation", null);
+        job.conversation = conv;
+        job.log = log;
+        job.path = logFile;
+        job.logConv = logConv;
+        GlodaIndexer.indexJob(job);
+        cache[fileName] = lastModifiedTime;
+      }
     }.bind(this)).catch(Components.utils.reportError);
 
     // Now clear the job, so we can index in the future.
     this._knownConversations[convId].scheduledIndex = null;
   },
 
   observe: function logger_observe(aSubject, aTopic, aData) {
     if (aTopic == "new-ui-conversation") {