move imap back to using a view to optimize what we've seen before twisty
authorMark Hammond <mhammond@skippinet.com.au>
Wed, 25 Mar 2009 23:12:34 +1100
branchtwisty
changeset 123 ebc4050e4049b80671ed7aedc17177033c3f8e44
parent 122 6c69a445aee2a2da7ac9f65deaf819942217f47e
child 124 f161b6e3b54a301bda27cd684c8ebdea4c5213bb
push id1
push userroot
push dateWed, 08 Apr 2009 01:46:05 +0000
move imap back to using a view to optimize what we've seen before
schema/proto/imap/seen-map.js
server/python/junius/proto/imap.py
new file mode 100644
--- /dev/null
+++ b/schema/proto/imap/seen-map.js
@@ -0,0 +1,8 @@
+function(doc) {
+    if (doc.type=='proto/imap') {
+        // imap uses a storage_key field which is already [foldername, uid]
+        // we include [flags, _rev] as the value so the protocol impl can see
+        // if things have changed.
+        emit(doc.storage_key, [doc.imap_flags, doc._rev]);
+    }
+}
--- a/server/python/junius/proto/imap.py
+++ b/server/python/junius/proto/imap.py
@@ -74,61 +74,51 @@ class ImapClient(imap4.IMAP4Client):
       return self._process_next_folder()
 
     return self.examine(name
                  ).addCallback(self._examineFolder, name
                  ).addErrback(self._cantExamineFolder, name)
 
   def _examineFolder(self, result, folder_path):
     logger.debug('Looking for messages already fetched for folder %s', folder_path)
-    startkey=['rfc822', self.account.details['_id'], [folder_path, 0]]
-    endkey=['rfc822', self.account.details['_id'], [folder_path, 4000000000]]
-    return get_db().openView('raindrop!messages!by', 'by_storage',
-                             startkey=startkey, endkey=endkey,
+    return get_db().openView('raindrop!proto!imap', 'seen',
+                             startkey=[folder_path], endkey=[folder_path, {}]
               ).addCallback(self._fetchAndProcess, folder_path)
 
   def _fetchAndProcess(self, rows, folder_path):
+    # XXX - we should look at the flags and update the message if it's not
+    # the same - later.
+    seen_uids = set(row['key'][1] for row in rows)
+    # now build a list of all message currently in the folder.
     allMessages = imap4.MessageSet(1, None)
-    seen_ids = [r['key'][2][1] for r in rows]
-    logger.debug("%d messages exist in %s", len(seen_ids), folder_path)
     return self.fetchUID(allMessages, True).addCallback(
-            self._gotUIDs, folder_path, seen_ids)
+            self._gotUIDs, folder_path, seen_uids)
 
   def _gotUIDs(self, uidResults, name, seen_uids):
-    uids = [int(result['UID']) for result in uidResults.values()]
-    logger.info("Folder %s has %d messages", name, len(uids))
-    self.messages_remaining = uids
+    all_uids = set(int(result['UID']) for result in uidResults.values())
+    self.messages_remaining = all_uids - seen_uids
+    logger.info("Folder %s has %d messages, %d new", name,
+                len(all_uids), len(self.messages_remaining))
     return self._process_next_message()
 
   def _process_next_message(self):
     logger.debug("processNextMessage has %d messages to go...",
                  len(self.messages_remaining))
     if not self.messages_remaining:
       return self._process_next_folder()
 
     uid = self.messages_remaining.pop()
     # XXX - we need something to make this truly unique.
     did = "%s#%d" % (self.current_folder, uid)
-    logger.debug("seeing if imap message %r exists", did)
-    return self.doc_model.open_document(did,
-                    ).addCallback(self._cb_process_message, uid, did
-                    )
-
-  def _cb_process_message(self, existing_doc, uid, did):
-    if existing_doc is None:
-      logger.debug("new imap message %r - fetching content", did)
-      # grr - we have to get the rfc822 body and the flags in separate requests.
-      to_fetch = imap4.MessageSet(uid)
-      return self.fetchMessage(to_fetch, uid=True
-                  ).addCallback(self._cb_got_body, uid, did, to_fetch
-                  )
-    else:
-      logger.debug("Skipping message %r - already exists", did)
-      # we are done with this message.
-      return self._process_next_message()
+    logger.debug("new imap message %r - fetching content", did)
+    # grr - we have to get the rfc822 body and the flags in separate requests.
+    to_fetch = imap4.MessageSet(uid)
+    return self.fetchMessage(to_fetch, uid=True
+                ).addCallback(self._cb_got_body, uid, did, to_fetch
+                )
 
   def _cb_got_body(self, result, uid, did, to_fetch):
     _, result = result.popitem()
     content = result['RFC822']
     # grr - get the flags
     logger.debug("message %r has %d bytes; fetching flags", did, len(content))
     return self.fetchFlags(to_fetch, uid=True
                 ).addCallback(self._cb_got_flags, uid, did, content