twisty gets imap mail! twisty
authorMark Hammond <mhammond@skippinet.com.au>
Wed, 11 Mar 2009 22:35:20 +1100
branchtwisty
changeset 75 907284571c783f7bb01d973d9281a7a56e19714b
parent 74 5243f0a1ca92366edaca30e14ade7ce7a3a182e7
child 76 775e2092d7a912cdbf07483abddd376d050f767c
push id1
push userroot
push dateWed, 08 Apr 2009 01:46:05 +0000
twisty gets imap mail!
docs/INSTALL
schema/messages/by/views/by_storage/map.js
server/python/junius/bootstrap.py
server/python/junius/sync.py
--- a/docs/INSTALL
+++ b/docs/INSTALL
@@ -4,55 +4,59 @@ Have Python 2.5 or later installed.
 
  If not 2.6: 
     install setup_tools
     easy_install simplejson
 
  Prereqs:
     easy_install couchdb (0.5 or later)
 
-Install couchdb trunk
+Install couchdb trunk.  This currently means you need to google for
+instructions that work only on Linux - Windows doesn't work.  If you want to
+work on Windows you must install Couch on a Linux box and point your windows
+~/.raindrop file at the server.
 
-Pull asuth's version of gocept-imapapi:
-  hg clone http://clicky.visophyte.org/cgi-bin/hgwebdir.cgi/asuth/gocept-imapapi/
-  cd gocept-imapapi
-  python setup.py develop
+Install twisted.
 
-install junius:
+Install 'paisley' from launchpad.
+
+Either install junius:
   cd junius/server/python/junius
   python setup.py develop
+*or* just add it to your PYTHONPATH
+  set PYTHONPATH=c:\src\path\to\junius\server\python
 
 configure junuis:
   * edit ~/.raindrop
+
   * if your 'local' couchdb isn't on localhost, add a section along the lines of:
         [couch-local]
         host=hostname
         port=port
 
   * Add imap accounts along the lines of
-        [account-some-account-name]
+        [account-some-account-name] # the 'account-' prefix is important!
         type=imap
-        server=imap.gmail.com
+        host=imap.gmail.com
         port=993
         username=username@gmail.com
         password=topsecret
         ssl=True
 
 With Couchdb running:
 
 Setup the tables:
-  python bootstrap.py [nuke]
 
-  WARNING: using 'nuke' will wipe ALL your databases we may touch.
+  python run-raindrop.py --nuke-db-and-delete-everything-forever
+
+  WARNING: using this coommand will wipe ALL your databases we may touch.
   USE WITH CAUTION
 
 Go to http://127.0.0.1:5984/_utils/index.html to check that the table(s) are 
 setup
 
 Load the mail:
-  python getmail.py (loads IMAP data)
+  python run-raindrop.py sync_messages
 
 (reload http://127.0.0.1:5984/_utils/index.html to see stuff in the messages view)
 
-Go to http://127.0.0.1:5984/junius/files/index.xhtml
-
-and do autocomplete to do searches.
-
+Go to http://127.0.0.1:5984/junius/files/index.xhtml and do autocomplete to do
+searches.
--- a/schema/messages/by/views/by_storage/map.js
+++ b/schema/messages/by/views/by_storage/map.js
@@ -1,5 +1,5 @@
 function(doc) {
-  if (doc.type == 'message' && doc.subtype == "rfc822")
+  if ((doc.type == 'message' || doc.type== 'rawMessage') && doc.subtype == "rfc822")
     if (doc.storage_path && doc.storage_id)
       emit([doc.account_id, doc.storage_path, doc.storage_id], null);
-}
\ No newline at end of file
+}
--- a/server/python/junius/bootstrap.py
+++ b/server/python/junius/bootstrap.py
@@ -1,19 +1,21 @@
 #!/usr/bin/env python
 
 '''
 Setup the CouchDB server so that it is fully usable and what not.
 '''
 import sys
-from twisted.internet import reactor, defer
 import twisted.web.error
+from twisted.internet import defer
 import os, os.path, mimetypes, base64, pprint
 import model
 
+from junius.config import get_config
+
 import logging
 logger = logging.getLogger(__name__)
 
 def setup_account(dbs):
     if len(model.Account.all(dbs.accounts)):
         print 'Account presumably already exists, not adding it!'
         return
     
@@ -57,38 +59,39 @@ def path_part_nuke(path, count):
         path = os.path.dirname(path)
     return path
     
 
 FILES_DOC = 'files' #'_design/files'
 
 def install_client_files(whateva):
     '''
-    cram everyone in 'client' into the 'junius' app database
+    cram everyone in 'client' into the app database
     '''
     from model import get_db
     d = get_db()
 
     def _opened_ok(doc):
-        logger.info('Design doc already exists, will be updating/overwriting files')
+        logger.info("document '%(_id)s' already exists, will be updating/overwriting existing records",
+                    doc)
         return doc
 
     def _open_not_exists(failure, *args, **kw):
         failure.trap(twisted.web.error.Error)
         if failure.value.status != '404': # not found.
             failure.raiseException()
         return {} # return an empty doc.
 
     def _update_doc(design_doc):
         attachments = design_doc['_attachments'] = {}
         # we cannot go in a zipped egg...
         junius_root_dir = path_part_nuke(model.__file__, 4)
         client_dir = os.path.join(junius_root_dir, 'client')
-        logger.info("listing contents of '%s'", client_dir)
-        
+        logger.debug("listing contents of '%s' to look for client files", client_dir)
+
         for filename in os.listdir(client_dir):
             path = os.path.join(client_dir, filename)
             if os.path.isfile(path):
                 f = open(path, 'rb')
                 ct = mimetypes.guess_type(filename)[0]
                 if ct is None and sys.platform=="win32":
                     # A very simplistic check in the windows registry.
                     import _winreg
@@ -100,66 +103,52 @@ def install_client_files(whateva):
                         pass
                 assert ct, "can't guess the content type for '%s'" % filename
                 attachments[filename] = {
                     'content_type': ct,
                     'data': base64.b64encode(f.read())
                 }
                 f.close()
             logger.debug("filename '%s' (%s)", filename, ct)
-        return d.saveDoc('raindrop', design_doc, FILES_DOC)
+        return d.saveDoc(design_doc, FILES_DOC)
 
-    defrd = d.openDoc('raindrop', FILES_DOC) # XXX - why the db name??
+    defrd = d.openDoc(FILES_DOC)
     defrd.addCallbacks(_opened_ok, _open_not_exists)
     defrd.addCallback(_update_doc)
     return defrd
 
+def install_accounts(whateva):
+    from model import get_db
+    db = get_db()
+    config = get_config()
 
-def main():
-    import sys
+    def _opened_ok(doc):
+        logger.info("account '%(_id)s' already exists, will be updating existing account",
+                    doc)
+        return doc
+
+    def _open_not_exists(failure, doc_id, *args, **kw):
+        failure.trap(twisted.web.error.Error)
+        if failure.value.status != '404': # not found.
+            failure.raiseException()
+        return {'_id': doc_id} # return an empty doc for the account.
+
+    def _update_acct(doc, info):
+        logger.debug("updating %s with %s", doc, info)
+        doc.update(info)
+        doc['type'] = 'account'
+        return db.saveDoc(doc, doc['_id'])
+
+    def _open_doc(whateva, key):
+        return db.openDoc(key)
 
     d = defer.Deferred()
-    def mutter(whateva):
-        print "Raindrops keep falling on my head..."
-    d.addCallback(mutter)
-
-    if 'nuke' in sys.argv:
-        def do_nuke(whateva):
-            # returns a deferred.
-            return model.nuke_db()
-        d.addCallback(do_nuke)
-
-    def do_fab(whateva):
-        # returns a deferred.
-        return model.fab_db(update_views='updateviews' in sys.argv)
-
-    d.addCallback(do_fab)
-
-    d.addCallback(install_client_files)
-
-    #dbs = model.fab_db(update_views='updateviews' in sys.argv)
-    #
-    #setup_account(dbs)
-    #setup_twitter_account(dbs)
 
-    #install_client_files(dbs)
-
-    def done(whateva):
-        print "Finished."
-        reactor.stop()
-
-    def error(*args, **kw):
-        from twisted.python import log
-        log.err(*args, **kw)
-        print "FAILED."
-        reactor.stop()
+    for acct_name, acct_info in config.accounts.iteritems():
+        acct_id = acct_info['_id']
+        logger.info("Adding account '%s'", acct_id)
+        d.addCallback(_open_doc, acct_id)
+        d.addCallbacks(_opened_ok, _open_not_exists, errbackArgs=(acct_id,))
+        d.addCallback(_update_acct, acct_info)
 
-    d.addCallbacks(done, error)
-
-    reactor.callWhenRunning(d.callback, None)
-
-    logger.debug('starting reactor')
-    reactor.run()
-    logger.debug('reactor done')
-    
-
-if __name__ == '__main__':
-    main()
+    # Start the chain and return.
+    d.callback(None)
+    return d
--- a/server/python/junius/sync.py
+++ b/server/python/junius/sync.py
@@ -1,50 +1,72 @@
 import logging
 
 from twisted.internet import reactor, defer
 import paisley
 
 import junius.proto as proto
 from junius.model import get_db
 
+logger = logging.getLogger(__name__)
+
+_conductor = None
+def get_conductor():
+  global _conductor
+  if _conductor is None:
+    _conductor = SyncConductor()
+  return _conductor
+  
+  
 class SyncConductor(object):
   def __init__(self):
-    self.log = logging.getLogger('sync')
+    self.log = logger
 
     self.db = get_db()
 
     self.active_accounts = []
 
   def _ohNoes(self, failure, *args, **kwargs):
     self.log.error('OH NOES! failure! %s', failure)
     reactor.stop()
 
   def _getAllAccounts(self):
-    return self.db.openView('accounts', 'all'
+    return self.db.openView('raindrop!accounts!all', 'all'
       ).addCallback(self._gotAllAccounts
       ).addErrback(self._ohNoes)
 
   def _gotAllAccounts(self, rows, *args, **kwargs):
     self.log.info("Have %d accounts to synch", len(rows))
     for row in rows:
       account_details = row['value']
-      if account_details['kind'] in proto.protocols:
-        account = proto.protocols[account['kind']](self.db, account_details)
-        log.info('Starting sync of %s account: %s',
-                 account_details['kind'],
-                 account_details.get('name', '(un-named)'))
+      kind = account_details['kind']
+      if kind in proto.protocols:
+        account = proto.protocols[kind](self.db, account_details)
+        self.log.info('Starting sync of %s account: %s',
+                      account_details['kind'],
+                      account_details.get('name', '(un-named)'))
         account.startSync(self)
         self.active_accounts.append(account)
       else:
         log.error("Don't know what to do with account kind: %s",
                   account_details['kind'])
 
-  def sync(self):
-    reactor.callWhenRunning(self._getAllAccounts)
-    self.log.debug('starting reactor')
-    reactor.run()
-    self.log.debug('reactor done')
+  def accountFinishedSync(self, account):
+    self.active_accounts.remove(account)
+    if not self.active_accounts:
+      self.log.info("sync has finished; stopping reactor")
+      return reactor.stop()
+
+  def sync(self, whateva=None):
+    return self._getAllAccounts()
 
 
 if __name__ == '__main__':
-  conductor = SyncConductor()
-  conductor.sync()
+  # normal entry-point is the app itself; this is purely for debugging...
+  logging.basicConfig()
+  logging.getLogger().setLevel(logging.DEBUG)
+
+  conductor = get_conductor()
+  reactor.callWhenRunning(conductor.sync)
+
+  logger.debug('starting reactor')
+  reactor.run()
+  logger.debug('reactor done')