add tweets (really, really early draft)
authorDavid Ascher <david@mozillamessaging.com>
Tue, 20 Jan 2009 20:30:12 -0800
changeset 33 80d0ddcd7f58a224e87e9154ef51af4e60aec26b
parent 32 f841237f781476cf374dea72fccf5e390bcc2824
child 34 2ef3084bf0aee665f70421fc20ab90f787d4e19e
push id1
push userroot
push dateWed, 08 Apr 2009 01:46:05 +0000
add tweets (really, really early draft)
server/python/junius/bootstrap.py
server/python/junius/getmail.py
server/python/junius/gettweets.py
server/python/junius/model.py
--- a/server/python/junius/bootstrap.py
+++ b/server/python/junius/bootstrap.py
@@ -27,16 +27,33 @@ def setup_account(dbs):
     ssl = not (sslstr.strip().lower() in ['false', 'f', 'no', '0'])
     
     account = model.Account(
         kind='imap', host=host, port=int(portstr), ssl=ssl,
         username=username, password=password,
     )
     account.store(dbs.accounts)
 
+def setup_twitter_account(dbs):
+    # we want a file of the form:
+    #  username,password
+    # example:
+    #  davidascher,sekret
+    import os, os.path
+    configPath = os.path.join(os.environ['HOME'], ".junius_twitter")
+    f = open(configPath, 'r')
+    data = f.read()
+    f.close()
+    username, password = data.split(',')
+
+    account = model.Account(
+        kind='twitter', username=username, password=password,
+    )
+    account.store(dbs.accounts)
+
 def path_part_nuke(path, count):
     for i in range(count):
         path = os.path.dirname(path)
     return path
     
 
 FILES_DOC = 'files' #'_design/files'
 
@@ -75,13 +92,14 @@ def main():
     import sys
     if 'nuke' in sys.argv:
       print 'NUKING DATABASE!!!'
       model.nuke_db()
 
     dbs = model.fab_db(update_views='updateviews' in sys.argv)
 
     setup_account(dbs)
+    setup_twitter_account(dbs)
     install_client_files(dbs)
     
 
 if __name__ == '__main__':
     main()
--- a/server/python/junius/getmail.py
+++ b/server/python/junius/getmail.py
@@ -249,17 +249,18 @@ class JuniusAccount(object):
         return me
 
 class Grabber(object):
     def __init__(self, dbs):
         self.dbs = dbs
     
     def syncAccounts(self):
         for account in model.Account.all(self.dbs.accounts):
-            junius_account = JuniusAccount(self.dbs, account)
-            junius_account.sync()
+            if account.kind == 'imap':
+                junius_account = JuniusAccount(self.dbs, account)
+                junius_account.sync()
 
 if __name__ == '__main__':
     import os
     #acct = JuniusAccount('localhost', 8143, os.environ['USER'], 'pass')
     dbs = model.fab_db()
     grabber = Grabber(dbs)
     grabber.syncAccounts()
new file mode 100644
--- /dev/null
+++ b/server/python/junius/gettweets.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+import base64, datetime, email.utils
+import pprint
+
+import twitter
+
+import junius.model as model
+
+'''
+Fetch contacts, and tweets.
+'''
+
+
+class JuniusAccount(object):
+    def __init__(self, dbs, account_def):
+        self.dbs = dbs
+        self.account_def = account_def
+        self.twitter_account = twitter.Api(username=account_def.username, password=account_def.password) 
+
+    def create_account_if_necessary(self):
+        contacts = model.Contact.by_identity(self.dbs.contacts,
+                                             key=['twitter', self.account_def.username])
+        if len(contacts) == 0:
+            # the contact does't exist, create it
+            self.twitter_user = self.twitter_account.GetUser(self.account_def.username)
+            contact = model.Contact(
+                name=self.twitter_user.name,
+                screen_name=self.twitter_user.screen_name,
+                identities=[{'kind': 'twitter', 'value': self.twitter_user.id}]
+            )
+            contact.store(self.dbs.contacts)
+        self.author = contact
+    
+    def sync(self):
+        print '***** Fetching tweets'
+        
+        # -- find out about what message id's we already know about
+        # this is really just the most primitive synchronization logic
+        #  available!  but that's okay.
+        known_uids = set()
+        #startkey=0
+        #endkey=4000000000
+        #for row in model.Message.by_header_id(self.dbs.messages, startkey=startkey, endkey=endkey).rows:
+        ## XXX optimize this to only look for tweets from this user
+        for row in model.Message.by_header_id(self.dbs.messages).rows:
+            print row
+            known_uids.add(row.key)
+        
+        processed = 0
+        skipped = 0
+        for message in self.twitter_account.GetUserTimeline():
+            if message.id not in known_uids:
+                self.grok_message(self.author, message)
+                processed += 1
+            else:
+                skipped += 1
+        print '  processed', processed, 'skipped', skipped
+    
+    def grok_message(self, author, imsg):
+
+        cmsg = model.Message(
+            account_id=self.account_def.id,
+            storage_path='http://twitter.com/' + self.account_def.username,
+            storage_id=str(imsg.id),
+            #
+            conversation_id='http://twitter.com/' + self.account_def.username,
+            header_message_id=str(imsg.id),
+            references=[],
+            #
+            from_contact_id=str(author.id),
+            to_contact_ids=[],
+            cc_contact_ids=[],
+            involves_contact_ids=[str(author.id)],
+            #
+            date=datetime.datetime.utcfromtimestamp(imsg.created_at_in_seconds),
+            timestamp=int(imsg.created_at_in_seconds),
+            #
+            read=False,
+            #
+            headers={},
+            bodyPart={"data":imsg.text, "contentType":"text/plain"},
+            _attachments={}
+        )
+        
+        print self.dbs.messages
+
+        cmsg.store(self.dbs.messages)
+        
+
+class Grabber(object):
+    def __init__(self, dbs):
+        self.dbs = dbs
+    
+    def syncAccounts(self):
+        for account in model.Account.all(self.dbs.accounts):
+            if account.kind == 'twitter':
+                junius_account = JuniusAccount(self.dbs, account)
+                junius_account.create_account_if_necessary();
+                junius_account.sync()
+
+if __name__ == '__main__':
+    import os
+    dbs = model.fab_db()
+    grabber = Grabber(dbs)
+    grabber.syncAccounts()
--- a/server/python/junius/model.py
+++ b/server/python/junius/model.py
@@ -8,21 +8,21 @@ class WildField(schema.Field):
     def _to_python(self, value):
         return value
     
     def _to_json(self, value):
         return value
 
 class Account(schema.Document):
     kind = schema.TextField()
-    host = schema.TextField()
-    port = schema.IntegerField()
+    host = schema.TextField(default='')
+    port = schema.IntegerField(default=0)
     username = schema.TextField()
     password = schema.TextField()
-    ssl = schema.BooleanField()
+    ssl = schema.BooleanField(default=False)
     
     folderStatuses = WildField(default={})
     
     # could we just do _all_docs?  I don't want the damn design docs though...
     # (ironically, this is the first one :)
     all = schema.View('all', '''\
         function(doc) {
             emit(null, doc);