Bug 16913 - Filter on any header, for news. r=bienvenu,Standard8 sr=Neil
authorJoshua Cranmer <Pidgeot18@gmail.com>
Mon, 28 Jul 2008 12:17:26 -0700
changeset 40 23da5cab793a16a758f400d6f76406ca46987c2d
parent 39 6e9c4ef897066c0fa97e6d7f8fde11bee064d7f7
child 41 4cb9ccd2a1ca2a88d630c01adc67517615cfdaf0
push idunknown
push userunknown
push dateunknown
reviewersbienvenu, Standard8, Neil
bugs16913
Bug 16913 - Filter on any header, for news. r=bienvenu,Standard8 sr=Neil
mail/locales/en-US/chrome/messenger/news.properties
mailnews/base/search/src/nsMsgSearchAdapter.cpp
mailnews/base/search/src/nsMsgSearchNews.cpp
mailnews/db/msgdb/src/nsMsgDatabase.cpp
mailnews/news/public/nsINNTPNewsgroupList.idl
mailnews/news/src/nsNNTPNewsgroupList.cpp
mailnews/news/src/nsNNTPNewsgroupList.h
mailnews/news/src/nsNNTPProtocol.cpp
mailnews/news/src/nsNNTPProtocol.h
mailnews/news/test/postings/auto-add/post1.eml
mailnews/news/test/postings/auto-add/post2.eml
mailnews/news/test/postings/auto-add/post3.eml
mailnews/news/test/postings/auto-add/post4.eml
mailnews/news/test/postings/auto-add/post5.eml
mailnews/news/test/postings/auto-add/post6.eml
mailnews/news/test/postings/auto-add/post7.eml
mailnews/news/test/unit/head_server_setup.js
mailnews/news/test/unit/test_filter.js
mailnews/news/test/unit/test_server.js
mailnews/test/fakeserver/nntpd.js
suite/locales/en-US/chrome/mailnews/news.properties
--- a/mail/locales/en-US/chrome/messenger/news.properties
+++ b/mail/locales/en-US/chrome/messenger/news.properties
@@ -48,16 +48,17 @@ cancelConfirm=Are you sure you want to c
 messageCancelled=Message cancelled.
 enterUsername=Please enter a username for news server access:
 enterPassword=Please enter a password for news server access:
 enterPasswordTitle=News Server Password Required
 okButtonText=Download
 
 noNewMessages=There are no new messages on the server.
 downloadingHeaders=Downloading %S of %S headers
+downloadingFilterHeaders=Getting headers for filters: %S (%S/%S)
 downloadingArticles=Downloading articles %S-%S
 bytesReceived=Downloading newsgroups: %S received (%SKB read at %SKB/sec)
 checkingForNewNews=Checking newsgroup %S of %S on %S for new messages
 downloadingArticlesForOffline=Downloading articles %S-%S in %S
 
 onlyCancelOneMessage=You can only cancel one article at a time.
 
 # LOCALIZATION NOTE (autoUnsubscribeText): %1$S is the newsgroup and %2$S is the newsgroup-server it is being removed from.
--- a/mailnews/base/search/src/nsMsgSearchAdapter.cpp
+++ b/mailnews/base/search/src/nsMsgSearchAdapter.cpp
@@ -1010,21 +1010,25 @@ NS_IMETHODIMP nsMsgSearchValidityManager
       rv = InitOnlineMailFilterTable ();
     if (m_onlineMailFilterTable)
       rv = SetOtherHeadersInTable(m_onlineMailFilterTable, customHeaders.get());
     *ppOutTable = m_onlineMailFilterTable;
     break;
   case nsMsgSearchScope::news:
     if (!m_newsTable)
       rv = InitNewsTable();
+    if (m_newsTable)
+      rv = SetOtherHeadersInTable(m_newsTable, customHeaders.get());
     *ppOutTable = m_newsTable;
     break;
   case nsMsgSearchScope::newsFilter:
     if (!m_newsFilterTable)
       rv = InitNewsFilterTable();
+    if (m_newsFilterTable)
+      rv = SetOtherHeadersInTable(m_newsFilterTable, customHeaders.get());
     *ppOutTable = m_newsFilterTable;
     break;
   case nsMsgSearchScope::localNews:
     if (!m_localNewsTable)
       rv = InitLocalNewsTable();
     if (m_localNewsTable)
       rv = SetOtherHeadersInTable(m_localNewsTable, customHeaders.get());
     *ppOutTable = m_localNewsTable;
--- a/mailnews/base/search/src/nsMsgSearchNews.cpp
+++ b/mailnews/base/search/src/nsMsgSearchNews.cpp
@@ -140,18 +140,24 @@ char *nsMsgSearchNews::EncodeTerm (nsIMs
   {
   case nsMsgSearchAttrib::Sender:
     attribEncoding = m_kNntpFrom;
     break;
   case nsMsgSearchAttrib::Subject:
     attribEncoding = m_kNntpSubject;
     break;
   default:
-    NS_ASSERTION(PR_FALSE,"malformed search"); // malformed search term?
-    return nsnull;
+    nsCString header;
+    term->GetArbitraryHeader(header);
+    if (header.IsEmpty())
+    {
+      NS_ASSERTION(PR_FALSE,"malformed search"); // malformed search term?
+      return nsnull;
+    }
+    attribEncoding = header.get();
   }
 
   // Build a string to represent the string pattern
   PRBool leadingStar = PR_FALSE;
   PRBool trailingStar = PR_FALSE;
   int overhead = 1; // null terminator
   nsMsgSearchOpValue op;
   term->GetOp(&op);
@@ -456,16 +462,33 @@ nsresult nsMsgSearchValidityManager::Ini
     m_newsTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Contains, 1);
     m_newsTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Contains, 1);
     m_newsTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Is, 1);
     m_newsTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Is, 1);
     m_newsTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::BeginsWith, 1);
     m_newsTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::BeginsWith, 1);
     m_newsTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::EndsWith, 1);
     m_newsTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::EndsWith, 1);
+
+#if 0
+    // Size should be handled after the fact...
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsGreaterThan, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsGreaterThan, 1);
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsLessThan, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsLessThan, 1);
+#endif
+    
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+    m_newsTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
+    m_newsTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
   }
 
   return rv;
 }
 
 nsresult nsMsgSearchValidityManager::InitNewsFilterTable()
 {
   NS_ASSERTION (nsnull == m_newsFilterTable, "news filter table already initted");
@@ -507,12 +530,26 @@ nsresult nsMsgSearchValidityManager::Ini
     m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsBefore, 1);
     m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsBefore, 1);
     m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsAfter, 1);
     m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsAfter, 1);
     m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::Is, 1);
     m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::Is, 1);
     m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::Isnt, 1);
     m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::Isnt, 1);
+  
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsGreaterThan, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsGreaterThan, 1);
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsLessThan, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::Size, nsMsgSearchOp::IsLessThan, 1);
+
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+    m_newsFilterTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
+    m_newsFilterTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
   }
 
   return rv;
 }
--- a/mailnews/db/msgdb/src/nsMsgDatabase.cpp
+++ b/mailnews/db/msgdb/src/nsMsgDatabase.cpp
@@ -2322,24 +2322,28 @@ NS_IMETHODIMP nsMsgDatabase::MarkHdrRead
   NS_ENSURE_SUCCESS(rv, rv);
 
   // if the flag is already correct in the db, don't change it.
   // Check msg flags as well as IsHeaderRead in case it's a newsgroup
   // and the msghdr flags are out of sync with the newsrc settings.
   // (we could override this method for news db's, but it's a trivial fix here.
   if (bRead != isRead || isRead != isReadInDB)
   {
-    nsCOMPtr <nsIMsgThread> threadHdr;
     nsMsgKey msgKey;
     msgHdr->GetMessageKey(&msgKey);
-
-    rv = GetThreadForMsgKey(msgKey, getter_AddRefs(threadHdr));
-    if (threadHdr)
+    
+    PRBool inDB = PR_FALSE;
+    (void)ContainsKey(msgKey, &inDB);
+
+    if (inDB)
     {
-      threadHdr->MarkChildRead(bRead);
+      nsCOMPtr <nsIMsgThread> threadHdr;
+      rv = GetThreadForMsgKey(msgKey, getter_AddRefs(threadHdr));
+      if (threadHdr)
+        threadHdr->MarkChildRead(bRead);
     }
     return MarkHdrReadInDB(msgHdr, bRead, instigator);
   }
   return NS_OK;
 }
 
 NS_IMETHODIMP nsMsgDatabase::MarkHdrReplied(nsIMsgDBHdr *msgHdr, PRBool bReplied,
                          nsIDBChangeListener *instigator)
--- a/mailnews/news/public/nsINNTPNewsgroupList.idl
+++ b/mailnews/news/public/nsINNTPNewsgroupList.idl
@@ -15,69 +15,112 @@
  * The Original Code is mozilla.org code.
  *
  * The Initial Developer of the Original Code is
  * Netscape Communications Corporation.
  * Portions created by the Initial Developer are Copyright (C) 1998
  * the Initial Developer. All Rights Reserved.
  *
  * Contributor(s):
+ *   Joshua Cranmer <Pidgeot18@gmail.com>
  *
  * Alternatively, the contents of this file may be used under the terms of
  * either of the GNU General Public License Version 2 or later (the "GPL"),
  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  * in which case the provisions of the GPL or the LGPL are applicable instead
  * of those above. If you wish to allow use of your version of this file only
  * under the terms of either the GPL or the LGPL, and not to allow others to
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
-/* this interface is basically for the old ListNewsGroupState class
- * (the implementation of this class probably wants to also implement
- * or contain ChangeListener so that it can react to OnAnnouncerGoingAway()
- * to destroy the DBView)
- */
 #include "nsISupports.idl"
 
 interface nsIMsgNewsFolder;
 interface nsINntpUrl;
 interface nsIMsgWindow;
 
+/**
+ * A utility class for nsINNTPProtocol that handles the list of new headers.
+ */
 [scriptable, uuid(E628ED19-9452-11d2-B7EA-00805F05FFA5)]
 interface nsINNTPNewsgroupList : nsISupports {
   
   void initialize(in nsINntpUrl runningURL, in nsIMsgNewsFolder newsFolder);
   
   long getRangeOfArtsToDownload(in nsIMsgWindow aMsgWindow, in long first_message,
                                 in long last_message,
                                 in long maxextra,
                                 out long real_first_message,
                                 out long real_last_message);
   
   void addToKnownArticles(in long first_message, in long last_message);
 
-
-  /* The NNTP module of netlib calls these to feed XOVER data to the message
-     library, in response to a news:group.name URL having been opened.
-     If MSG_FinishXOVER() returns a message ID, that message will be loaded
-     next (used for selecting the first unread message in a group after
-     listing that group.)
-
-     deprecated:
-     The "out" arguments are (if non-NULL) a file descriptor to write the XOVER
-     line to, followed by a "\n".  This is used by the XOVER-caching code.
-  */
-  /* the XOVER Sink should maintain the ListNewsGroupState */
+  /**
+   * Initializes the internal state to get the messages.
+   *
+   * This method should be called before sending the line
+   * <tt>XOVER @arg first_message-@arg last_message</tt> to the server.
+   *
+   * @param first_message  The first message of the download range.
+   * @param last_message   The last message of the download range.
+   */
   void initXOVER(in long first_message, in long last_message);  
   void processXOVERLINE(in string line, out unsigned long status);
-  void processNonXOVER(in string line);
   void resetXOVER();
   void finishXOVERLINE(in long status, out long newstatus);
-  void clearXOVERState();
+
+  /**
+   * Initalizes the state in preparation for a call to XHDR.
+   *
+   * @return               The next header to get, or an empty string if done.
+   */
+  ACString initXHDR();
+  /**
+   * Processes a line of the server's response to XHDR.
+   *
+   * It will calculate the message number and other information itself, so the
+   * unadulterated line itself should be sent.
+   *
+   * @param aLine          The line as sent from the server.
+   */
+  void processXHDRLine(in ACString aLine);
+
+  /**
+   * Initalizes the internal state to process a HEAD command.
+   *
+   * This method should be called before sending the line
+   * <tt>HEAD @arg aMessage</tt> to the server.
+   *
+   * @param aMessage       The message number that will be sent.
+   */
+  void initHEAD(in long aMessage);
+  /**
+   * Processes a line of the server's response to HEAD.
+   *
+   * This will not check for a quoted '.' at the beginning.
+   *
+   * @param aLine          The line the server sent.
+   */
+  void processHEADLine(in ACString aLine);
+  /**
+   * Manages the internal state if the call to HEAD failed.
+   *
+   * @param aMessage       The message key that caused the HEAD failure.
+   */
+  void HEADFailed(in long aMessage);
+
+  /**
+   * Calls the filters after all messages have been processed.
+   *
+   * This method also cleans out some internal state relating to the messages
+   * that have been processed, so it should always be called at the end of
+   * XOVER/XHDR/HEAD processing.
+   */
+  void callFilters();
 
   attribute boolean getOldMessages;
 };
 
--- a/mailnews/news/src/nsNNTPNewsgroupList.cpp
+++ b/mailnews/news/src/nsNNTPNewsgroupList.cpp
@@ -123,16 +123,45 @@ NS_IMPL_ISUPPORTS2(nsNNTPNewsgroupList, 
 
 nsresult
 nsNNTPNewsgroupList::Initialize(nsINntpUrl *runningURL, nsIMsgNewsFolder *newsFolder)
 {
   m_newsFolder = newsFolder;
   m_runningURL = runningURL;
   m_knownArts.set = nsMsgKeySet::Create();
 
+  nsresult rv;
+
+  nsCOMPtr <nsIMsgFolder> folder = do_QueryInterface(m_newsFolder, &rv);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  rv = folder->GetFilterList(m_msgWindow, getter_AddRefs(m_filterList));
+  NS_ENSURE_SUCCESS(rv,rv);
+  nsCString ngHeaders;
+  m_filterList->GetArbitraryHeaders(ngHeaders);
+  m_filterHeaders.ParseString(ngHeaders.get(), " ");
+
+  nsCOMPtr<nsIMsgIncomingServer> server;
+  rv = folder->GetServer(getter_AddRefs(server));
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  rv = server->GetFilterList(m_msgWindow, getter_AddRefs(m_serverFilterList));
+  NS_ENSURE_SUCCESS(rv,rv);
+  nsCAutoString servHeaders;
+  m_serverFilterList->GetArbitraryHeaders(servHeaders);
+
+  nsCStringArray servArray;
+  servArray.ParseString(servHeaders.get(), " ");
+
+  // servArray may have duplicates already in m_filterHeaders.
+  for (PRInt32 i = 0; i < servArray.Count(); i++)
+  {
+    if (m_filterHeaders.IndexOf(*(servArray[i])) == -1)
+      m_filterHeaders.AppendCString(*(servArray[i]));
+  }
   return NS_OK;
 }
 
 nsresult
 nsNNTPNewsgroupList::CleanUp()
 {
   // here we make sure that there aren't missing articles in the unread set
   // So if an article is the unread set, and the known arts set, but isn't in the
@@ -151,16 +180,18 @@ nsNNTPNewsgroupList::CleanUp()
         folderInfo->GetUint32Property("lastMissingCheck", 0, &lastMissingCheck);
         if (lastMissingCheck)
           firstKnown = lastMissingCheck + 1;
       }
       PRBool foundMissingArticle = PR_FALSE;
       while (firstKnown <= lastKnown)
       {
         PRInt32 firstUnreadStart, firstUnreadEnd;
+        if (firstKnown == 0)
+          firstKnown = 1;
         m_set->FirstMissingRange(firstKnown, lastKnown, &firstUnreadStart, &firstUnreadEnd);
         if (firstUnreadStart)
         {
           while (firstUnreadStart <= firstUnreadEnd)
           {
             PRBool containsKey;
             m_newsDB->ContainsKey(firstUnreadStart, &containsKey);
             if (!containsKey)
@@ -468,41 +499,36 @@ nsNNTPNewsgroupList::AddToKnownArticles(
     }
   }
   return status;
 }
 
 nsresult
 nsNNTPNewsgroupList::InitXOVER(PRInt32 first_msg, PRInt32 last_msg)
 {
-  int status = 0;
-
-  // Tell the FE to show the GetNewMessages progress dialog
-#ifdef HAVE_PANES
-  FE_PaneChanged (m_pane, PR_FALSE, MSG_PanePastPasswordCheck, 0);
-#endif
   /* Consistency checks, not that I know what to do if it fails (it will
    probably handle it OK...) */
   NS_ASSERTION(first_msg <= last_msg, "first > last");
 
   /* If any XOVER lines from the last time failed to come in, mark those
      messages as read. */
   if (m_lastProcessedNumber < m_lastMsgNumber)
   {
     m_set->AddRange(m_lastProcessedNumber + 1, m_lastMsgNumber);
   }
   m_firstMsgNumber = first_msg;
   m_lastMsgNumber = last_msg;
   m_lastProcessedNumber = first_msg > 1 ? first_msg - 1 : 1;
-  return status;
+  m_currentXHDRIndex = -1;
+  return NS_OK;
 }
 
 // from RFC 822, don't translate
 #define FROM_HEADER "From: "
-#define SUBECT_HEADER "Subject: "
+#define SUBJECT_HEADER "Subject: "
 #define DATE_HEADER "Date: "
 
 nsresult
 nsNNTPNewsgroupList::ParseLine(char *line, PRUint32 * message_number)
 {
   nsresult rv = NS_OK;
   nsCOMPtr <nsIMsgDBHdr> newMsgHdr;
   char *dateStr = nsnull;  // keep track of date str, for filters
@@ -536,22 +562,24 @@ nsNNTPNewsgroupList::ParseLine(char *lin
     const char *subject = line;  /* #### const evilness */
     PRUint32 subjectLen = strlen(line);
 
     PRUint32 flags = 0;
     // ### should call IsHeaderRead here...
     /* strip "Re: " */
     nsCString modifiedSubject;
     if (NS_MsgStripRE(&subject, &subjectLen, getter_Copies(modifiedSubject)))
-      (void) newMsgHdr->OrFlags(MSG_FLAG_HAS_RE, &flags); // this will make sure read flags agree with newsrc
-
+      (void) newMsgHdr->OrFlags(MSG_FLAG_HAS_RE, &flags);
+    
+    // this will make sure read flags agree with newsrc
     if (! (flags & MSG_FLAG_READ))
       rv = newMsgHdr->OrFlags(MSG_FLAG_NEW, &flags);
 
     rv = newMsgHdr->SetSubject(modifiedSubject.IsEmpty() ? subject : modifiedSubject.get());
+
     if (NS_FAILED(rv))
       return rv;
   }
 
   GET_TOKEN (); /* author */
   if (line) {
     authorStr = line;
     rv = newMsgHdr->SetAuthor(line);
@@ -607,145 +635,17 @@ nsNNTPNewsgroupList::ParseLine(char *lin
     PRUint32 numLines = 0;
     numLines = line ? atol (line) : 0;
     rv = newMsgHdr->SetLineCount(numLines);
     if (NS_FAILED(rv)) return rv;
   }
 
   GET_TOKEN (); /* xref */
 
-  // apply filters
-  // XXX TODO
-  // do spam classification for news
-
-  nsCOMPtr <nsIMsgFolder> folder = do_QueryInterface(m_newsFolder, &rv);
-  NS_ENSURE_SUCCESS(rv,rv);
-
-  if (!m_filterList)
-  {
-    rv = folder->GetFilterList(m_msgWindow, getter_AddRefs(m_filterList));
-    NS_ENSURE_SUCCESS(rv,rv);
-  }
-
-  if (!m_serverFilterList)
-  {
-    nsCOMPtr<nsIMsgIncomingServer> server;
-    rv = folder->GetServer(getter_AddRefs(server));
-    NS_ENSURE_SUCCESS(rv,rv);
-
-    rv = server->GetFilterList(m_msgWindow, getter_AddRefs(m_serverFilterList));
-    NS_ENSURE_SUCCESS(rv,rv);
-  }
-
-  // flag for kill
-  // if the action is Delete, and we get a hit (see ApplyFilterHit())
-  // we set this to PR_FALSE.  if false, we won't add it to the db.
-  m_addHdrToDB = PR_TRUE;
-
-  PRUint32 filterCount = 0;
-  if (m_filterList) {
-    rv = m_filterList->GetFilterCount(&filterCount);
-    NS_ENSURE_SUCCESS(rv,rv);
-  }
-
-  PRUint32 serverFilterCount = 0;
-  if (m_serverFilterList) {
-    rv = m_serverFilterList->GetFilterCount(&serverFilterCount);
-    NS_ENSURE_SUCCESS(rv,rv);
-  }
-
-  // only do this if we have filters
-  if (filterCount || serverFilterCount)
-  {
-    // build up a "headers" for filter code
-    nsCString subject;
-    rv = newMsgHdr->GetSubject(getter_Copies(subject));
-    NS_ENSURE_SUCCESS(rv,rv);
-
-    PRUint32 headersSize = 0;
-
-    // +1 to separate headers with a null byte
-    if (authorStr)
-      headersSize += strlen(FROM_HEADER) + strlen(authorStr) + 1;
-
-    if (!(subject.IsEmpty()))
-      headersSize += strlen(SUBECT_HEADER) + subject.Length() + 1;
-
-    if (dateStr)
-     headersSize += strlen(DATE_HEADER) + strlen(dateStr) + 1;
-
-    if (headersSize) {
-      char *headers = (char *)PR_Malloc(headersSize);
-      char *headerPos = headers;
-      if (!headers)
-        return NS_ERROR_OUT_OF_MEMORY;
-
-      if (authorStr) {
-        PL_strcpy(headerPos, FROM_HEADER);
-        headerPos += strlen(FROM_HEADER);
-
-        PL_strcpy(headerPos, authorStr);
-        headerPos += strlen(authorStr);
-
-        *headerPos = '\0';
-        headerPos++;
-      }
-
-      if (!(subject.IsEmpty())) {
-        PL_strcpy(headerPos, SUBECT_HEADER);
-        headerPos += strlen(SUBECT_HEADER);
-
-        PL_strcpy(headerPos, subject.get());
-        headerPos += subject.Length();
-
-        *headerPos = '\0';
-        headerPos++;
-      }
-
-      if (dateStr) {
-        PL_strcpy(headerPos, DATE_HEADER);
-        headerPos += strlen(DATE_HEADER);
-
-        PL_strcpy(headerPos, dateStr);
-        headerPos += strlen(dateStr);
-
-        *headerPos = '\0';
-        headerPos++;
-      }
-
-      // on a filter hit (see ApplyFilterHit()), we'll be modifying the header
-      // so keep track of the header
-      m_newMsgHdr = newMsgHdr;
-
-      // the per-newsgroup filters should probably go first. It doesn't matter
-      // right now since nothing stops filter execution for newsgroups, but if something
-      // does, like adding a "stop execution" action, then users should be able to
-      // override the global filters in the per-newsgroup filters.
-      if (filterCount)
-      {
-        rv = m_filterList->ApplyFiltersToHdr(nsMsgFilterType::NewsRule, newMsgHdr, folder, m_newsDB,
-          headers, headersSize, this, m_msgWindow, nsnull);
-      }
-      if (serverFilterCount)
-      {
-        rv = m_serverFilterList->ApplyFiltersToHdr(nsMsgFilterType::NewsRule, newMsgHdr, folder, m_newsDB,
-          headers, headersSize, this, m_msgWindow, nsnull);
-      }
-
-      PR_Free ((void*) headers);
-      NS_ENSURE_SUCCESS(rv,rv);
-    }
-  }
-
-  // if we deleted it, don't add it
-  if (m_addHdrToDB) {
-    rv = m_newsDB->AddNewHdrToDB(newMsgHdr, PR_TRUE);
-    NS_ENSURE_SUCCESS(rv,rv);
-  }
-
+  m_newHeaders.AppendObject(newMsgHdr);
   return NS_OK;
 }
 
 NS_IMETHODIMP nsNNTPNewsgroupList::ApplyFilterHit(nsIMsgFilter *aFilter, nsIMsgWindow *aMsgWindow, PRBool *aApplyMore)
 {
   NS_ENSURE_ARG_POINTER(aFilter);
   NS_ENSURE_ARG_POINTER(aApplyMore);
   NS_ENSURE_TRUE(m_newMsgHdr, NS_ERROR_UNEXPECTED);
@@ -917,86 +817,32 @@ nsNNTPNewsgroupList::ProcessXOVERLINE(co
   /* Update the progress meter with a percentage of articles retrieved */
   if (m_lastMsgNumber > m_firstMsgNumber)
   {
     PRInt32 totalToDownload = m_lastMsgToDownload - m_firstMsgToDownload + 1;
     PRInt32 lastIndex = m_lastProcessedNumber - m_firstMsgNumber + 1;
     PRInt32 numDownloaded = lastIndex;
     PRInt32 totIndex = m_lastMsgNumber - m_firstMsgNumber + 1;
 
-    PRInt32  percent = (totIndex) ? (PRInt32)(100.0 * (double)numDownloaded / (double)totalToDownload) : 0;
-
-    PRTime elapsedTime;
-
-    LL_SUB(elapsedTime, PR_Now(), m_lastStatusUpdate);
-
-    if (LL_CMP(elapsedTime, >, MIN_STATUS_UPDATE_INTERVAL) ||
-        lastIndex == totIndex)
-    {
-      nsAutoString numDownloadedStr;
-      numDownloadedStr.AppendInt(numDownloaded);
-
-      nsAutoString totalToDownloadStr;
-      totalToDownloadStr.AppendInt(totalToDownload);
-
-      nsString statusString;
-      nsCOMPtr<nsIStringBundleService> bundleService = do_GetService(NS_STRINGBUNDLE_CONTRACTID, &rv);
-      NS_ENSURE_SUCCESS(rv, rv);
+    PRTime elapsedTime = PR_Now() - m_lastStatusUpdate;
 
-      nsCOMPtr<nsIStringBundle> bundle;
-      rv = bundleService->CreateBundle(NEWS_MSGS_URL, getter_AddRefs(bundle));
-      NS_ENSURE_SUCCESS(rv, rv);
-
-      const PRUnichar *formatStrings[2] = { numDownloadedStr.get(), totalToDownloadStr.get() };
-      rv = bundle->FormatStringFromName(NS_LITERAL_STRING("downloadingHeaders").get(), formatStrings, 2, getter_Copies(statusString));
-      NS_ENSURE_SUCCESS(rv, rv);
-
-#ifdef DEBUG_NEWS
-      PRInt32 elapsed;
-      LL_L2I(elapsed, elapsedTime);
-      printf("usecs elapsed since last update: %d\n", elapsed);
-#endif
-
-      SetProgressStatus(statusString.get());
-      m_lastStatusUpdate = PR_Now();
-
-      // only update the progress meter if it has changed
-      if (percent != m_lastPercent) {
-        SetProgressBarPercent(percent);
-        m_lastPercent = percent;
-      }
-    }
+    if (elapsedTime > MIN_STATUS_UPDATE_INTERVAL || lastIndex == totIndex)
+      UpdateStatus(PR_FALSE, numDownloaded, totalToDownload);
   }
   return NS_OK;
 }
 
 nsresult
 nsNNTPNewsgroupList::ResetXOVER()
 {
   m_lastMsgNumber = m_firstMsgNumber;
   m_lastProcessedNumber = m_lastMsgNumber;
   return 0;
 }
 
-/* When we don't have XOVER, but use HEAD, this is called instead.
-   It reads lines until it has a whole header block, then parses the
-   headers; then takes selected headers and creates an XOVER line
-   from them.  This is more for simplicity and code sharing than
-   anything else; it means we end up parsing some things twice.
-   But if we don't have XOVER, things are going to be so horribly
-   slow anyway that this just doesn't matter.
- */
-
-nsresult
-nsNNTPNewsgroupList::ProcessNonXOVER (const char * /*line*/)
-{
-  // ### dmb write me
-  return NS_ERROR_NOT_IMPLEMENTED;
-}
-
 nsresult
 nsNNTPNewsgroupList::FinishXOVERLINE(int status, int *newstatus)
 {
   nsresult rv;
   struct MSG_NewsKnown* k;
 
   /* If any XOVER lines from the last time failed to come in, mark those
      messages as read. */
@@ -1056,20 +902,304 @@ nsNNTPNewsgroupList::FinishXOVERLINE(int
   }
 
   if (newstatus)
     *newstatus=0;
 
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsNNTPNewsgroupList::InitXHDR(nsACString &header)
+{
+  if (++m_currentXHDRIndex >= m_filterHeaders.Count())
+    header.Truncate();
+  else
+    header.Assign(*m_filterHeaders[m_currentXHDRIndex]);
+  // Don't include these in our XHDR bouts, as they are already provided through
+  // XOVER. 
+  if (header.EqualsLiteral("message-id") ||
+      header.EqualsLiteral("references"))
+    return InitXHDR(header);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNNTPNewsgroupList::ProcessXHDRLine(const nsACString &line)
+{
+  PRInt32 middle = line.FindChar(' ');
+  nsCString value, key = PromiseFlatCString(line);
+  if (middle == -1)
+    return NS_OK;
+  value = Substring(line, middle+1);
+  key.Truncate((PRUint32)middle);
+
+  // According to RFC 2980, some will send (none) instead.
+  // So we don't treat this is an error.
+  if (key.CharAt(0) < '0' || key.CharAt(0) > '9')
+    return NS_OK;
+
+  PRInt32 code;
+  PRInt32 number = key.ToInteger(&code);
+  if (code != NS_OK)
+    return NS_ERROR_FAILURE;
+  // RFC 2980 specifies one or more spaces.
+  value.Trim(" ");
+
+  nsCOMPtr <nsIMsgDBHdr> header;
+  nsresult rv = m_newsDB->GetMsgHdrForKey(number, getter_AddRefs(header));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = header->SetStringProperty(m_filterHeaders[m_currentXHDRIndex]->get(), value.get());
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  PRInt32 totalToDownload = m_lastMsgToDownload - m_firstMsgToDownload + 1;
+  PRInt32 numDownloaded = number - m_firstMsgNumber + 1;
+
+  PRTime elapsedTime = PR_Now() - m_lastStatusUpdate;
+
+  if (elapsedTime > MIN_STATUS_UPDATE_INTERVAL)
+    UpdateStatus(PR_TRUE, numDownloaded, totalToDownload);
+  return rv;
+}
+
+NS_IMETHODIMP
+nsNNTPNewsgroupList::InitHEAD(PRInt32 number)
+{
+  if (m_newMsgHdr)
+  {
+    // Finish processing for this header
+    // If HEAD didn't properly return, then the header won't be set
+    m_newHeaders.AppendObject(m_newMsgHdr);
+
+    PRInt32 totalToDownload = m_lastMsgToDownload - m_firstMsgToDownload + 1;
+    PRInt32 lastIndex = m_lastProcessedNumber - m_firstMsgNumber + 1;
+    PRInt32 numDownloaded = lastIndex;
+    PRInt32 totIndex = m_lastMsgNumber - m_firstMsgNumber + 1;
+
+    PRTime elapsedTime = PR_Now() - m_lastStatusUpdate;
+
+    if (elapsedTime > MIN_STATUS_UPDATE_INTERVAL || lastIndex == totIndex)
+      UpdateStatus(PR_FALSE, numDownloaded, totalToDownload);
+  }
+
+  if (number >= 0)
+  {
+    if (m_newHeaders.Count() > 0 && m_lastMsgNumber == m_lastProcessedNumber)
+    {
+      // We have done some processing of messages. This means that we have
+      // relics of headers from XOVER. Since we will get everything from HEAD
+      // anyways, just clear the array.
+      m_newHeaders.Clear();
+    }
+
+    nsresult rv = m_newsDB->CreateNewHdr(number, getter_AddRefs(m_newMsgHdr));
+    m_lastProcessedNumber = number;
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+  else
+  {
+    AddToKnownArticles(m_firstMsgNumber, m_lastProcessedNumber);
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNNTPNewsgroupList::HEADFailed(PRInt32 number)
+{
+  m_set->Add(number);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNNTPNewsgroupList::ProcessHEADLine(const nsACString &line)
+{
+  PRInt32 colon = line.FindChar(':');
+  nsCString header = PromiseFlatCString(line), value;
+  if (colon != -1)
+  {
+    value = Substring(line, colon+1);
+    header.Truncate((PRUint32)colon);
+  }
+  else if (line.CharAt(0) == ' ' || line.CharAt(0) == '\t') // We are continuing the header
+  {
+    m_thisLine += header; // Preserve whitespace (should we?)
+    return NS_OK;
+  }
+  else
+  {
+    return NS_OK; // We are malformed. Just ignore and hope for the best...
+  }
+  
+  nsresult rv;
+  if (!m_lastHeader.IsEmpty())
+  {
+    rv = AddHeader(m_lastHeader.get(), m_thisLine.get());
+    NS_ENSURE_SUCCESS(rv,rv);
+  }
+  
+  value.Trim(" ");
+
+  ToLowerCase(header, m_lastHeader);
+  m_thisLine.Assign(value);
+  return NS_OK;
+}
+
 nsresult
-nsNNTPNewsgroupList::ClearXOVERState()
+nsNNTPNewsgroupList::AddHeader(const char *header, const char *value)
+{
+  nsresult rv = NS_OK;
+  // The From, Date, and Subject headers have special requirements.
+  if (PL_strcmp(header, "from") == 0)
+  {
+    rv = m_newMsgHdr->SetAuthor(value);
+  }
+  else if (PL_strcmp(header, "date") == 0)
+  {
+    PRTime date;
+    PRStatus status = PR_ParseTimeString (value, PR_FALSE, &date);
+    if (PR_SUCCESS == status)
+      rv = m_newMsgHdr->SetDate(date);
+  }
+  else if (PL_strcmp(header, "subject") == 0)
+  {
+    const char *subject = value;
+    PRUint32 subjectLen = strlen(value);
+
+    PRUint32 flags = 0;
+    // ### should call IsHeaderRead here...
+    /* strip "Re: " */
+    nsCString modifiedSubject;
+    if (NS_MsgStripRE(&subject, &subjectLen, getter_Copies(modifiedSubject)))
+      // this will make sure read flags agree with newsrc
+     (void) m_newMsgHdr->OrFlags(MSG_FLAG_HAS_RE, &flags);
+
+    if (! (flags & MSG_FLAG_READ))
+      rv = m_newMsgHdr->OrFlags(MSG_FLAG_NEW, &flags);
+
+    rv = m_newMsgHdr->SetSubject(modifiedSubject.IsEmpty() ? subject :
+      modifiedSubject.get());
+  }
+  else if (PL_strcmp(header, "message-id") == 0)
+  {
+    rv = m_newMsgHdr->SetMessageId(value);
+  }
+  else if (PL_strcmp(header, "references") == 0)
+  {
+    rv = m_newMsgHdr->SetReferences(value);
+  }
+  else if (PL_strcmp(header, "bytes") == 0)
+  {
+    rv = m_newMsgHdr->SetMessageSize(atol(value));
+  }
+  else if (PL_strcmp(header, "lines") == 0)
+  {
+    rv = m_newMsgHdr->SetLineCount(atol(value));
+  }
+  else if (m_filterHeaders.IndexOf(nsDependentCString(header)) != -1)
+  {
+    rv = m_newMsgHdr->SetStringProperty(header, value);
+  }
+  return rv;
+}
+
+nsresult
+nsNNTPNewsgroupList::CallFilters()
 {
-    return NS_OK;
+  nsresult rv;
+  nsCString filterString;
+  
+  nsCOMPtr <nsIMsgFolder> folder = do_QueryInterface(m_newsFolder, &rv);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  PRUint32 filterCount = 0;
+  if (m_filterList)
+  {
+    rv = m_filterList->GetFilterCount(&filterCount);
+    NS_ENSURE_SUCCESS(rv,rv);
+  }
+
+  PRUint32 serverFilterCount = 0;
+  if (m_serverFilterList)
+  {
+    rv = m_serverFilterList->GetFilterCount(&serverFilterCount);
+    NS_ENSURE_SUCCESS(rv,rv);
+  }
+
+  PRUint32 count = m_newHeaders.Count();
+  
+  for (PRUint32 i = 0; i < count; i++)
+  {
+    if (!filterCount && !serverFilterCount)
+    {
+      m_newsDB->AddNewHdrToDB(m_newHeaders[i], PR_TRUE);
+      continue;
+    }
+    m_addHdrToDB = PR_TRUE;
+    m_newMsgHdr = m_newHeaders[i];
+
+    // build up a "headers" for filter code
+    nsCString subject, author, date;
+    rv = m_newMsgHdr->GetSubject(getter_Copies(subject));
+    NS_ENSURE_SUCCESS(rv,rv);
+    rv = m_newMsgHdr->GetAuthor(getter_Copies(author));
+    NS_ENSURE_SUCCESS(rv,rv);
+
+    nsCString fullHeaders;
+    if (!(author.IsEmpty()))
+    {
+      fullHeaders.AppendLiteral(FROM_HEADER);
+      fullHeaders += author;
+      fullHeaders += '\0';
+    }
+
+    if (!(subject.IsEmpty()))
+    {
+      fullHeaders.AppendLiteral(SUBJECT_HEADER);
+      fullHeaders += subject;
+      fullHeaders += '\0';
+    }
+
+    for (PRInt32 header = 0; header < m_filterHeaders.Count(); header++)
+    {
+      nsCString retValue;
+      m_newMsgHdr->GetStringProperty(m_filterHeaders[header]->get(),
+                                     getter_Copies(retValue));
+      if (!retValue.IsEmpty())
+      {
+        fullHeaders += *(m_filterHeaders[header]);
+        fullHeaders.AppendLiteral(": ");
+        fullHeaders += retValue;
+        fullHeaders += '\0';
+      }
+    }
+
+    // The per-newsgroup filters should go first. If something stops filter
+    // execution, then users should be able to override the global filters in
+    // the per-newsgroup filters.
+    if (filterCount)
+    {
+      rv = m_filterList->ApplyFiltersToHdr(nsMsgFilterType::NewsRule,
+          m_newMsgHdr, folder, m_newsDB, fullHeaders.get(),
+          fullHeaders.Length(), this, m_msgWindow, nsnull);
+    }
+    if (serverFilterCount)
+    {
+      rv = m_serverFilterList->ApplyFiltersToHdr(nsMsgFilterType::NewsRule,
+          m_newMsgHdr, folder, m_newsDB, fullHeaders.get(),
+          fullHeaders.Length(), this, m_msgWindow, nsnull);
+    }
+
+    NS_ENSURE_SUCCESS(rv,rv);
+
+    if (m_addHdrToDB)
+      m_newsDB->AddNewHdrToDB(m_newMsgHdr, PR_TRUE);
+  }
+  m_newHeaders.Clear();
+  return NS_OK;
 }
 
 void
 nsNNTPNewsgroupList::SetProgressBarPercent(PRInt32 percent)
 {
   if (!m_runningURL)
     return;
 
@@ -1096,16 +1226,69 @@ nsNNTPNewsgroupList::SetProgressStatus(c
     mailnewsUrl->GetStatusFeedback(getter_AddRefs(feedback));
 
     if (feedback) {
       feedback->ShowStatusString(nsDependentString(message));
     }
   }
 }
 
+void
+nsNNTPNewsgroupList::UpdateStatus(PRBool filtering, PRInt32 numDLed, PRInt32 totToDL)
+{
+  PRInt32 numerator = (filtering ? m_currentXHDRIndex + 1 : 1) * numDLed;
+  PRInt32 denominator = (m_filterHeaders.Count() + 1) * totToDL;
+  PRInt32 percent = numerator * 100 / denominator;
+  
+  nsAutoString numDownloadedStr;
+  numDownloadedStr.AppendInt(numDLed);
+
+  nsAutoString totalToDownloadStr;
+  totalToDownloadStr.AppendInt(totToDL);
+
+  nsresult rv;
+  nsString statusString;
+  nsCOMPtr<nsIStringBundleService> bundleService = do_GetService(NS_STRINGBUNDLE_CONTRACTID, &rv);
+  if (!NS_SUCCEEDED(rv))
+    return;
+
+  nsCOMPtr<nsIStringBundle> bundle;
+  rv = bundleService->CreateBundle(NEWS_MSGS_URL, getter_AddRefs(bundle));
+  if (!NS_SUCCEEDED(rv))
+    return;
+
+  if (filtering)
+  {
+    NS_ConvertUTF8toUTF16 header(*m_filterHeaders[m_currentXHDRIndex]);
+    const PRUnichar *formatStrings[3] = { header.get(),
+      numDownloadedStr.get(), totalToDownloadStr.get() };
+    rv = bundle->FormatStringFromName(NS_LITERAL_STRING("downloadingFilterHeaders").get(),
+      formatStrings, 3, getter_Copies(statusString));
+  }
+  else
+  {
+    const PRUnichar *formatStrings[2] = { numDownloadedStr.get(),
+      totalToDownloadStr.get() };
+    rv = bundle->FormatStringFromName(NS_LITERAL_STRING("downloadingHeaders").get(),
+      formatStrings, 2, getter_Copies(statusString));
+  }
+  if (!NS_SUCCEEDED(rv))
+    return;
+
+  SetProgressStatus(statusString.get());
+  m_lastStatusUpdate = PR_Now();
+
+  // only update the progress meter if it has changed
+  if (percent != m_lastPercent)
+  {
+    SetProgressBarPercent(percent);
+    m_lastPercent = percent;
+  }
+}
+
 NS_IMETHODIMP nsNNTPNewsgroupList::SetGetOldMessages(PRBool aGetOldMessages)
 {
   m_getOldMessages = aGetOldMessages;
   return NS_OK;
 }
 
 NS_IMETHODIMP nsNNTPNewsgroupList::GetGetOldMessages(PRBool *aGetOldMessages)
 {
--- a/mailnews/news/src/nsNNTPNewsgroupList.h
+++ b/mailnews/news/src/nsNNTPNewsgroupList.h
@@ -48,16 +48,17 @@
 #include "nsINNTPNewsgroupList.h"
 #include "nsIMsgNewsFolder.h"
 #include "nsIMsgDatabase.h"
 #include "nsMsgKeySet.h"
 #include "nsINntpUrl.h"
 #include "nsIMsgFilterList.h"
 #include "nsIMsgHdr.h"
 #include "nsIMsgWindow.h"
+#include "nsCOMArray.h"
 
 /* The below is all stuff that we remember for netlib about which
    articles we've already seen in the current newsgroup. */
 
 typedef struct MSG_NewsKnown {
   nsMsgKeySet* set; /* Set of articles we've already gotten
                        from the newsserver (if it's marked
                        "read", then we've already gotten it).
@@ -98,39 +99,57 @@ private:
 #ifdef HAVE_CHANGELISTENER
   virtual void OnAnnouncerGoingAway (ChangeAnnouncer *instigator);
 #endif
   nsresult ParseLine(char *line, PRUint32 *message_number);
   nsresult GetDatabase(const char *uri, nsIMsgDatabase **db);
   void SetProgressBarPercent(PRInt32 percent);
   void SetProgressStatus(const PRUnichar *message);
 
+  void UpdateStatus(PRBool filtering, PRInt32 numDled, PRInt32 totToDL);
+
+  nsresult AddHeader(const char * header, const char * value);
 protected:
   PRBool m_getOldMessages;
   PRBool m_promptedAlready;
   PRBool m_downloadAll;
   PRInt32 m_maxArticles;
   PRInt32 m_lastPercent;
   PRTime m_lastStatusUpdate;
 
   nsCOMPtr <nsIMsgNewsFolder> m_newsFolder;
   nsCOMPtr <nsIMsgDatabase> m_newsDB;
   nsCOMPtr <nsINntpUrl> m_runningURL;
-  
+ 
+  /**
+   * The last message that we have processed (XOVER or HEAD).
+   */
   nsMsgKey m_lastProcessedNumber;
-  nsMsgKey m_firstMsgNumber;
-  nsMsgKey m_lastMsgNumber;
-  PRInt32 m_firstMsgToDownload;
-  PRInt32 m_lastMsgToDownload;
+  /**
+   * The endpoints of the message chunk we are actually downloading.
+   */
+  nsMsgKey m_firstMsgNumber, m_lastMsgNumber;
+  /**
+   * The endpoints of the message chunk we are capable of downloading.
+   */
+  PRInt32 m_firstMsgToDownload, m_lastMsgToDownload;
   
   struct MSG_NewsKnown m_knownArts;
   nsMsgKeySet *m_set;
 
+  nsCStringArray m_filterHeaders;
+  PRInt32 m_currentXHDRIndex;
+  nsCString m_lastHeader;
+  nsCString m_thisLine;
+
 private:
   nsCOMPtr <nsIMsgWindow> m_msgWindow;
   nsCOMPtr <nsIMsgFilterList> m_filterList;
   nsCOMPtr <nsIMsgFilterList> m_serverFilterList;
-  nsCOMPtr <nsIMsgDBHdr> m_newMsgHdr; /* current message header we're building */
+  nsCOMPtr <nsIMsgDBHdr> m_newMsgHdr; // current message header we're building
+  nsCOMArray<nsIMsgDBHdr> m_newHeaders;
+
   PRBool m_addHdrToDB;
+
 };
     
 #endif /* nsNNTPNewsgroupListState_h___ */
 
--- a/mailnews/news/src/nsNNTPProtocol.cpp
+++ b/mailnews/news/src/nsNNTPProtocol.cpp
@@ -92,16 +92,18 @@
 
 #include "nsIPrompt.h"
 #include "nsIMsgStatusFeedback.h"
 
 #include "nsIMsgFolder.h"
 #include "nsIMsgNewsFolder.h"
 #include "nsIDocShell.h"
 
+#include "nsIMsgFilterList.h"
+
 // for the memory cache...
 #include "nsICacheEntryDescriptor.h"
 #include "nsICacheSession.h"
 #include "nsIStreamListener.h"
 #include "nsNetCID.h"
 
 #include "nsIPrefBranch.h"
 #include "nsIPrefService.h"
@@ -197,16 +199,18 @@ const char *const stateLabels[] = {
 "NNTP_BEGIN_ARTICLE",
 "NNTP_READ_ARTICLE",
 "NNTP_XOVER_BEGIN",
 "NNTP_FIGURE_NEXT_CHUNK",
 "NNTP_XOVER_SEND",
 "NNTP_XOVER_RESPONSE",
 "NNTP_XOVER",
 "NEWS_PROCESS_XOVER",
+"NNTP_XHDR_SEND",
+"NNTP_XHDR_RESPONSE",
 "NNTP_READ_GROUP",
 "NNTP_READ_GROUP_RESPONSE",
 "NNTP_READ_GROUP_BODY",
 "NNTP_SEND_GROUP_FOR_ARTICLE",
 "NNTP_SEND_GROUP_FOR_ARTICLE_RESPONSE",
 "NNTP_PROFILE_ADD",
 "NNTP_PROFILE_ADD_RESPONSE",
 "NNTP_PROFILE_DELETE",
@@ -3287,18 +3291,21 @@ void nsNNTPProtocol::HandleAuthenticatio
     // but we need to handle the case where the password has
     // changed while the app is running, and
     // reprompt the user for the (potentially) new password.
     // So clear the userAuthenticated flag.
     m_nntpServer->SetUserAuthenticated(PR_FALSE);
   }
 }
 
-/* start the xover command
- */
+///////////////////////////////////////////////////////////////////////////////
+// XOVER, XHDR, and HEAD processing code
+// Used for filters
+// State machine explanation located in doxygen comments for nsNNTPProtocol
+///////////////////////////////////////////////////////////////////////////////
 
 PRInt32 nsNNTPProtocol::BeginReadXover()
 {
   PRInt32 count;     /* Response fields */
   nsresult rv = NS_OK;
 
   rv = SetCurrentGroup();
   if (NS_FAILED(rv)) return -1;
@@ -3372,18 +3379,17 @@ PRInt32 nsNNTPProtocol::FigureNextChunk(
     }
 
     rv = m_newsgroupList->SetGetOldMessages(getOldMessages);
     if (NS_FAILED(rv)) return status;
 
     rv = m_newsgroupList->GetRangeOfArtsToDownload(m_msgWindow,
       m_firstPossibleArticle,
       m_lastPossibleArticle,
-      m_numArticlesWanted -
-      m_numArticlesLoaded,
+      m_numArticlesWanted - m_numArticlesLoaded,
       &(m_firstArticle),
       &(m_lastArticle),
       &status);
 
   if (NS_FAILED(rv)) return status;
 
   if (m_firstArticle <= 0 || m_firstArticle > m_lastArticle)
   {
@@ -3492,17 +3498,17 @@ PRInt32 nsNNTPProtocol::ReadXover(nsIInp
     return 0;
   }
 
   if(!line)
     return(status);  /* no line yet or TCP error */
 
   if(line[0] == '.' && line[1] == '\0')
   {
-    m_nextState = NNTP_FIGURE_NEXT_CHUNK;
+    m_nextState = NNTP_XHDR_SEND;
     ClearFlag(NNTP_PAUSE_FOR_READ);
     PR_Free(lineToFree);
     return(0);
   }
   else if (line [0] == '.' && line [1] == '.')
     /* The NNTP server quotes all lines beginning with "." by doubling it. */
     line++;
 
@@ -3514,51 +3520,127 @@ PRInt32 nsNNTPProtocol::ReadXover(nsIInp
     mBytesReceivedSinceLastStatusUpdate += status;
   }
 
   rv = m_newsgroupList->ProcessXOVERLINE(line, &status);
   NS_ASSERTION(NS_SUCCEEDED(rv), "failed to process the XOVERLINE");
 
   m_numArticlesLoaded++;
   PR_Free(lineToFree);
-  return NS_SUCCEEDED(rv) ? status : -1; /* keep going if no error */
+  return NS_SUCCEEDED(rv) ? (PRInt32)status : -1; /* keep going if no error */
 }
 
 /* Finished processing all the XOVER data.
 */
 
 PRInt32 nsNNTPProtocol::ProcessXover()
 {
   nsresult rv;
 
   /* xover_parse_state stored in MSG_Pane cd->pane */
   NS_ASSERTION(m_newsgroupList, "no newsgroupList");
   if (!m_newsgroupList) return -1;
 
   PRInt32 status = 0;
+  m_newsgroupList->CallFilters();
   rv = m_newsgroupList->FinishXOVERLINE(0,&status);
   m_newsgroupList = nsnull;
   if (NS_SUCCEEDED(rv) && status < 0) return status;
 
   m_nextState = NEWS_DONE;
 
   return(MK_DATA_LOADED);
 }
 
-PRInt32 nsNNTPProtocol::ReadNewsgroup()
+PRInt32 nsNNTPProtocol::XhdrSend()
+{
+  nsCString header;
+  m_newsgroupList->InitXHDR(header);
+  if (header.IsEmpty())
+  {
+    m_nextState = NNTP_FIGURE_NEXT_CHUNK;
+    return 0;
+  }
+  
+  char outputBuffer[OUTPUT_BUFFER_SIZE];
+  PR_snprintf(outputBuffer, OUTPUT_BUFFER_SIZE, "XHDR %s %d-%d" CRLF,
+              header.get(), m_firstArticle, m_lastArticle);
+
+  m_nextState = NNTP_RESPONSE;
+  m_nextStateAfterResponse = NNTP_XHDR_RESPONSE;
+  SetFlag(NNTP_PAUSE_FOR_READ);
+
+  nsCOMPtr<nsIMsgMailNewsUrl> mailnewsurl = do_QueryInterface(m_runningURL);
+  return mailnewsurl ? SendData(mailnewsurl, outputBuffer) : 0;
+}
+
+PRInt32 nsNNTPProtocol::XhdrResponse(nsIInputStream *inputStream)
+{
+  if (m_responseCode != MK_NNTP_RESPONSE_XHDR_OK)
+  {
+    m_nextState = NNTP_READ_GROUP;
+    // The reasoning behind setting this flag and not an XHDR flag is that we
+    // are going to have to use HEAD instead. At that point, using XOVER as
+    // well is just wasting bandwidth.
+    SetFlag(NNTP_NO_XOVER_SUPPORT);
+    return 0;
+  }
+  
+  char *line, *lineToFree;
+  nsresult rv;
+  PRUint32 status = 1;
+
+  PRBool pauseForMoreData = PR_FALSE;
+  line = lineToFree = m_lineStreamBuffer->ReadNextLine(inputStream, status, pauseForMoreData);
+
+  if (pauseForMoreData)
+  {
+    SetFlag(NNTP_PAUSE_FOR_READ);
+    return 0;
+  }
+
+  if (!line)
+    return status;  /* no line yet or TCP error */
+
+  if (line[0] == '.' && line[1] == '\0')
+  {
+    m_nextState = NNTP_XHDR_SEND;
+    ClearFlag(NNTP_PAUSE_FOR_READ);
+    PR_Free(lineToFree);
+    return(0);
+  }
+
+  if (status > 1)
+  {
+    mBytesReceived += status;
+    mBytesReceivedSinceLastStatusUpdate += status;
+  }
+
+  rv = m_newsgroupList->ProcessXHDRLine(nsDependentCString(line));
+  NS_ASSERTION(NS_SUCCEEDED(rv), "failed to process the XHDRLINE");
+
+  m_numArticlesLoaded++;
+  PR_Free(lineToFree);
+  return NS_SUCCEEDED(rv) ? (PRInt32)status : -1; /* keep going if no error */
+}
+
+PRInt32 nsNNTPProtocol::ReadHeaders()
 {
   if(m_articleNumber > m_lastArticle)
   {  /* end of groups */
 
+    m_newsgroupList->InitHEAD(-1);
     m_nextState = NNTP_FIGURE_NEXT_CHUNK;
     ClearFlag(NNTP_PAUSE_FOR_READ);
     return(0);
   }
   else
   {
+    m_newsgroupList->InitHEAD(m_articleNumber);
+
     char outputBuffer[OUTPUT_BUFFER_SIZE];
     PR_snprintf(outputBuffer,
       OUTPUT_BUFFER_SIZE,
       "HEAD %ld" CRLF,
       m_articleNumber++);
     m_nextState = NNTP_RESPONSE;
     m_nextStateAfterResponse = NNTP_READ_GROUP_RESPONSE;
 
@@ -3571,35 +3653,30 @@ PRInt32 nsNNTPProtocol::ReadNewsgroup()
   }
 }
 
 /* See if the "HEAD" command was successful
 */
 
 PRInt32 nsNNTPProtocol::ReadNewsgroupResponse()
 {
-  nsresult rv;
-
   if (m_responseCode == MK_NNTP_RESPONSE_ARTICLE_HEAD)
   {     /* Head follows - parse it:*/
     m_nextState = NNTP_READ_GROUP_BODY;
 
     if(m_messageID)
       *m_messageID = '\0';
 
     m_key = nsMsgKey_None;
 
-    /* Give the message number to the header parser. */
-    rv = m_newsgroupList->ProcessNonXOVER(m_responseText);
-    /* convert nsresult->status */
-    return NS_FAILED(rv);
+    return 0;
   }
   else
   {
-    NNTP_LOG_NOTE(("Bad group header found!"));
+    m_newsgroupList->HEADFailed(m_articleNumber);
     m_nextState = NNTP_READ_GROUP;
     return(0);
   }
 }
 
 /* read the body of the "HEAD" command
 */
 PRInt32 nsNNTPProtocol::ReadNewsgroupBody(nsIInputStream * inputStream, PRUint32 length)
@@ -3624,24 +3701,26 @@ PRInt32 nsNNTPProtocol::ReadNewsgroupBod
 
   PR_LOG(NNTP,PR_LOG_ALWAYS,("(%p) read_group_body: got line: %s|",this,line));
 
   /* End of body? */
   if (line[0]=='.' && line[1]=='\0')
   {
     m_nextState = NNTP_READ_GROUP;
     ClearFlag(NNTP_PAUSE_FOR_READ);
+    return 0;
   }
   else if (line [0] == '.' && line [1] == '.')
     /* The NNTP server quotes all lines beginning with "." by doubling it. */
     line++;
 
-  rv = m_newsgroupList->ProcessNonXOVER(line);
+  nsCString safe_line(line);
+  rv = m_newsgroupList->ProcessHEADLine(safe_line);
+  PR_Free(lineToFree);
   /* convert nsresult->status */
-  PR_Free(lineToFree);
   return NS_FAILED(rv);
 }
 
 
 nsresult nsNNTPProtocol::GetNewsStringByID(PRInt32 stringID, PRUnichar **aString)
 {
   nsresult rv;
   nsAutoString resultString(NS_LITERAL_STRING("???"));
@@ -5026,29 +5105,37 @@ nsresult nsNNTPProtocol::ProcessProtocol
     case NEWS_PROCESS_XOVER:
     case NEWS_PROCESS_BODIES:
 #ifdef UNREADY_CODE
       NET_Progress(ce->window_id, XP_GetString(XP_PROGRESS_SORT_ARTICLES));
 #endif
       status = ProcessXover();
       break;
 
+    case NNTP_XHDR_SEND:
+      status = XhdrSend();
+      break;
+
+    case NNTP_XHDR_RESPONSE:
+      status = XhdrResponse(inputStream);
+      break;
+
     case NNTP_READ_GROUP:
-      status = ReadNewsgroup();
+      status = ReadHeaders();
       break;
 
     case NNTP_READ_GROUP_RESPONSE:
       if (inputStream == nsnull)
         SetFlag(NNTP_PAUSE_FOR_READ);
       else
         status = ReadNewsgroupResponse();
       break;
 
     case NNTP_READ_GROUP_BODY:
-      status = ReadNewsgroupResponse();
+      status = ReadNewsgroupBody(inputStream, length);
       break;
 
     case NNTP_SEND_POST_DATA:
       status = PostData();
       break;
     case NNTP_SEND_POST_DATA_RESPONSE:
       if (inputStream == nsnull)
         SetFlag(NNTP_PAUSE_FOR_READ);
--- a/mailnews/news/src/nsNNTPProtocol.h
+++ b/mailnews/news/src/nsNNTPProtocol.h
@@ -111,16 +111,18 @@ NNTP_NEWGROUPS,
 NNTP_BEGIN_ARTICLE,
 NNTP_READ_ARTICLE,
 NNTP_XOVER_BEGIN,
 NNTP_FIGURE_NEXT_CHUNK,
 NNTP_XOVER_SEND,
 NNTP_XOVER_RESPONSE,
 NNTP_XOVER,
 NEWS_PROCESS_XOVER,
+NNTP_XHDR_SEND,
+NNTP_XHDR_RESPONSE,
 NNTP_READ_GROUP,
 NNTP_READ_GROUP_RESPONSE,
 NNTP_READ_GROUP_BODY,
 NNTP_SEND_GROUP_FOR_ARTICLE,
 NNTP_SEND_GROUP_FOR_ARTICLE_RESPONSE,
 NNTP_PROFILE_ADD,
 NNTP_PROFILE_ADD_RESPONSE,
 NNTP_PROFILE_DELETE,
@@ -263,37 +265,36 @@ private:
   // Per news article state information. (article number, author, subject, id, etc
   char   *m_messageID;
   PRInt32   m_articleNumber;   /* current article number */
   char   *m_commandSpecificData;
   char   *m_searchData;
 
   PRInt32   m_originalContentLength; /* the content length at the time of calling graph progress */
 
-
   nsCOMPtr<nsIStringBundle> m_stringBundle;
 
   nsCOMPtr<nsINntpIncomingServer> m_nntpServer;
 
   nsresult GetNewsStringByName(const char *aName, PRUnichar **aString);
   nsresult GetNewsStringByID(PRInt32 stringID, PRUnichar **aString);
 
   PRInt32 PostMessageInFile(nsIFile * filePath);
 
-  ////////////////////////////////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////
   // Communication methods --> Reading and writing protocol
-  ////////////////////////////////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////
 
   PRInt32 ReadLine(nsIInputStream * inputStream, PRUint32 length, char ** line);
 
-  ////////////////////////////////////////////////////////////////////////////////////////
-  // Protocol Methods --> This protocol is state driven so each protocol method is
-  //            designed to re-act to the current "state". I've attempted to
+  //////////////////////////////////////////////////////////////////////////////
+  // Protocol Methods --> This protocol is state driven so each protocol method
+  //            is designed to re-act to the current "state". I've attempted to
   //            group them together based on functionality.
-  ////////////////////////////////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////
 
   // gets the response code from the nntp server and the response line. Returns the TCP return code
   // from the read.
   PRInt32 NewsResponse(nsIInputStream * inputStream, PRUint32 length);
 
   // Interpret the server response after the connect.
   // Returns negative if the server responds unexpectedly
   PRInt32 LoginResponse();
@@ -341,44 +342,120 @@ private:
   PRInt32 BeginReadNewsList();
   PRInt32 ReadNewsList(nsIInputStream * inputStream, PRUint32 length);
 
   // Newsgroup specific protocol handlers
   PRInt32 DisplayNewsgroups();
   PRInt32 BeginNewsgroups();
   PRInt32 ProcessNewsgroups(nsIInputStream * inputStream, PRUint32 length);
 
-  PRInt32 ReadNewsgroup();
-  PRInt32 ReadNewsgroupResponse();
-
-  PRInt32 ReadNewsgroupBody(nsIInputStream * inputStream, PRUint32 length);
-
   // Protocol handlers used for posting data
   PRInt32 PostData();
   PRInt32 PostDataResponse();
 
   PRInt32 CheckForArticle();
 
   // NewsRC specific
   PRInt32 DisplayNewsRC();
   PRInt32 DisplayNewsRCResponse();
 
-  // start off the xover command
+  /////////////////////////////////////////////////////////////////////////////
+  // XHDR, XOVER, HEAD filtering process handlers
+  // These are ordered by the rough order of usage
+  /////////////////////////////////////////////////////////////////////////////
+ 
+  /**
+   * The first step in the filtering process, the state NNTP_XOVER_BEGIN.
+   * This method sets up m_newsgroupList.
+   * Followed by: NNTP_FIGURE_NEXT_CHUNK
+   */
   PRInt32 BeginReadXover();
+  /**
+   * The loop control for filtering, the state NNTP_FIGURE_NEXT_CHUNK.
+   * This method contacts the newsgroupList to figure out which articles to
+   * download and then prepares it for XOVER support.
+   * Followed by: NEWS_PROCESS_XOVER       if everything is finished
+   *              NNTP_READ_GROUP          if XOVER doesn't work
+   *              NNTP_XOVER_SEND          if XOVER does work
+   */
+  PRInt32 FigureNextChunk();
 
-  // process the xover list as it comes from the server and load it into the sort list.
+  // The XOVER process core
+  /**
+   * The state NNTP_XOVER_SEND, which actually sends the message.
+   * Followed by: NNTP_XOVER_RESPONSE
+   */
+  PRInt32 XoverSend();
+  /**
+   * This state, NNTP_XOVER_RESPONSE, actually checks the XOVER capabiliity.
+   * Followed by: NNTP_XOVER               if XOVER is supported
+   *              NNTP_READ_GROUP          if it isn't
+   */
+  PRInt32 ReadXoverResponse();
+  /**
+   * This state, NNTP_XOVER, processes the results from the XOVER command.
+   * It asks nsNNTPNewsgroupList to process the line using ProcessXOVERLINE.
+   * Followed by: NNTP_XHDR_SEND
+   */
   PRInt32 ReadXover(nsIInputStream * inputStream, PRUint32 length);
-  // See if the xover response is going to return us data. If the proper code isn't returned then
-  // assume xover isn't supported and use normal read_group.
-  PRInt32 ReadXoverResponse();
+
+  // The XHDR process core
+  /**
+   * This state, NNTP_XHDR_SEND, sends the XHDR command.
+   * The headers are all managed by nsNNTPNewsgroupList, and this picks them up
+   * one by one as they are needed.
+   * Followed by: NNTP_XHDR_RESPONSE       if there is a header to be sent
+   *              NNTP_FIGURE_NEXT_CHUNK   if all headers have been sent
+   */
+  PRInt32 XhdrSend();
+  /**
+   * This state, NNTP_XHDR_RESPONSE, processes the XHDR response.
+   * It mostly passes the information off to nsNNTPNewsgroupList, and only does
+   * response code checking and a bit of preprocessing. Note that if XHDR
+   * doesn't work properly, HEAD fallback is switched on and all subsequent
+   * chunks will NOT use XOVER.
+   * Followed by: NNTP_READ_GROUP          if XHDR doesn't work properly
+   *              NNTP_XHDR_SEND           when finished processing XHR.
+   */
+  PRInt32 XhdrResponse(nsIInputStream *inputStream);
 
-  PRInt32 XoverSend();
+  // HEAD processing core
+  /**
+   * This state, NNTP_READ_GROUP, is the control for the HEAD processor.
+   * It sends the HEAD command and increments the article number until it is
+   * finished. WARNING: HEAD is REALLY SLOW.
+   * Followed by: NNTP_FIGURE_NEXT_CHUNK   when it is finished 
+   *              NNTP_READ_GROUP_RESPONSE when it is not
+   */
+  PRInt32 ReadHeaders();
+  /**
+   * This state, NNTP_READ_GROUP_RESPONSE, checks if the article exists.
+   * Because it is required by NNTP, if it doesn't work, the only problem would
+   * be that the article doesn't exist. Passes off article number data to 
+   * nsNNTPNewsgroupList.
+   * Followed by: NNTP_READ_GROUP_BODY     if the article exists
+   *              NNTP_READ_GROUP          if it doesn't.
+   */
+  PRInt32 ReadNewsgroupResponse();
+  /**
+   * This state, NNTP_READ_GROUP_BODY, reads the body of the HEAD command.
+   * Once again, it passes information off to nsNNTPNewsgroupList.
+   * Followed by: NNTP_READ_GROUP
+   */
+  PRInt32 ReadNewsgroupBody(nsIInputStream * inputStream, PRUint32 length);
+
+  /**
+   * This state, NNTP_PROCESS_XOVER, is the final step of the filter-processing
+   * code. Currently, all it does is cleans up the unread count and calls the
+   * filters, both via nsNNTPNewsgroupList.
+   * Followed by: NEWS_DONE
+   */
   PRInt32 ProcessXover();
 
-  PRInt32 FigureNextChunk();
+
 
   // Canceling
   PRInt32 StartCancel();
   PRInt32 DoCancel();
 
   // XPAT
   PRInt32 XPATSend();
   PRInt32 XPATResponse(nsIInputStream * inputStream, PRUint32 length);
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post1.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: First post!
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <1@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:1
+
+This is the first body post.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post2.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Odd Subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <2@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:2
+
+This is the second body post.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post3.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Odd Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Regular subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <3@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:3
+
+This is the third body post.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post4.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Sat, 1 Jan 2000 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Regular subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <4@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:4
+
+This is the fourth body post.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post5.eml
@@ -0,0 +1,50 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Regular subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <5@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:5
+Bytes: 2057
+
+This is the fifth body post, with extra special padding to make sure
+that no one else has the same length as it. You see, we have to ensure
+that we have at least two KB of data because search is stupid and
+searches in terms of KB. If we didn't, we'd be stuck with a lot of
+messages merely matching 1, so we use this to pad the size.
+
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
+commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
+velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
+occaecat cupidatat non proident, sunt in culpa qui officia deserunt
+mollit anim id est laborum.
+
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
+commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
+velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
+occaecat cupidatat non proident, sunt in culpa qui officia deserunt
+mollit anim id est laborum.
+
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
+commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
+velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
+occaecat cupidatat non proident, sunt in culpa qui officia deserunt
+mollit anim id est laborum.
+
+.
+
+But that wasn't all :-)
+
+-- 
+A signature for the heck of it.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post6.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Program/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Regular subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <6.odd@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:6
+
+This is the sixth body post.
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/postings/auto-add/post7.eml
@@ -0,0 +1,14 @@
+Path: border1.nntp.dca.giganews.com!nntp.giganews.com!local02.nntp.dca.giganews.com!nntp.mozilla.org!news.mozilla.org.POSTED!not-for-mail
+Date: Mon, 23 Jun 2008 19:58:07 +0400
+From: Normal Person <fake@acme.invalid>
+User-Agent: Odd/1.0
+MIME-Version: 1.0
+Newsgroups: test.filter
+Subject: Regular subject
+Content-Type: text/plain; charset=ISO-8859-1; format=flowed
+Content-Transfer-Encoding: 7bit
+Message-ID: <7@regular.invalid>
+NNTP-Posting-Host: 127.0.0.1
+Xref: test.filter:7
+
+This is the seventh body post.
--- a/mailnews/news/test/unit/head_server_setup.js
+++ b/mailnews/news/test/unit/head_server_setup.js
@@ -6,38 +6,77 @@ do_import_script("../mailnews/test/fakes
 do_import_script("../mailnews/test/resources/mailDirService.js")
 
 // The groups to set up on the fake server.
 // It is an array of tuples, where the first element is the group name and the
 // second element is whether or not we should subscribe to it.
 var groups = [
   ["test.empty", false],
   ["test.subscribe.empty", true],
-  ["test.subscribe.simple", true]
+  ["test.subscribe.simple", true],
+  ["test.filter", true]
 ];
 // Sets up the NNTP daemon object for use in fake server
 function setupNNTPDaemon() {
   var daemon = new nntpDaemon();
 
   groups.forEach(function (element) {
     daemon.addGroup(element[0]);
   });
 
+  var auto_add = do_get_file("../mailnews/news/test/postings/auto-add/");
+  var files = [];
+  var enumerator = auto_add.directoryEntries;
+  while (enumerator.hasMoreElements())
+    files.push(enumerator.getNext().QueryInterface(Ci.nsIFile));
+  
+  files.sort(function (a, b) {
+    if (a.leafName == b.leafName) return 0;
+    return a.leafName < b.leafName ? -1 : 1;
+  });
+
+  files.forEach(function (file) {
+      var fstream = Cc["@mozilla.org/network/file-input-stream;1"]
+                      .createInstance(Ci.nsIFileInputStream);
+      var sstream = Cc["@mozilla.org/scriptableinputstream;1"]
+                      .createInstance(Ci.nsIScriptableInputStream);
+      fstream.init(file, -1, 0, 0);
+      sstream.init(fstream);
+      
+      var post = "";
+      for (let part = sstream.read(4096); part.length > 0;) {
+        post += part;
+        part = sstream.read(4096);
+      }
+      sstream.close();
+      fstream.close();
+      daemon.addArticle(new newsArticle(post));
+  });
+
   var article = new newsArticle("From: John Doe <john.doe@example.com>\n"+
       "Date: Sat, 24 Mar 1990 10:59:24 -0500\n"+
       "Newsgroups: test.subscribe.simple\n"+
       "Subject: H2G2 -- What does it mean?\n"+
       "Message-ID: <TSS1@nntp.test>\n"+
       "\n"+
       "What does the acronym H2G2 stand for? I've seen it before...\n");
   daemon.addArticleToGroup(article, "test.subscribe.simple", 1);
 
   return daemon;
 }
 
+// Enable strict threading
+var prefs = Cc["@mozilla.org/preferences-service;1"]
+              .getService(Ci.nsIPrefBranch);
+prefs.setBoolPref("mail.strict_threading", true);
+
+
+// Make sure we don't try to use a protected port. I like adding 1024 to the
+// default port when doing so...
+const NNTP_PORT = 1024+119;
 
 var _server = null;
 
 // Sets up the client-side portion of fakeserver
 function setupLocalServer(port) {
   if (_server != null)
     return _server;
   var acctmgr = Cc["@mozilla.org/messenger/account-manager;1"]
@@ -110,8 +149,21 @@ function create_post(baseURL, file) {
   url.QueryInterface(Ci.nsINntpUrl);
 
   var post = Cc["@mozilla.org/messenger/nntpnewsgrouppost;1"]
                .createInstance(Ci.nsINNTPNewsgroupPost);
   post.postMessageFile = do_get_file(file);
   url.messageToPost = post;
   return url;
 }
+
+function resetFolder(folder) {
+  var headerEnum = folder.getMessages(null);
+  var headers = [];
+  while (headerEnum.hasMoreElements())
+    headers.push(headerEnum.getNext().QueryInterface(Ci.nsIMsgDBHdr));
+
+  var db = folder.getMsgDatabase(null);
+  db.dBFolderInfo.knownArtsSet = "";
+  for each (var header in headers) {
+    db.DeleteHeader(header, null, true, false);
+  }
+}
new file mode 100644
--- /dev/null
+++ b/mailnews/news/test/unit/test_filter.js
@@ -0,0 +1,234 @@
+////////////////////////////////////////////////////////////////////////////////
+// Tests for the filtering code of NNTP. The same tests are run for each of the
+// different NNTP setups, to test code in a variety of cases.
+//
+// Different suites:
+// * Perfect 3977 compliance (not tested)
+// * Perfect 2980 compliance (XOVER and XHDR work)
+// * Giganews compliance (XHDR doesn't work for practical purposes)
+// * Only 977 compliance (no XOVER support)
+// Basic operations:
+// * Test that the following headers trigger:
+//   - Subject
+//   - From
+//   - Date
+//   - Size
+//   - Message-ID (header retrievable by XOVER)
+//   - User-Agent (header not retrievable by XHDR)
+// * Test all actions
+////////////////////////////////////////////////////////////////////////////////
+
+var contains = Ci.nsMsgSearchOp.Contains;
+// This maps strings to a filter attribute (excluding the parameter)
+var ATTRIB_MAP = {
+  // Template : [attrib, op, field of value, otherHeader]
+  "subject" : [Ci.nsMsgSearchAttrib.Subject, contains, "str", null],
+  "from" : [Ci.nsMsgSearchAttrib.Sender, contains, "str", null],
+  "date" : [Ci.nsMsgSearchAttrib.Date, Ci.nsMsgSearchOp.Is, "date", null],
+  "size" : [Ci.nsMsgSearchAttrib.Size, Ci.nsMsgSearchOp.Is, "size", null],
+  "message-id" : [Ci.nsMsgSearchAttrib.OtherHeader+1, contains, "str",
+                  "Message-ID"],
+  "user-agent" : [Ci.nsMsgSearchAttrib.OtherHeader+2, contains, "str",
+                  "User-Agent"]
+};
+// And this maps strings to filter actions
+var ACTION_MAP = {
+  // Template : [action, auxiliary attribute field, auxiliary value]
+  "priority" : [Ci.nsMsgFilterAction.ChangePriority, "priority", 6],
+  "delete" : [Ci.nsMsgFilterAction.Delete],
+  "read" : [Ci.nsMsgFilterAction.MarkRead],
+  "kill" : [Ci.nsMsgFilterAction.KillThread],
+  "watch" : [Ci.nsMsgFilterAction.WatchThread],
+  "flag" : [Ci.nsMsgFilterAction.MarkFlagged],
+  "stop": [Ci.nsMsgFilterAction.StopExecution],
+  "tag" : [Ci.nsMsgFilterAction.AddTag, "strValue", "tag"]
+};
+
+function createFilter(list, header, value, action) {
+  var filter = list.createFilter(header+action+"Test");
+  filter.filterType = Ci.nsMsgFilterType.NewsRule;
+
+  var searchTerm = filter.createTerm();
+  searchTerm.matchAll = false;
+  if (header in ATTRIB_MAP) {
+    let information = ATTRIB_MAP[header];
+    searchTerm.attrib = information[0];
+    if (information[3] != null)
+      searchTerm.arbitraryHeader = information[3];
+    searchTerm.op = information[1];
+    var oldValue = searchTerm.value;
+    oldValue.attrib = information[0];
+    oldValue[information[2]] = value;
+    searchTerm.value = oldValue;
+  } else {
+    throw "Unknown header "+header;
+  }
+  searchTerm.booleanAnd = true;
+  filter.appendTerm(searchTerm);
+
+  var filterAction = filter.createAction();
+  if (action in ACTION_MAP) {
+    let information = ACTION_MAP[action];
+    filterAction.type = information[0];
+    if (1 in information)
+      filterAction[information[1]] = information[2];
+  } else {
+    throw "Unknown action "+action;
+  }
+  filter.appendAction(filterAction);
+
+  filter.enabled = true;
+
+  // Add to the end
+  list.insertFilterAt(list.filterCount, filter);
+}
+
+// These are the expected results for testing filter triggers
+var attribResults = {
+  "1@regular.invalid" : ["isRead", false],
+  "2@regular.invalid" : ["isRead", true],
+  "3@regular.invalid" : ["isRead", true],
+  "4@regular.invalid" : ["isRead", true],
+  "5@regular.invalid" : ["isRead", true],
+  "6.odd@regular.invalid" : ["isRead", true],
+  "7@regular.invalid" : ["isRead", true]
+};
+function testAttrib(handler, localserver) {
+  var server = new nsMailServer(handler);
+  server.start(NNTP_PORT);
+
+  // Get the folder and force filters to run
+  var folder = localserver.rootFolder.getChildNamed("test.filter");
+  folder.getNewMessages(null, {
+    OnStopRunningUrl: function () { localserver.closeCachedConnections() }});
+  server.performTest();
+
+  var headerEnum = folder.getMessages(null);
+  var headers = [];
+  while (headerEnum.hasMoreElements())
+    headers.push(headerEnum.getNext().QueryInterface(Ci.nsIMsgDBHdr));
+
+  try
+  {
+    do_check_eq(headers.length, 7);
+    for each (var header in headers) {
+      var id = header.messageId;
+      dump("Testing message "+id+"\n");
+      do_check_eq(header[attribResults[id][0]], attribResults[id][1]);
+    }
+  } catch (e) {
+    print(server.playTransaction().them);
+    throw e;
+  } finally {
+    server.stop();
+  }
+
+  resetFolder(folder);
+}
+
+// These are the results for testing actual actions
+var actionResults = {
+  "1@regular.invalid" : ["priority", 6],
+  // "2@regular.invalid" should not be in database
+  "3@regular.invalid" : function (header, folder) {
+    var db = folder.getMsgDatabase(null);
+    var flags = db.GetThreadContainingMsgHdr(header).flags;
+    // This is checking the thread's kill flag
+    return (flags & 0x40000) == 0x40000;
+  },
+  "4@regular.invalid" : function (header, folder) {
+    var db = folder.getMsgDatabase(null);
+    var flags = db.GetThreadContainingMsgHdr(header).flags;
+    // This is checking the thread's watch flag
+    return (flags & 0x100) == 0x100;
+  },
+  "5@regular.invalid" : ["isFlagged", true],
+  "6.odd@regular.invalid" : ["isRead", false],
+  "7@regular.invalid" : function (header, folder) {
+    return header.getStringProperty("keywords") == "tag";
+  }
+};
+function testAction(handler, localserver) {
+  var server = new nsMailServer(handler);
+  server.start(NNTP_PORT);
+
+  // Get the folder and force filters to run
+  var folder = localserver.rootFolder.getChildNamed("test.filter");
+  folder.getNewMessages(null, {
+    OnStopRunningUrl: function () { localserver.closeCachedConnections() }});
+  server.performTest();
+
+  var headerEnum = folder.getMessages(null);
+  var headers = [];
+  while (headerEnum.hasMoreElements())
+    headers.push(headerEnum.getNext().QueryInterface(Ci.nsIMsgDBHdr));
+
+  try
+  {
+    do_check_eq(headers.length, 6);
+    for each (var header in headers) {
+      var id = header.messageId;
+      dump("Testing message "+id+"\n");
+      if (actionResults[id] instanceof Array)
+        do_check_eq(header[actionResults[id][0]], actionResults[id][1]);
+      else
+        do_check_true(actionResults[id](header, folder));
+    }
+  } catch (e) {
+    print(server.playTransaction().them);
+    throw e;
+  } finally {
+    server.stop();
+  }
+
+  resetFolder(folder);
+}
+
+// These are the various server handlers
+var handlers = [NNTP_RFC977_handler, NNTP_Giganews_handler,
+                NNTP_RFC2980_handler];
+function run_test() {
+  // Set up the server and add in filters
+  var daemon = setupNNTPDaemon();
+  var localserver = setupLocalServer(NNTP_PORT);
+  var serverFilters = localserver.getFilterList(null);
+
+  createFilter(serverFilters, "subject", "Odd", "read");
+  createFilter(serverFilters, "from", "Odd Person", "read");
+  // A PRTime is the time in μs, but a JS date is time in ms.
+  createFilter(serverFilters, "date", new Date(2000, 0, 1)*1000, "read");
+  createFilter(serverFilters, "size", 2, "read");
+  createFilter(serverFilters, "message-id", "odd", "read");
+  createFilter(serverFilters, "user-agent", "Odd/1.0", "read");
+  localserver.setFilterList(serverFilters);
+
+  handlers.forEach( function (handler) {
+    var handlerObj = new handler(daemon);
+    testAttrib(handlerObj, localserver);
+  });
+
+  // Now we test folder-filters... and actions
+  // Clear out the server filters
+  while (serverFilters.filterCount > 0)
+    serverFilters.removeFilterAt(0);
+  localserver.setFilterList(serverFilters);
+
+  var folder = localserver.rootFolder.getChildNamed("test.filter");
+  var folderFilters = folder.getFilterList(null);
+  createFilter(folderFilters, "subject", "First", "priority");
+  createFilter(folderFilters, "subject", "Odd", "delete");
+  createFilter(folderFilters, "from", "Odd Person", "kill");
+  createFilter(folderFilters, "date", new Date(2000, 0, 1)*1000, "watch");
+  createFilter(folderFilters, "size", 2, "flag");
+  createFilter(folderFilters, "message-id", "odd", "stop");
+  // This shouldn't be hit, because of the previous filter
+  createFilter(folderFilters, "message-id", "6.odd", "read");
+  createFilter(folderFilters, "user-agent", "Odd/1.0", "tag");
+  folderFilters.loggingEnabled = true;
+  folder.setFilterList(folderFilters);
+
+  handlers.forEach( function (handler) {
+    var handlerObj = new handler(daemon);
+    testAction(handlerObj, localserver);
+  });
+}
--- a/mailnews/news/test/unit/test_server.js
+++ b/mailnews/news/test/unit/test_server.js
@@ -1,17 +1,16 @@
 ////////////////////////////////////////////////////////////////////////////////
 // Protocol tests for NNTP. These actually aren't too important, but their main
 // purpose is to make sure that maild is working properly and to provide
 // examples for how using maild. They also help make sure that I coded nntpd.js
 // right, both logically and for RFC compliance.
 // TODO:
 // * We need to hook up mochitest,
 // * TLS negotiation.
-// * Filter tests... very important (see UI requests)
 ////////////////////////////////////////////////////////////////////////////////
 
 // The basic daemon to use for testing nntpd.js implementations
 var daemon = setupNNTPDaemon();
 
 // Define these up here for checking with the transaction
 var type = null;
 var test = null;
@@ -22,20 +21,16 @@ function do_check_transaction(real, expe
   // excise this from the list
   if (real.them[real.them.length-1] == "QUIT")
     real.them.pop();
 
   do_check_eq(real.them.join(","), expected.join(","));
   dump("Passed test " + test + "\n");
 }
 
-// Make sure we don't try to use a protected port. I like adding 1024 to the
-// default port when doing so...
-const NNTP_PORT = 1024+119;
-
 ////////////////////////////////////////////////////////////////////////////////
 //                             NNTP SERVER TESTS                              //
 ////////////////////////////////////////////////////////////////////////////////
 // Functions in order as defined in nntpd.js. Each function tests the URLs    //
 // that are located over the implementation of nsNNTPProtocol::LoadURL and    //
 // added in bug 400331. Furthermore, they are tested in rough order as they   //
 // would be expected to be used in a session. If more URL types are modified, //
 // please add a corresponding type to the following tests.                    //
@@ -71,18 +66,19 @@ function testRFC977() {
     do_check_transaction(transaction, []);*/
 
     // Test - newsrc
     test = "news:";
     server.resetTest();
     setupProtocolTest(NNTP_PORT, prefix+"");
     server.performTest();
     transaction = server.playTransaction();
-    do_check_transaction(transaction, ["MODE READER",
-        "GROUP test.subscribe.empty", "GROUP test.subscribe.simple"]);
+    do_check_transaction(transaction, ["MODE READER"].concat(
+          groups.filter(function (group) { return group[1]; })
+                .map(function (group) { return "GROUP "+group[0]; })));
 
     // Test - getting an article
     test = "news:MESSAGE_ID";
     server.resetTest();
     setupProtocolTest(NNTP_PORT, prefix+"TSS1@nntp.test");
     server.performTest();
     transaction = server.playTransaction();
     do_check_transaction(transaction, ["MODE READER",
--- a/mailnews/test/fakeserver/nntpd.js
+++ b/mailnews/test/fakeserver/nntpd.js
@@ -5,22 +5,34 @@ function nntpDaemon(flags) {
   this._messages = {};
   this._flags = flags;
 }
 nntpDaemon.prototype = {
   addGroup : function(group, postable) {
     var flags = 0;
     if (postable)
       flags |= NNTP_POSTABLE;
-    this._groups[group] = { keys : [], flags : flags};
+    this._groups[group] = { keys : [], flags : flags, nextKey : 1};
+  },
+  addArticle : function (article) {
+   this._messages[article.messageID] = article;
+   for each (var group in article.groups) {
+     if (group in this._groups) {
+       var key = this._groups[group].nextKey++;
+       this._groups[group][key] = article;
+       this._groups[group]['keys'].push(key);
+     }
+   }
   },
   addArticleToGroup : function(article, group, key) {
     this._groups[group][key] = article;
     this._messages[article.messageID] = article;
     this._groups[group]['keys'].push(key);
+    if (this._groups[group].nextKey <= key)
+      this._groups[group].nextKey = key+1;
   },
   getGroup : function(group) {
     if (this._groups.hasOwnProperty(group))
       return this._groups[group];
     return null;
   },
   getGroupStats : function (group) {
     if (group['keys'].length == 0)
@@ -46,34 +58,53 @@ nntpDaemon.prototype = {
     return null;
   }
 }
 
 function newsArticle(text) {
   this.headers = {};
   this.body = "";
   this.messageID = "";
+  this.fullText = text;
 
   var lines = text.split("\n"), passedHeaders = false;
+  var preamble = "";
   for each(var line in lines) {
     if (!passedHeaders) {
       if (line.length == 0) {
         passedHeaders = true;
         continue;
       }
-      var parts = text.split(":[ \t]*");
-      this.headers[parts[0]] = parts[1];
+      preamble += line + '\n';
+      var parts = line.split(/:[ \t]*/);
+      this.headers[parts[0].toLowerCase()] = parts[1];
       switch (parts[0].toLowerCase()) {
         case "message-id":
-          this.messageID = parts[1];
+          var start = parts[1].indexOf('<');
+          var end = parts[1].indexOf('>', start);
+          this.messageID = parts[1].substring(start, end+1);
+          break;
+        case "newsgroups":
+          this.groups = parts[1].split(/[ \t]*,[ \t]*/);
+          break;
       }
     } else {
       this.body += line + "\n";
     }
   }
+
+  // Add in non-existent fields
+  if (!("lines" in this.headers))
+  {
+    let lines = this.body.split('\n').length;
+    this.headers["lines"] = lines;
+    preamble += "Lines: "+lines;
+  }
+
+  this.fulltext = preamble + '\n' + this.body;
 }
 
 // NNTP FLAGS
 const NNTP_POSTABLE = 0x0001;
 
 const NNTP_REAL_LENGTH = 0x0100;
 
 function hasFlag(flags, flag) {
@@ -85,141 +116,77 @@ function hasFlag(flags, flag) {
 ////////////////////////////////////////////////////////////////////////////////
 // To be comprehensive about testing and fallback, we define these varying    //
 // levels of RFC-compliance:                                                  //
 // * RFC 977 solely (there's not a lot there!)                                //
 // * RFC 977 + 2980 (note that there are varying levels of this impl)         //
 // * RFC 3977 bare bones                                                      //
 // * RFC 3977 full                                                            //
 // * RFC 3977 + post-3977 extensions                                          //
-// * INN 2.4 (Gold standard common implementation; highest importance)        //
-// * Giganews (Common newsserver for ISP stuff; second highest importance)    //
+// * Giganews (Common newsserver for ISP stuff; highest importance)           //
+// * INN 2.4 (Gold standard common implementation; second highest importance) //
 // Note too that we want various levels of brokenness:                        //
 // * Perm errors that require login                                           //
 // * "I can't handle that" (e.g., news.mozilla.org only supports XOVER for    //
 //   searching with XHDR)                                                     //
 // * Naive group counts, missing articles                                     //
 // * Limitations on what can be posted                                        //
 ////////////////////////////////////////////////////////////////////////////////
 
 
 // This handler implements the bare minimum required by RFC 977. Actually, not
 // even that much: IHAVE and SLAVE are not implemented, as those two are
 // explicitly server implementations.
 function NNTP_RFC977_handler(daemon) {
   this._daemon = daemon;
   this.closing = false;
   this.group = null;
-  this.article = null;
+  this.articleKey = null;
 }
 NNTP_RFC977_handler.prototype = {
   ARTICLE : function (args) {
-     var art;
-     if (args == "") {
-       if (this.article == null)
-         return "420 no current article has been selected";
-
-       art = this.article;
-     } else if (args.charAt(0) == '<') {
-       art = this._daemon.getArticle(args);
-       if (art == null)
-         return "430 no such article found";
-     } else {
-       if (this.group == null)
-         return "412 no newsgroup has been selected";
+     var info = this._selectArticle(args, 220);
+     if (info[0] == null)
+       return info[1];
 
-       var index = Integer.parseInt(args);
-       if (index in this.group.keys) {
-         this.article = this.group.keys[index];
-         art = this.article;
-       } else {
-         return "423 no such article number in this group";
-       }
-     }
-
-     var response = "220 " + art.key + " " + art.messageID +
-                    " article retrieved - head and body follows.\n";
-     for (var header in art.headers) {
-       response += header + ": " + art.headers[header] + "\n";
-     }
-     response += art.body.replace("(?=\n).", "..");
+     var response = info[1]+'\n';
+     response += info[0].fullText.replace("(?=\n).", "..");
      response += ".";
      return response;
   },
   BODY : function (args) {
-     var art;
-     if (args == "") {
-       if (this.article == null)
-         return "420 no current article has been selected";
-
-       art = this.article;
-     } else if (args.charAt(0) == '<') {
-       art = this._daemon.getArticle(args);
-       if (article == null)
-         return "430 no such article found";
-     } else {
-       if (this.group == null)
-         return "412 no newsgroup has been selected";
+     var info = this._selectArticle(args, 222);
+     if (info[0] == null)
+       return info[1];
 
-       var index = Integer.parseInt(args);
-       if (index in this.group.keys) {
-         this.article = this.group.keys[index];
-         art = this.article;
-       } else {
-         return "423 no such article number in this group";
-       }
-     }
-
-     var response = "222 "+art.key+" "+art.messageID+
-                    " article retrieved - body follows.\n";
-     response += "\n";
-     response += art.body.replace("(?=\n).","..");
+     var response = info[1]+'\n';
+     response += info[0].body.replace("(?=\n).","..");
      response += ".";
      return response;
   },
   GROUP : function(args) {
     var group = this._daemon.getGroup(args);
     if (group == null)
       return "411 no such news group";
 
-    this.article = null;
+    this.group = group;
+    this.articleKey = 0 in this.group.keys ? this.group.keys[0] : null;
 
     var stats = this._daemon.getGroupStats(group);
     return "211 " + stats[0] + " " + stats[1] + " " + stats[2] + " " + args +
            " group selected";
   },
   HEAD : function (args) {
-     var art;
-     if (args == "") {
-       if (this.article == null)
-         return "420 no current article has been selected";
-
-       art = this.article;
-     } else if (args.charAt(0) == '<') {
-       art = this._daemon.getArticle(args);
-       if (art == null)
-         return "430 no such article found";
-     } else {
-       if (this.group == null)
-         return "412 no newsgroup has been selected";
+     var info = this._selectArticle(args, 221);
+     if (info[0] == null)
+       return info[1];
 
-       var index = Integer.parseInt(args);
-       if (index in this.group.keys) {
-         this.article = this.group.keys[index];
-         art = this.article;
-       } else {
-         return "423 no such article number in this group";
-       }
-     }
-
-     var response = "221 " + art.key + " " + art.messageID +
-                    " article retrieved - head follows.\n";
-     for (var header in art.headers) {
-       response += header + ": " + article.headers[header] + "\n";
-     }
+     var response = info[1]+'\n';
+     for (var header in info[0].headers)
+       response += header + ": " + info[0].headers[header] + "\n";
      response += ".";
      return response;
   },
   HELP : function (args) {
     var response = "100 Why certainly, here is my help:\n";
     response += "Mozilla fake NNTP RFC 977 testing server";
     response += "Commands supported:\n";
     response += "\tARTICLE <message-id> | [nnn]\n";
@@ -234,19 +201,19 @@ NNTP_RFC977_handler.prototype = {
     response += "\tNEXT\n";
     response += "\tPOST\n";
     response += "\tQUIT\n";
     response += "\tSTAT\n";
     response += ".";
     return response;
   },
   LAST : function (args) {
-    if (group == null)
+    if (this.group == null)
       return "412 no newsgroup selected";
-    if (article == null)
+    if (this.articleKey == null)
       return "420 no current article has been selected";
     return "502 Command not implemented";
   },
   LIST : function (args) {
     var response = "215 list of newsgroup follows\n";
     for (group in this._daemon._groups) {
       var stats = this._daemon.getGroupStats(this._daemon._groups[group]);
       response += group + " " + stats[1] + " " + stats[0] + " " +
@@ -257,67 +224,44 @@ NNTP_RFC977_handler.prototype = {
   },
   NEWGROUPS : function (args) {
     return "502 Command not implemented";
   },
   NEWNEWS : function (args) {
     return "502 Command not implemented";
   },
   NEXT : function (args) {
-    if (group == null)
+    if (this.group == null)
       return "412 no newsgroup selected";
-    if (article == null)
+    if (this.articleKey == null)
       return "420 no current article has been selected";
     return "502 Command not implemented";
   },
   POST : function(args) {
     this.posting = true;
     this.post = "";
     return "340 Please continue";
   },
   QUIT : function(args) {
     this.closing = true;
     return "205 closing connection - goodbye!";
   },
   STAT : function (args) {
-     var art;
-     if (args == "") {
-       if (this.article == null)
-         return "420 no current article has been selected";
-
-       art = this.article;
-     } else if (args.charAt(0) == '<') {
-       art = this._daemon.getArticle(args);
-       if (article == null)
-         return "430 no such article found";
-     } else {
-       if (this.group == null)
-         return "412 no newsgroup has been selected";
-
-       var index = Integer.parseInt(args);
-       if (index in this.group.keys) {
-         this.article = this.group.keys[index];
-         art = this.article;
-       } else {
-         return "423 no such article number in this group";
-       }
-     }
-
-     return "223 " + art.key + " " + art.messageID +
-            " article retrieved - request text separately.";
+     var info = this._selectArticle(args, 223);
+     return info[1];
   },
   LISTGROUP : function (args) {
     // Yes, I know this isn't RFC 977, but I doubt that mailnews will ever drop
     // its requirement for this, so I'll stuff it in here anyways...
     var group = (args == "" ? this.group : this._daemon.getGroup(args));
     if (group == null)
       return "411 This newsgroup does not exist";
 
     var response = "211 Articles follow:\n";
-    for (var key in group['keys'])
+    for each (var key in group['keys'])
       response += key + "\n";
     response += ".\n";
     return response;
   },
 
 
   onError : function (command, args) {
     return "500 command not recognized";
@@ -328,16 +272,17 @@ NNTP_RFC977_handler.prototype = {
     this.article = null;
     this.posting = false;
     return "200 posting allowed";
   },
   onMultiline : function (line) {
     if (line == ".") {
       if (this.posting) {
         var article = new newsArticle(this.post);
+        this._daemon.addArticle(article);
         this.posting = false;
         return "240 Wonderful article, your style is gorgeous!";
       }
     }
 
     if (this.posting) {
       if (line[0] == '.')
         line = line.substring(1);
@@ -346,10 +291,156 @@ NNTP_RFC977_handler.prototype = {
     }
 
     return undefined;
   },
   postCommand : function (obj) {
     if (this.closing)
       obj.closeSocket();
     obj.setMultiline(this.posting);
+  },
+
+  /**
+   * Selects an article based on args.
+   *
+   * Returns an array of objects consisting of:
+   * # The selected article (or null if non was selected
+   * # The first line response
+   */
+  _selectArticle : function (args, responseCode) {
+    var art, key;
+    if (args == "") {
+      if (this.group == null)
+        return [null, "412 no newsgroup has been selected"];
+      if (this.articleKey == null)
+        return [null, "420 no current article has been selected"];
+
+      art = this.group[this.articleKey];
+      key = this.articleKey;
+    } else if (args.charAt(0) == '<') {
+      art = this._daemon.getArticle(args);
+      key = 0;
+
+      if (art == null)
+        return [null, "430 no such article found"];
+    } else {
+      if (this.group == null)
+        return [null, "412 no newsgroup has been selected"];
+
+      key = parseInt(args);
+      if (key in this.group) {
+        this.articleKey = key;
+        art = this.group[key];
+      } else {
+        return [null, "423 no such article number in this group"];
+      }
+    }
+
+    var respCode = responseCode + " " + key + " " + art.messageID +
+      " article selected";
+    return [art, respCode];
+  }
+}
+
+/**
+ * Utility method to define a subclass
+ *
+ * @param sub   The function object of the subclass
+ * @param super The function object of the superclass
+ * @param def   The object definition of the subclass prototype.
+ */
+function subclass(sub, sup, def) {
+  sub.prototype = new sup();
+  for (var obj in def) {
+    sub.prototype[obj] = def[obj];
+  }
+}
+function subconstructor(sub, sup) {
+  sup.apply(sub, Array.prototype.slice.call(arguments, 2));
+  sub.parent = new Object();
+  sub.parent.__noSuchMethod__ = function (name, args) {
+    return sup.prototype[name].apply(sub, args);
   }
 }
+function NNTP_RFC2980_handler(daemon) {
+  subconstructor(this, NNTP_RFC977_handler, daemon);
+}
+subclass(NNTP_RFC2980_handler, NNTP_RFC977_handler, {
+//NNTP_RFC2980_handler.prototype = new NNTP_RFC977_handler();
+//var subprototype = {
+  DATE : function (args) {
+    return "502 Command not implemented";
+  },
+  LIST : function (args) {
+    var index = args.indexOf(" ");
+    var command = index == -1 ? args : args.substring(0,index);
+    args = index == -1 ? "" : args.substring(index+1);
+    command = command.toUpperCase();
+    if ("LIST_"+command in this)
+      return this["LIST_"+command](args);
+    return this.parent.LIST(command+" "+args);
+  },
+  LIST_ACTIVE : function (args) {
+    return this.parent.LIST(args);
+  },
+  MODE : function (args) {
+    if (args == "READER")
+      return this.onStartup();
+    return "500 What do you think you're trying to pull here?";
+  },
+  XHDR : function (args) {
+    if (!this.group)
+      return "412 No group selected";
+
+    args = args.split(" ");
+    var header = args[0].toLowerCase();
+    var found = false;
+    var response = "221 Headers abound\n";
+    for each (var key in this.group.keys) {
+      if (!(header in this.group[key].headers))
+        continue;
+      found = true;
+      response += key + " " +this.group[key].headers[header] + '\n';
+    }
+    if (!found)
+      return "420 No such article";
+    response += '.';
+    return response;
+  },
+  XOVER : function (args) {
+    if (!this.group)
+      return "412 No group selected";
+    
+    var response = "224 List of articles\n";
+    for each (var key in this.group.keys) {
+      response += key + "\t";
+      var article = this.group[key];
+      response += article.headers["subject"] + "\t" +
+                  article.headers["from"] + "\t" +
+                  article.headers["date"] + "\t" +
+                  article.headers["message-id"] + "\t" +
+                  (article.headers["references"] ? article.headers["references"]
+                                                : "") + "\t" +
+                  article.fullText.replace(/\r?\n/,'\r\n').length + "\t" +
+                  article.body.split(/\r?\n/).length + "\t" +
+                  article.headers["xref"] + "\n";
+    }
+    response += '.\n';
+    return response;
+  },
+  XPAT : function (args) {
+    return "502 Command not implemented";
+  }
+});
+
+function NNTP_Giganews_handler(daemon) {
+  subconstructor(this, NNTP_RFC2980_handler, daemon);
+}
+subclass(NNTP_Giganews_handler, NNTP_RFC2980_handler, {
+  XHDR : function (args) {
+    var header = args.split(" ")[0].toLowerCase();
+    if (header in ["subject", "from", "xref", "date", "message-id",
+                   "references"]) {
+      return this.parent.XHDR(args);
+    }
+    return "503 unsupported header field";
+  }
+});
--- a/suite/locales/en-US/chrome/mailnews/news.properties
+++ b/suite/locales/en-US/chrome/mailnews/news.properties
@@ -48,16 +48,17 @@ cancelConfirm=Are you sure you want to c
 messageCancelled=Message cancelled.
 enterUsername=Please enter a username for news server access:
 enterPassword=Please enter a password for news server access:
 enterPasswordTitle=News Server Password Required
 okButtonText=Download
 
 noNewMessages=There are no new messages on the server.
 downloadingHeaders=Downloading %S of %S headers
+downloadingFilterHeaders=Getting headers for filters: %S (%S/%S)
 downloadingArticles=Downloading articles %S-%S
 bytesReceived=Downloading newsgroups: %S received (%SKB read at %SKB/sec)
 checkingForNewNews=Checking newsgroup %S of %S on %S for new messages
 downloadingArticlesForOffline=Downloading articles %S-%S in %S
 
 onlyCancelOneMessage=You can only cancel one article at a time.
 
 # LOCALIZATION NOTE (autoUnsubscribeText): %1$S is the newsgroup and %2$S is the newsgroup-server it is being removed from.