mailnews/imap/src/nsImapProtocol.cpp
author Magnus Melin <mkmelin+mozilla@iki.fi>
Sat, 06 Oct 2018 21:38:58 +0200
changeset 33321 cd7b757fff58ee7e59982c990223d0dda379a7e2
parent 33307 f77c18acceafaa7e9fed65ab27e293cf117850a4
child 33327 d3a3e12d2654959356b71569177b6ede3790fd9e
permissions -rw-r--r--
Bug 825513 - Fix crash in nsImapProtocol::GetMessageSize() by adding null check for m_runningUrl && m_hostSessionList. r=jorgk

/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// as does this
#include "msgCore.h"  // for pre-compiled headers
#include "nsMsgUtils.h"

#include "nsImapStringBundle.h"
#include "nsVersionComparator.h"

#include "nsMsgImapCID.h"
#include "nsThreadUtils.h"
#include "nsIMsgStatusFeedback.h"
#include "nsImapCore.h"
#include "nsImapProtocol.h"
#include "nsIMsgMailNewsUrl.h"
#include "nsIIMAPHostSessionList.h"
#include "nsIMAPBodyShell.h"
#include "nsImapMailFolder.h"
#include "nsIMsgAccountManager.h"
#include "nsImapServerResponseParser.h"
#include "nspr.h"
#include "plbase64.h"
#include "nsIImapService.h"
#include "nsISocketTransportService.h"
#include "nsIStreamListenerTee.h"
#include "nsIInputStreamPump.h"
#include "nsNetUtil.h"
#include "nsIDBFolderInfo.h"
#include "nsIPipe.h"
#include "nsIMsgFolder.h"
#include "nsMsgMessageFlags.h"
#include "nsTextFormatter.h"
#include "nsIMsgHdr.h"
#include "nsMsgI18N.h"
// for the memory cache...
#include "nsICacheEntry.h"
#include "nsICacheStorage.h"
#include "nsICacheEntryOpenCallback.h"
#include "nsIURIMutator.h"

#include "nsIDocShell.h"
#include "nsILoadInfo.h"
#include "nsCOMPtr.h"
#include "nsMimeTypes.h"
#include "nsIInterfaceRequestor.h"
#include "nsXPCOMCIDInternal.h"
#include "nsIXULAppInfo.h"
#include "nsSyncRunnableHelpers.h"
#include "nsICancelable.h"

// netlib required files
#include "nsIStreamListener.h"
#include "nsIMsgIncomingServer.h"
#include "nsIImapIncomingServer.h"
#include "nsIPrefBranch.h"
#include "nsIPrefService.h"
#include "nsIPrefLocalizedString.h"
#include "nsImapUtils.h"
#include "nsIStreamConverterService.h"
#include "nsIProxyInfo.h"
#include "nsISSLSocketControl.h"
#include "nsProxyRelease.h"
#include "nsDebug.h"
#include "nsMsgCompressIStream.h"
#include "nsMsgCompressOStream.h"
#include "mozilla/Logging.h"
#include "mozilla/Attributes.h"
#include "mozilla/SlicedInputStream.h"
#include "nsIPrincipal.h"
#include "nsContentSecurityManager.h"

// imap event sinks
#include "nsIImapMailFolderSink.h"
#include "nsIImapServerSink.h"
#include "nsIImapMessageSink.h"

using namespace mozilla;

LazyLogModule IMAP("IMAP");
LazyLogModule IMAP_CS("IMAP_CS");

#define ONE_SECOND ((uint32_t)1000)    // one second

#define OUTPUT_BUFFER_SIZE (4096*2) // mscott - i should be able to remove this if I can use nsMsgLineBuffer???

#define IMAP_ENV_HEADERS "From To Cc Bcc Subject Date Message-ID "
#define IMAP_DB_HEADERS "Priority X-Priority References Newsgroups In-Reply-To Content-Type Reply-To"
#define IMAP_ENV_AND_DB_HEADERS IMAP_ENV_HEADERS IMAP_DB_HEADERS
static const PRIntervalTime kImapSleepTime = PR_MillisecondsToInterval(60000);
static int32_t gPromoteNoopToCheckCount = 0;
static const uint32_t kFlagChangesBeforeCheck = 10;
static const int32_t kMaxSecondsBeforeCheck = 600;

class AutoProxyReleaseMsgWindow
{
  public:
    AutoProxyReleaseMsgWindow()
      : mMsgWindow()
    {}
    ~AutoProxyReleaseMsgWindow()
    {
      NS_ReleaseOnMainThreadSystemGroup("AutoProxyReleaseMsgWindow::mMsgWindow", dont_AddRef(mMsgWindow));
    }
    nsIMsgWindow** StartAssignment()
    {
      MOZ_ASSERT(!mMsgWindow);
      return &mMsgWindow;
    }
    operator nsIMsgWindow*()
    {
      return mMsgWindow;
    }
  private:
    nsIMsgWindow* mMsgWindow;
};

nsIMsgWindow**
getter_AddRefs(AutoProxyReleaseMsgWindow& aSmartPtr)
{
  return aSmartPtr.StartAssignment();
}

NS_IMPL_ISUPPORTS(nsMsgImapHdrXferInfo, nsIImapHeaderXferInfo)

nsMsgImapHdrXferInfo::nsMsgImapHdrXferInfo()
  : m_hdrInfos(kNumHdrsToXfer)
{
  m_nextFreeHdrInfo = 0;
}

nsMsgImapHdrXferInfo::~nsMsgImapHdrXferInfo()
{
}

NS_IMETHODIMP nsMsgImapHdrXferInfo::GetNumHeaders(int32_t *aNumHeaders)
{
  *aNumHeaders = m_nextFreeHdrInfo;
  return NS_OK;
}

NS_IMETHODIMP nsMsgImapHdrXferInfo::GetHeader(int32_t hdrIndex, nsIImapHeaderInfo **aResult)
{
  // If the header index is more than (or equal to) our next free pointer, then
  // its a header we haven't really got and the caller has done something
  // wrong.
  NS_ENSURE_TRUE(hdrIndex < m_nextFreeHdrInfo, NS_ERROR_NULL_POINTER);

  NS_IF_ADDREF(*aResult = m_hdrInfos.SafeObjectAt(hdrIndex));
  if (!*aResult)
    return NS_ERROR_NULL_POINTER;
  return NS_OK;
}

static const int32_t kInitLineHdrCacheSize = 512; // should be about right

nsIImapHeaderInfo* nsMsgImapHdrXferInfo::StartNewHdr()
{
  if (m_nextFreeHdrInfo >= kNumHdrsToXfer)
    return nullptr;

  nsIImapHeaderInfo *result = m_hdrInfos.SafeObjectAt(m_nextFreeHdrInfo++);
  if (result)
    return result;

  nsMsgImapLineDownloadCache *lineCache = new nsMsgImapLineDownloadCache();
  if (!lineCache)
    return nullptr;

  lineCache->GrowBuffer(kInitLineHdrCacheSize);

  m_hdrInfos.AppendObject(lineCache);

  return lineCache;
}

// maybe not needed...
void nsMsgImapHdrXferInfo::FinishCurrentHdr()
{
  // nothing to do?
}

void nsMsgImapHdrXferInfo::ResetAll()
{
  int32_t count = m_hdrInfos.Count();
  for (int32_t i = 0; i < count; i++)
  {
    nsIImapHeaderInfo *hdrInfo = m_hdrInfos[i];
    if (hdrInfo)
      hdrInfo->ResetCache();
  }
  m_nextFreeHdrInfo = 0;
}

void nsMsgImapHdrXferInfo::ReleaseAll()
{
  m_hdrInfos.Clear();
  m_nextFreeHdrInfo = 0;
}

NS_IMPL_ISUPPORTS(nsMsgImapLineDownloadCache, nsIImapHeaderInfo)

// **** helper class for downloading line ****
nsMsgImapLineDownloadCache::nsMsgImapLineDownloadCache()
{
    fLineInfo = (msg_line_info *) PR_CALLOC(sizeof( msg_line_info));
    fLineInfo->uidOfMessage = nsMsgKey_None;
    m_msgSize = 0;
}

nsMsgImapLineDownloadCache::~nsMsgImapLineDownloadCache()
{
    PR_Free( fLineInfo);
}

uint32_t nsMsgImapLineDownloadCache::CurrentUID()
{
    return fLineInfo->uidOfMessage;
}

uint32_t nsMsgImapLineDownloadCache::SpaceAvailable()
{
    return kDownLoadCacheSize - m_bufferPos;
}

msg_line_info *nsMsgImapLineDownloadCache::GetCurrentLineInfo()
{
  AppendBuffer("", 1); // null terminate the buffer
  fLineInfo->adoptedMessageLine = GetBuffer();
  return fLineInfo;
}

NS_IMETHODIMP nsMsgImapLineDownloadCache::ResetCache()
{
    ResetWritePos();
    return NS_OK;
}

bool nsMsgImapLineDownloadCache::CacheEmpty()
{
    return m_bufferPos == 0;
}

NS_IMETHODIMP nsMsgImapLineDownloadCache::CacheLine(const char *line, uint32_t uid)
{
    NS_ASSERTION((PL_strlen(line) + 1) <= SpaceAvailable(),
                 "Oops... line length greater than space available");

    fLineInfo->uidOfMessage = uid;

    AppendString(line);
    return NS_OK;
}

/* attribute nsMsgKey msgUid; */
NS_IMETHODIMP nsMsgImapLineDownloadCache::GetMsgUid(nsMsgKey *aMsgUid)
{
    *aMsgUid = fLineInfo->uidOfMessage;
    return NS_OK;
}
NS_IMETHODIMP nsMsgImapLineDownloadCache::SetMsgUid(nsMsgKey aMsgUid)
{
    fLineInfo->uidOfMessage = aMsgUid;
    return NS_OK;
}

/* attribute long msgSize; */
NS_IMETHODIMP nsMsgImapLineDownloadCache::GetMsgSize(int32_t *aMsgSize)
{
    *aMsgSize = m_msgSize;
    return NS_OK;
}

NS_IMETHODIMP nsMsgImapLineDownloadCache::SetMsgSize(int32_t aMsgSize)
{
    m_msgSize = aMsgSize;
    return NS_OK;
}

/* attribute string msgHdrs; */
NS_IMETHODIMP nsMsgImapLineDownloadCache::GetMsgHdrs(const char **aMsgHdrs)
{
  // this doesn't copy the string
    AppendBuffer("", 1); // null terminate the buffer
    *aMsgHdrs = GetBuffer();
    return NS_OK;
}

/* the following macros actually implement addref, release and query interface for our component. */

NS_IMPL_ADDREF_INHERITED(nsImapProtocol, nsMsgProtocol)
NS_IMPL_RELEASE_INHERITED(nsImapProtocol, nsMsgProtocol )

NS_INTERFACE_MAP_BEGIN(nsImapProtocol)
   NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIImapProtocol)
   NS_INTERFACE_MAP_ENTRY(nsIRunnable)
   NS_INTERFACE_MAP_ENTRY(nsIImapProtocol)
   NS_INTERFACE_MAP_ENTRY(nsIProtocolProxyCallback)
   NS_INTERFACE_MAP_ENTRY(nsIInputStreamCallback)
   NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference)
   NS_INTERFACE_MAP_ENTRY(nsIImapProtocolSink)
   NS_INTERFACE_MAP_ENTRY(nsIMsgAsyncPromptListener)
NS_INTERFACE_MAP_END

static int32_t gTooFastTime = 2;
static int32_t gIdealTime = 4;
static int32_t gChunkAddSize = 16384;
static int32_t gChunkSize = 250000;
static int32_t gChunkThreshold = gChunkSize + gChunkSize/2;
static bool gChunkSizeDirty = false;
static bool gFetchByChunks = true;
static bool gInitialized = false;
static bool gHideUnusedNamespaces = true;
static bool gHideOtherUsersFromList = false;
static bool gUseEnvelopeCmd = false;
static bool gUseLiteralPlus = true;
static bool gExpungeAfterDelete = false;
static bool gCheckDeletedBeforeExpunge = false; //bug 235004
static int32_t gResponseTimeout = 60;
static nsCString gForceSelectDetect;
static nsTArray<nsCString> gForceSelectServersArray;

// let delete model control expunging, i.e., don't ever expunge when the
// user chooses the imap delete model, otherwise, expunge when over the
// threshold. This is the normal TB behavior.
static const int32_t kAutoExpungeDeleteModel = 0;
// Expunge whenever the folder is opened
static const int32_t kAutoExpungeAlways = 1;
// Expunge when over the threshold, independent of the delete model.
static const int32_t kAutoExpungeOnThreshold = 2;
static int32_t gExpungeOption = kAutoExpungeDeleteModel;
static int32_t gExpungeThreshold = 20;

const int32_t kAppBufSize = 100;
// can't use static nsCString because it shows up as a leak.
static char gAppName[kAppBufSize];
static char gAppVersion[kAppBufSize];

nsresult nsImapProtocol::GlobalInitialization(nsIPrefBranch *aPrefBranch)
{
    gInitialized = true;

    aPrefBranch->GetIntPref("mail.imap.chunk_fast", &gTooFastTime);   // secs we read too little too fast
    aPrefBranch->GetIntPref("mail.imap.chunk_ideal", &gIdealTime);    // secs we read enough in good time
    aPrefBranch->GetIntPref("mail.imap.chunk_add", &gChunkAddSize);   // buffer size to add when wasting time
    aPrefBranch->GetIntPref("mail.imap.chunk_size", &gChunkSize);
    aPrefBranch->GetIntPref("mail.imap.min_chunk_size_threshold",
                            &gChunkThreshold);
    aPrefBranch->GetBoolPref("mail.imap.hide_other_users",
                             &gHideOtherUsersFromList);
    aPrefBranch->GetBoolPref("mail.imap.hide_unused_namespaces",
                             &gHideUnusedNamespaces);
    aPrefBranch->GetIntPref("mail.imap.noop_check_count",
                            &gPromoteNoopToCheckCount);
    aPrefBranch->GetBoolPref("mail.imap.use_envelope_cmd",
                             &gUseEnvelopeCmd);
    aPrefBranch->GetBoolPref("mail.imap.use_literal_plus", &gUseLiteralPlus);
    aPrefBranch->GetBoolPref("mail.imap.expunge_after_delete",
                             &gExpungeAfterDelete);
    aPrefBranch->GetBoolPref("mail.imap.check_deleted_before_expunge",
                             &gCheckDeletedBeforeExpunge);
    aPrefBranch->GetIntPref("mail.imap.expunge_option", &gExpungeOption);
    aPrefBranch->GetIntPref("mail.imap.expunge_threshold_number",
                            &gExpungeThreshold);
    aPrefBranch->GetIntPref("mailnews.tcptimeout", &gResponseTimeout);
    aPrefBranch->GetCharPref("mail.imap.force_select_detect",
                             gForceSelectDetect);
    ParseString(gForceSelectDetect, ';', gForceSelectServersArray);

    nsCOMPtr<nsIXULAppInfo> appInfo(do_GetService(XULAPPINFO_SERVICE_CONTRACTID));

    if (appInfo)
    {
      nsCString appName, appVersion;
      appInfo->GetName(appName);
      appInfo->GetVersion(appVersion);
      PL_strncpyz(gAppName, appName.get(), kAppBufSize);
      PL_strncpyz(gAppVersion, appVersion.get(), kAppBufSize);
    }
    return NS_OK;
}

nsImapProtocol::nsImapProtocol() : nsMsgProtocol(nullptr),
    m_dataAvailableMonitor("imapDataAvailable"),
    m_urlReadyToRunMonitor("imapUrlReadyToRun"),
    m_pseudoInterruptMonitor("imapPseudoInterrupt"),
    m_dataMemberMonitor("imapDataMember"),
    m_threadDeathMonitor("imapThreadDeath"),
    m_waitForBodyIdsMonitor("imapWaitForBodyIds"),
    m_fetchBodyListMonitor("imapFetchBodyList"),
    m_passwordReadyMonitor("imapPasswordReady"),
    mLock("nsImapProtocol.mLock"),
    m_parser(*this)
{
  m_urlInProgress = false;
  m_idle = false;
  m_retryUrlOnError = false;
  m_useIdle = true; // by default, use it
  m_useCondStore = true;
  m_useCompressDeflate = true;
  m_ignoreExpunges = false;
  m_prefAuthMethods = kCapabilityUndefined;
  m_failedAuthMethods = 0;
  m_currentAuthMethod = kCapabilityUndefined;
  m_socketType = nsMsgSocketType::trySTARTTLS;
  m_connectionStatus = NS_OK;
  m_safeToCloseConnection = false;
  m_hostSessionList = nullptr;
  m_fetchBodyIdList = nullptr;
  m_isGmailServer = false;
  m_fetchingWholeMessage = false;

  nsCOMPtr<nsIPrefBranch> prefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID));
  NS_ASSERTION(prefBranch, "FAILED to create the preference service");

  // read in the accept languages preference
  if (prefBranch)
  {
    if (!gInitialized)
      GlobalInitialization(prefBranch);

    nsCOMPtr<nsIPrefLocalizedString> prefString;
    prefBranch->GetComplexValue("intl.accept_languages",
                                NS_GET_IID(nsIPrefLocalizedString),
                                getter_AddRefs(prefString));
    if (prefString)
      prefString->ToString(getter_Copies(mAcceptLanguages));

    nsCString customDBHeaders;
    prefBranch->GetCharPref("mailnews.customDBHeaders", customDBHeaders);

    ParseString(customDBHeaders, ' ', mCustomDBHeaders);
    prefBranch->GetBoolPref("mailnews.display.prefer_plaintext", &m_preferPlainText);

    nsAutoCString customHeaders;
    prefBranch->GetCharPref("mailnews.customHeaders", customHeaders);
    customHeaders.StripWhitespace();
    ParseString(customHeaders, ':', mCustomHeaders);
  }

    // ***** Thread support *****
  m_thread = nullptr;
  m_imapThreadIsRunning = false;
  m_currentServerCommandTagNumber = 0;
  m_active = false;
  m_folderNeedsSubscribing = false;
  m_folderNeedsACLRefreshed = false;
  m_threadShouldDie = false;
  m_inThreadShouldDie = false;
  m_pseudoInterrupted = false;
  m_nextUrlReadyToRun = false;
  m_trackingTime = false;
  m_curFetchSize = 0;
  m_startTime = 0;
  m_endTime = 0;
  m_lastActiveTime = 0;
  m_lastProgressTime = 0;
  ResetProgressInfo();

  m_tooFastTime = 0;
  m_idealTime = 0;
  m_chunkAddSize = 0;
  m_chunkStartSize = 0;
  m_fetchByChunks = true;
  m_sendID = true;
  m_chunkSize = 0;
  m_chunkThreshold = 0;
  m_fromHeaderSeen = false;
  m_closeNeededBeforeSelect = false;
  m_needNoop = false;
  m_noopCount = 0;
  m_fetchBodyListIsNew = false;
  m_flagChangeCount = 0;
  m_lastCheckTime = PR_Now();

  m_hierarchyNameState = kNoOperationInProgress;
  m_discoveryStatus = eContinue;

  // m_dataOutputBuf is used by Send Data
  m_dataOutputBuf = (char *) PR_CALLOC(sizeof(char) * OUTPUT_BUFFER_SIZE);
  m_allocatedSize = OUTPUT_BUFFER_SIZE;

  // used to buffer incoming data by ReadNextLine
  m_inputStreamBuffer = new nsMsgLineStreamBuffer(OUTPUT_BUFFER_SIZE, true /* allocate new lines */, false /* leave CRLFs on the returned string */);
  m_currentBiffState = nsIMsgFolder::nsMsgBiffState_Unknown;
  m_progressStringName.Truncate();
  m_stringIndex = IMAP_EMPTY_STRING_INDEX;
  m_progressExpectedNumber = 0;
  memset(m_progressCurrentNumber, 0, sizeof m_progressCurrentNumber);

  // since these are embedded in the nsImapProtocol object, but passed
  // through proxied xpcom methods, just AddRef them here.
  m_hdrDownloadCache = new nsMsgImapHdrXferInfo();
  m_downloadLineCache = new nsMsgImapLineDownloadCache();

  // subscription
  m_autoSubscribe = true;
  m_autoUnsubscribe = true;
  m_autoSubscribeOnOpen = true;
  m_deletableChildren = nullptr;

  mFolderLastModSeq = 0;

  Configure(gTooFastTime, gIdealTime, gChunkAddSize, gChunkSize,
                    gChunkThreshold, gFetchByChunks);
  m_forceSelect = false;
}

nsresult nsImapProtocol::Configure(int32_t TooFastTime, int32_t IdealTime,
                  int32_t ChunkAddSize, int32_t ChunkSize, int32_t ChunkThreshold,
                  bool FetchByChunks)
{
  m_tooFastTime = TooFastTime;    // secs we read too little too fast
  m_idealTime = IdealTime;    // secs we read enough in good time
  m_chunkAddSize = ChunkAddSize;    // buffer size to add when wasting time
  m_chunkStartSize = m_chunkSize = ChunkSize;
  m_chunkThreshold = ChunkThreshold;
  m_fetchByChunks = FetchByChunks;

  return NS_OK;
}


NS_IMETHODIMP
nsImapProtocol::Initialize(nsIImapHostSessionList * aHostSessionList,
                           nsIImapIncomingServer *aServer)
{
  NS_ASSERTION(aHostSessionList && aServer,
    "oops...trying to initialize with a null host session list or server!");
  if (!aHostSessionList || !aServer)
    return NS_ERROR_NULL_POINTER;

  nsresult rv = m_downloadLineCache->GrowBuffer(kDownLoadCacheSize);
  NS_ENSURE_SUCCESS(rv, rv);

  m_flagState = new nsImapFlagAndUidState(kImapFlagAndUidStateSize);
  if (!m_flagState)
    return NS_ERROR_OUT_OF_MEMORY;

  aServer->GetUseIdle(&m_useIdle);
  aServer->GetForceSelect(m_forceSelectValue);
  aServer->GetUseCondStore(&m_useCondStore);
  aServer->GetUseCompressDeflate(&m_useCompressDeflate);

  m_hostSessionList = aHostSessionList; // no ref count...host session list has life time > connection
  m_parser.SetHostSessionList(aHostSessionList);
  m_parser.SetFlagState(m_flagState);

  // Initialize the empty mime part string on the main thread.
  rv = IMAPGetStringBundle(getter_AddRefs(m_bundle));
  NS_ENSURE_SUCCESS(rv, rv);

  rv = m_bundle->GetStringFromName("imapEmptyMimePart", m_emptyMimePartString);
  NS_ENSURE_SUCCESS(rv, rv);

  // Now initialize the thread for the connection
  if (m_thread == nullptr)
  {
    nsresult rv = NS_NewThread(getter_AddRefs(m_iThread), this);
    if (NS_FAILED(rv))
    {
      NS_ASSERTION(m_iThread, "Unable to create imap thread.\n");
      return rv;
    }
    m_iThread->GetPRThread(&m_thread);

  }
  return NS_OK;
}

nsImapProtocol::~nsImapProtocol()
{
  PR_Free(m_fetchBodyIdList);

  PR_Free(m_dataOutputBuf);
  delete m_inputStreamBuffer;

  // **** We must be out of the thread main loop function
  NS_ASSERTION(!m_imapThreadIsRunning, "Oops, thread is still running.\n");
}

const nsCString&
nsImapProtocol::GetImapHostName()
{
  if (m_runningUrl && m_hostName.IsEmpty())
  {
    nsCOMPtr<nsIURI> url = do_QueryInterface(m_runningUrl);
    url->GetAsciiHost(m_hostName);
  }

  return m_hostName;
}

const nsCString&
nsImapProtocol::GetImapUserName()
{
  if (m_userName.IsEmpty() && m_imapServerSink)
  {
    m_imapServerSink->GetOriginalUsername(m_userName);
  }
  return m_userName;
}

const char*
nsImapProtocol::GetImapServerKey()
{
  if (m_serverKey.IsEmpty() && m_imapServerSink)
  {
    m_imapServerSink->GetServerKey(m_serverKey);
  }
  return m_serverKey.get();
}

nsresult
nsImapProtocol::SetupSinkProxy()
{
  nsresult res;
  if (m_runningUrl)
  {
    if (!m_imapMailFolderSink)
    {
      nsCOMPtr<nsIImapMailFolderSink> aImapMailFolderSink;
      (void) m_runningUrl->GetImapMailFolderSink(getter_AddRefs(aImapMailFolderSink));
      if (aImapMailFolderSink)
      {
        m_imapMailFolderSink = new ImapMailFolderSinkProxy(aImapMailFolderSink);
      }
    }

    if (!m_imapMessageSink)
    {
      nsCOMPtr<nsIImapMessageSink> aImapMessageSink;
      (void) m_runningUrl->GetImapMessageSink(getter_AddRefs(aImapMessageSink));
      if (aImapMessageSink) {
        m_imapMessageSink = new ImapMessageSinkProxy(aImapMessageSink);
      } else {
        return NS_ERROR_ILLEGAL_VALUE;
      }
    }
    if (!m_imapServerSink)
    {
      nsCOMPtr<nsIImapServerSink> aImapServerSink;
      res = m_runningUrl->GetImapServerSink(getter_AddRefs(aImapServerSink));
      if (aImapServerSink) {
        m_imapServerSink = new ImapServerSinkProxy(aImapServerSink);
      } else {
        return NS_ERROR_ILLEGAL_VALUE;
      }
    }
    if (!m_imapProtocolSink)
    {
      nsCOMPtr<nsIImapProtocolSink> anImapProxyHelper(do_QueryInterface(NS_ISUPPORTS_CAST(nsIImapProtocolSink*, this), &res));
      m_imapProtocolSink = new ImapProtocolSinkProxy(anImapProxyHelper);
    }
  }
  return NS_OK;
}

static void SetSecurityCallbacksFromChannel(nsISocketTransport* aTrans, nsIChannel* aChannel)
{
  nsCOMPtr<nsIInterfaceRequestor> callbacks;
  aChannel->GetNotificationCallbacks(getter_AddRefs(callbacks));

  nsCOMPtr<nsILoadGroup> loadGroup;
  aChannel->GetLoadGroup(getter_AddRefs(loadGroup));

  nsCOMPtr<nsIInterfaceRequestor> securityCallbacks;
  MsgNewNotificationCallbacksAggregation(callbacks, loadGroup,
                                         getter_AddRefs(securityCallbacks));
  if (securityCallbacks)
    aTrans->SetSecurityCallbacks(securityCallbacks);
}

// Setup With Url is intended to set up data which is held on a PER URL basis and not
// a per connection basis. If you have data which is independent of the url we are currently
// running, then you should put it in Initialize().
// This is only ever called from the UI thread. It is called from LoadUrl, right
// before the url gets run - i.e., the url is next in line to run.
nsresult nsImapProtocol::SetupWithUrl(nsIURI * aURL, nsISupports* aConsumer)
{
  nsresult rv = NS_ERROR_FAILURE;
  NS_ASSERTION(aURL, "null URL passed into Imap Protocol");
  if (aURL)
  {
    m_runningUrl = do_QueryInterface(aURL, &rv);
    if (NS_FAILED(rv))
      return rv;

    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl);
    nsCOMPtr<nsIMsgIncomingServer> server = do_QueryReferent(m_server);
    if (!server)
    {
      rv = mailnewsUrl->GetServer(getter_AddRefs(server));
      NS_ENSURE_SUCCESS(rv, rv);
      m_server = do_GetWeakReference(server);
    }
    nsCOMPtr <nsIMsgFolder> folder;
    mailnewsUrl->GetFolder(getter_AddRefs(folder));
    mFolderLastModSeq = 0;
    mFolderTotalMsgCount = 0;
    mFolderHighestUID = 0;
    m_uidValidity = kUidUnknown;
    if (folder)
    {
      nsCOMPtr<nsIMsgDatabase> folderDB;
      nsCOMPtr<nsIDBFolderInfo> folderInfo;
      folder->GetDBFolderInfoAndDB(getter_AddRefs(folderInfo), getter_AddRefs(folderDB));
      if (folderInfo)
      {
        nsCString modSeqStr;
        folderInfo->GetCharProperty(kModSeqPropertyName, modSeqStr);
        mFolderLastModSeq = ParseUint64Str(modSeqStr.get());
        folderInfo->GetNumMessages(&mFolderTotalMsgCount);
        folderInfo->GetUint32Property(kHighestRecordedUIDPropertyName, 0, &mFolderHighestUID);
        folderInfo->GetImapUidValidity(&m_uidValidity);
      }
    }
    nsCOMPtr<nsIImapIncomingServer> imapServer = do_QueryInterface(server);
    nsCOMPtr<nsIStreamListener> aRealStreamListener = do_QueryInterface(aConsumer);
    m_runningUrl->GetMockChannel(getter_AddRefs(m_mockChannel));
    imapServer->GetIsGMailServer(&m_isGmailServer);
    if (!m_mockChannel)
    {

      nsCOMPtr<nsIPrincipal> nullPrincipal =
        do_CreateInstance("@mozilla.org/nullprincipal;1", &rv);
      NS_ENSURE_SUCCESS(rv, rv);

      // there are several imap operations that aren't initiated via a nsIChannel::AsyncOpen call on the mock channel.
      // such as selecting a folder. nsImapProtocol now insists on a mock channel when processing a url.
      nsCOMPtr<nsIChannel> channel;
      rv = NS_NewChannel(getter_AddRefs(channel),
                         aURL,
                         nullPrincipal,
                         nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
                         nsIContentPolicy::TYPE_OTHER);
      m_mockChannel = do_QueryInterface(channel);

      // Certain imap operations (not initiated by the IO Service via AsyncOpen) can be interrupted by  the stop button on the toolbar.
      // We do this by using the loadgroup of the docshell for the message pane. We really shouldn't be doing this..
      // See the comment in nsMsgMailNewsUrl::GetLoadGroup.
      nsCOMPtr<nsILoadGroup> loadGroup;
      mailnewsUrl->GetLoadGroup(getter_AddRefs(loadGroup)); // get the message pane load group
      if (loadGroup)
        loadGroup->AddRequest(m_mockChannel, nullptr /* context isupports */);
    }

    if (m_mockChannel)
    {
      m_mockChannel->SetImapProtocol(this);
      // if we have a listener from a mock channel, over-ride the consumer that was passed in
      nsCOMPtr<nsIStreamListener> channelListener;
      m_mockChannel->GetChannelListener(getter_AddRefs(channelListener));
      if (channelListener) // only over-ride if we have a non null channel listener
        aRealStreamListener = channelListener;
      m_mockChannel->GetChannelContext(getter_AddRefs(m_channelContext));
      nsCOMPtr<nsIMsgWindow> msgWindow;
      GetMsgWindow(getter_AddRefs(msgWindow));
      if (!msgWindow)
        GetTopmostMsgWindow(getter_AddRefs(msgWindow));
      if (msgWindow)
      {
        nsCOMPtr<nsIDocShell> docShell;
        msgWindow->GetMessageWindowDocShell(getter_AddRefs(docShell));
        nsCOMPtr<nsIInterfaceRequestor> ir(do_QueryInterface(docShell));
        nsCOMPtr<nsIInterfaceRequestor> interfaceRequestor;
        msgWindow->GetNotificationCallbacks(getter_AddRefs(interfaceRequestor));
        nsCOMPtr<nsIInterfaceRequestor> aggregateIR;
        MsgNewInterfaceRequestorAggregation(interfaceRequestor, ir, getter_AddRefs(aggregateIR));
        m_mockChannel->SetNotificationCallbacks(aggregateIR);
      }
    }

    // since we'll be making calls directly from the imap thread to the channel listener,
    // we need to turn it into a proxy object....we'll assume that the listener is on the same thread
    // as the event sink queue
    if (aRealStreamListener)
    {
      NS_ASSERTION(!m_channelListener, "shouldn't already have a channel listener");
      m_channelListener = new StreamListenerProxy(aRealStreamListener);
    }

    server->GetRealHostName(m_realHostName);
    int32_t authMethod;
    (void) server->GetAuthMethod(&authMethod);
    InitPrefAuthMethods(authMethod, server);
    (void) server->GetSocketType(&m_socketType);
    bool shuttingDown;
    (void) imapServer->GetShuttingDown(&shuttingDown);
    if (!shuttingDown)
      (void) imapServer->GetUseIdle(&m_useIdle);
    else
      m_useIdle = false;
    imapServer->GetFetchByChunks(&m_fetchByChunks);
    imapServer->GetSendID(&m_sendID);

    nsAutoString trashFolderPath;
    if (NS_SUCCEEDED(imapServer->GetTrashFolderName(trashFolderPath)))
      CopyUTF16toMUTF7(trashFolderPath, m_trashFolderPath);

    nsCOMPtr<nsIPrefBranch> prefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID));
    if (prefBranch)
    {
      bool preferPlainText;
      prefBranch->GetBoolPref("mailnews.display.prefer_plaintext", &preferPlainText);
      // If the pref has changed since the last time we ran a url,
      // clear the shell cache for this host.
      if (preferPlainText != m_preferPlainText)
      {
        m_hostSessionList->ClearShellCacheForHost(GetImapServerKey());
        m_preferPlainText = preferPlainText;
      }
    }

    bool proxyCallback = false;
    if ( m_runningUrl && !m_transport /* and we don't have a transport yet */)
    {
      if (m_mockChannel) {
        rv = MsgExamineForProxyAsync(m_mockChannel, this, getter_AddRefs(m_proxyRequest));
        if (NS_FAILED(rv))
        {
          rv = SetupWithUrlCallback(nullptr);
        }
        else
        {
          proxyCallback = true;
        }
      }
    }

    if (!proxyCallback)
      rv = LoadImapUrlInternal();
  }

  return rv;
}

// nsIProtocolProxyCallback
NS_IMETHODIMP
nsImapProtocol::OnProxyAvailable(nsICancelable *aRequest, nsIChannel *aChannel,
                                 nsIProxyInfo *aProxyInfo, nsresult aStatus)
{
  // If we're called with NS_BINDING_ABORTED, the IMAP thread already died,
  // so we can't carry on. Otherwise, no checking of 'aStatus' here, see
  // nsHttpChannel::OnProxyAvailable(). Status is non-fatal and we just kick on.
  if (aStatus == NS_BINDING_ABORTED)
    return NS_ERROR_FAILURE;

  nsresult rv = SetupWithUrlCallback(aProxyInfo);
  if (NS_FAILED(rv)) {
    // Cancel the protocol and be done.
    if (m_mockChannel)
      m_mockChannel->Cancel(rv);
    return rv;
  }

  rv = LoadImapUrlInternal();
  if (NS_FAILED(rv)) {
    if (m_mockChannel)
      m_mockChannel->Cancel(rv);
  }

  return rv;
}

nsresult nsImapProtocol::SetupWithUrlCallback(nsIProxyInfo* aProxyInfo)
{
  m_proxyRequest = nullptr;

  nsresult rv;

  nsCOMPtr<nsISocketTransportService> socketService =
    do_GetService(NS_SOCKETTRANSPORTSERVICE_CONTRACTID, &rv);
  if (NS_FAILED(rv))
    return rv;

  Log("SetupWithUrlCallback", nullptr, "clearing IMAP_CONNECTION_IS_OPEN");
  ClearFlag(IMAP_CONNECTION_IS_OPEN);
  const char *connectionType = nullptr;

  if (m_socketType == nsMsgSocketType::SSL)
    connectionType = "ssl";
  else if (m_socketType == nsMsgSocketType::alwaysSTARTTLS)
    connectionType = "starttls";
  // This can go away once we think everyone is migrated
  // away from the trySTARTTLS socket type.
  else if (m_socketType == nsMsgSocketType::trySTARTTLS)
    connectionType = "starttls";

  int32_t port = -1;
  nsCOMPtr<nsIURI> uri = do_QueryInterface(m_runningUrl, &rv);
  if (NS_FAILED(rv))
    return rv;
  uri->GetPort(&port);

  rv = socketService->CreateTransport(&connectionType, connectionType != nullptr,
                                      m_realHostName, port, aProxyInfo,
                                      getter_AddRefs(m_transport));
  if (NS_FAILED(rv) && m_socketType == nsMsgSocketType::trySTARTTLS)
  {
    connectionType = nullptr;
    m_socketType = nsMsgSocketType::plain;
    rv = socketService->CreateTransport(&connectionType, connectionType != nullptr,
                                        m_realHostName, port, aProxyInfo,
                                        getter_AddRefs(m_transport));
  }

  // remember so we can know whether we can issue a start tls or not...
  m_connectionType = connectionType;
  if (m_transport && m_mockChannel)
  {
    uint8_t qos;
    rv = GetQoSBits(&qos);
    if (NS_SUCCEEDED(rv))
      m_transport->SetQoSBits(qos);

    // Ensure that the socket can get the notification callbacks
    SetSecurityCallbacksFromChannel(m_transport, m_mockChannel);

    // open buffered, blocking input stream
    rv = m_transport->OpenInputStream(nsITransport::OPEN_BLOCKING, 0, 0, getter_AddRefs(m_inputStream));
    if (NS_FAILED(rv))
      return rv;

    // open buffered, blocking output stream
    rv = m_transport->OpenOutputStream(nsITransport::OPEN_BLOCKING, 0, 0, getter_AddRefs(m_outputStream));
    if (NS_FAILED(rv))
      return rv;
    SetFlag(IMAP_CONNECTION_IS_OPEN);
  }

  return rv;
}


// when the connection is done processing the current state, free any per url state data...
void nsImapProtocol::ReleaseUrlState(bool rerunning)
{
  // clear out the socket's reference to the notification callbacks for this transaction
  {
    MutexAutoLock mon(mLock);
    if (m_transport)
    {
      m_transport->SetSecurityCallbacks(nullptr);
      m_transport->SetEventSink(nullptr, nullptr);
    }
  }

  if (m_mockChannel && !rerunning)
  {
    // Proxy the close of the channel to the ui thread.
    if (m_imapMailFolderSink)
      m_imapMailFolderSink->CloseMockChannel(m_mockChannel);
    else
      m_mockChannel->Close();

    {
      // grab a lock so m_mockChannel doesn't get cleared out
      // from under us.
      MutexAutoLock mon(mLock);
      if (m_mockChannel)
      {
        // Proxy the release of the channel to the main thread.  This is something
        // that the xpcom proxy system should do for us!
        NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol::m_mockChannel", m_mockChannel.forget());
      }
    }
  }

  m_channelContext = nullptr; // this might be the url - null it out before the final release of the url
  m_imapMessageSink = nullptr;

  // Proxy the release of the listener to the main thread.  This is something
  // that the xpcom proxy system should do for us!
  {
    // grab a lock so the m_channelListener doesn't get cleared.
    MutexAutoLock mon(mLock);
    if (m_channelListener)
    {
      NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol::m_channelListener", m_channelListener.forget());
    }
  }
  m_channelInputStream = nullptr;
  m_channelOutputStream = nullptr;

  nsCOMPtr<nsIMsgMailNewsUrl> mailnewsurl;
  nsCOMPtr<nsIImapMailFolderSink> saveFolderSink;

  {
    MutexAutoLock mon(mLock);
    if (m_runningUrl)
    {
      mailnewsurl = do_QueryInterface(m_runningUrl);
      // It is unclear what 'saveFolderSink' is used for, most likely to hold
      // a reference for a little longer. See bug 1324893 and bug 391259.
      saveFolderSink = m_imapMailFolderSink;

      m_runningUrl = nullptr; // force us to release our last reference on the url
      m_urlInProgress = false;
    }
  }
  // Need to null this out whether we have an m_runningUrl or not
  m_imapMailFolderSink = nullptr;

  // we want to make sure the imap protocol's last reference to the url gets released
  // back on the UI thread. This ensures that the objects the imap url hangs on to
  // properly get released back on the UI thread.
  if (mailnewsurl)
  {
    NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol::m_runningUrl", mailnewsurl.forget());
  }
  saveFolderSink = nullptr;
}


class nsImapThreadShutdownEvent : public mozilla::Runnable {
public:
  explicit nsImapThreadShutdownEvent(nsIThread *thread)
    : mozilla::Runnable("nsImapThreadShutdownEvent"), mThread(thread) {
  }
  NS_IMETHOD Run() {
    mThread->Shutdown();
    return NS_OK;
  }
private:
  nsCOMPtr<nsIThread> mThread;
};

class nsImapCancelProxy : public mozilla::Runnable {
public:
  explicit nsImapCancelProxy(nsICancelable *aProxyRequest)
    : mozilla::Runnable("nsImapCancelProxy"), mRequest(aProxyRequest) {
  }
  NS_IMETHOD Run() {
    if (mRequest)
      mRequest->Cancel(NS_BINDING_ABORTED);
    return NS_OK;
  }
private:
  nsCOMPtr<nsICancelable> mRequest;
};

NS_IMETHODIMP nsImapProtocol::Run()
{
  PR_CEnterMonitor(this);
  NS_ASSERTION(!m_imapThreadIsRunning,
                 "Oh. oh. thread is already running. What's wrong here?");
    if (m_imapThreadIsRunning)
    {
        PR_CExitMonitor(this);
        return NS_OK;
    }

  m_imapThreadIsRunning = true;
  PR_CExitMonitor(this);

  // call the platform specific main loop ....
  ImapThreadMainLoop();

  if (m_proxyRequest)
  {
    // Cancel proxy on main thread.
    RefPtr<nsImapCancelProxy> cancelProxy =
      new nsImapCancelProxy(m_proxyRequest);
    NS_DispatchToMainThread(cancelProxy, NS_DISPATCH_SYNC);
    m_proxyRequest = nullptr;
  }

  if (m_runningUrl)
  {
    NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol::m_runningUrl", m_runningUrl.forget());
  }

  // close streams via UI thread if it's not already done
  if (m_imapProtocolSink)
    m_imapProtocolSink->CloseStreams();

  m_imapMailFolderSink = nullptr;
  m_imapMessageSink = nullptr;

  // shutdown this thread, but do it from the main thread
  nsCOMPtr<nsIRunnable> ev = new nsImapThreadShutdownEvent(m_iThread);
  if (NS_FAILED(NS_DispatchToMainThread(ev)))
    NS_WARNING("Failed to dispatch nsImapThreadShutdownEvent");
  m_iThread = nullptr;

  // Release protocol object on the main thread to avoid destruction of 'this'
  // on the IMAP thread, which causes grief for weak references.
  nsCOMPtr<nsIImapProtocol> releaseOnMain(this);
  NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol::this", releaseOnMain.forget());
  return NS_OK;
}

//
// Must be called from UI thread only
//
NS_IMETHODIMP nsImapProtocol::CloseStreams()
{
  // make sure that it is called by the UI thread
  MOZ_ASSERT(NS_IsMainThread(), "CloseStreams() should not be called from an off UI thread");

  {
    MutexAutoLock mon(mLock);
    if (m_transport)
    {
        // make sure the transport closes (even if someone is still indirectly
        // referencing it).
        m_transport->Close(NS_ERROR_ABORT);
        m_transport = nullptr;
    }
    m_inputStream = nullptr;
    m_outputStream = nullptr;
    m_channelListener = nullptr;
    m_channelContext = nullptr;
    if (m_mockChannel)
    {
        m_mockChannel->Close();
        m_mockChannel = nullptr;
    }
    m_channelInputStream = nullptr;
    m_channelOutputStream = nullptr;

    // Close scope because we must let go of the monitor before calling
    // RemoveConnection to unblock anyone who tries to get a monitor to the
    // protocol object while holding onto a monitor to the server.
  }
  nsCOMPtr<nsIMsgIncomingServer> me_server = do_QueryReferent(m_server);
  if (me_server)
  {
      nsresult result;
      nsCOMPtr<nsIImapIncomingServer>
          aImapServer(do_QueryInterface(me_server, &result));
      if (NS_SUCCEEDED(result))
          aImapServer->RemoveConnection(this);
      me_server = nullptr;
  }
  m_server = nullptr;
  // take this opportunity of being on the UI thread to
  // persist chunk prefs if they've changed
  if (gChunkSizeDirty)
  {
    nsCOMPtr<nsIPrefBranch> prefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID));
    if (prefBranch)
    {
      prefBranch->SetIntPref("mail.imap.chunk_size", gChunkSize);
      prefBranch->SetIntPref("mail.imap.min_chunk_size_threshold", gChunkThreshold);
      gChunkSizeDirty = false;
    }
  }
  return NS_OK;
}

NS_IMETHODIMP nsImapProtocol::GetUrlWindow(nsIMsgMailNewsUrl *aUrl,
                                           nsIMsgWindow **aMsgWindow)
{
  NS_ENSURE_ARG_POINTER(aUrl);
  NS_ENSURE_ARG_POINTER(aMsgWindow);
  return aUrl->GetMsgWindow(aMsgWindow);
}

NS_IMETHODIMP nsImapProtocol::SetupMainThreadProxies()
{
  return SetupSinkProxy();
}

NS_IMETHODIMP nsImapProtocol::OnInputStreamReady(nsIAsyncInputStream *inStr)
{
  // should we check if it's a close vs. data available?
  if (m_idle)
  {
    uint64_t bytesAvailable = 0;
    (void) inStr->Available(&bytesAvailable);
    // check if data available - might be a close
    if (bytesAvailable != 0)
    {
      ReentrantMonitorAutoEnter mon(m_urlReadyToRunMonitor);
      m_lastActiveTime = PR_Now();
      m_nextUrlReadyToRun = true;
      mon.Notify();
    }
  }
  return NS_OK;
}

// this is to be called from the UI thread. It sets m_threadShouldDie,
// and then signals the imap thread, which, when it wakes up, should exit.
// The imap thread cleanup code will check m_safeToCloseConnection.
NS_IMETHODIMP
nsImapProtocol::TellThreadToDie(bool aIsSafeToClose)
{
  NS_WARNING_ASSERTION(NS_IsMainThread(),
                       "TellThreadToDie(aIsSafeToClose) should only be called from UI thread");
  MutexAutoLock mon(mLock);

  nsCOMPtr<nsIMsgIncomingServer> me_server = do_QueryReferent(m_server);
  if (me_server)
  {
    nsresult rv;
    nsCOMPtr<nsIImapIncomingServer>
      aImapServer(do_QueryInterface(me_server, &rv));
    if (NS_SUCCEEDED(rv))
      aImapServer->RemoveConnection(this);
    m_server = nullptr;
    me_server = nullptr;
  }
  {
    ReentrantMonitorAutoEnter deathMon(m_threadDeathMonitor);
    m_safeToCloseConnection = aIsSafeToClose;
    m_threadShouldDie = true;
  }
  ReentrantMonitorAutoEnter readyMon(m_urlReadyToRunMonitor);
  m_nextUrlReadyToRun = true;
  readyMon.Notify();
  return NS_OK;
}

void
nsImapProtocol::TellThreadToDie()
{
  nsresult rv = NS_OK;
  NS_WARNING_ASSERTION(!NS_IsMainThread(),
                       "TellThreadToDie() should not be called from UI thread");

  // prevent re-entering this method because it may lock the UI.
  if (m_inThreadShouldDie)
    return;
  m_inThreadShouldDie = true;

  // This routine is called only from the imap protocol thread.
  // The UI thread causes this to be called by calling TellThreadToDie.
  // In that case, m_safeToCloseConnection will be FALSE if it's dropping a
  // timed out connection, true when closing a cached connection.
  // We're using PR_CEnter/ExitMonitor because Monitors don't like having
  // us to hold one monitor and call code that gets a different monitor. And
  // some of the methods we call here use Monitors.
  PR_CEnterMonitor(this);

  m_urlInProgress = true;  // let's say it's busy so no one tries to use
                                // this about to die connection.
  bool urlWritingData = false;
  bool connectionIdle = !m_runningUrl;

  if (!connectionIdle)
    urlWritingData = m_imapAction == nsIImapUrl::nsImapAppendMsgFromFile
      || m_imapAction == nsIImapUrl::nsImapAppendDraftFromFile;

  bool closeNeeded = GetServerStateParser().GetIMAPstate() ==
                nsImapServerResponseParser::kFolderSelected && m_safeToCloseConnection;
  nsCString command;
  // if a url is writing data, we can't even logout, so we're just
  // going to close the connection as if the user pressed stop.
  if (m_currentServerCommandTagNumber > 0 && !urlWritingData)
  {
    bool isAlive = false;
    if (m_transport)
      rv = m_transport->IsAlive(&isAlive);

    if (TestFlag(IMAP_CONNECTION_IS_OPEN) && m_idle && isAlive)
      EndIdle(false);

    if (NS_SUCCEEDED(rv) && isAlive && closeNeeded && GetDeleteIsMoveToTrash() &&
        TestFlag(IMAP_CONNECTION_IS_OPEN) && m_outputStream)
      Close(true, connectionIdle);

    if (NS_SUCCEEDED(rv) && isAlive && TestFlag(IMAP_CONNECTION_IS_OPEN) &&
        NS_SUCCEEDED(GetConnectionStatus()) && m_outputStream)
      Logout(true, connectionIdle);
  }
  PR_CExitMonitor(this);
  // close streams via UI thread
  if (m_imapProtocolSink)
  {
    m_imapProtocolSink->CloseStreams();
    m_imapProtocolSink = nullptr;
  }
  Log("TellThreadToDie", nullptr, "close socket connection");

  {
    ReentrantMonitorAutoEnter mon(m_threadDeathMonitor);
    m_threadShouldDie = true;
  }
  {
    ReentrantMonitorAutoEnter dataMon(m_dataAvailableMonitor);
    dataMon.Notify();
  }
  ReentrantMonitorAutoEnter urlReadyMon(m_urlReadyToRunMonitor);
  urlReadyMon.NotifyAll();
}

NS_IMETHODIMP
nsImapProtocol::GetLastActiveTimeStamp(PRTime* aTimeStamp)
{
  if (aTimeStamp)
    *aTimeStamp = m_lastActiveTime;
  return NS_OK;
}

static void DoomCacheEntry(nsIMsgMailNewsUrl *url);
NS_IMETHODIMP
nsImapProtocol::PseudoInterruptMsgLoad(nsIMsgFolder *aImapFolder, nsIMsgWindow *aMsgWindow, bool *interrupted)
{
  NS_ENSURE_ARG (interrupted);

  *interrupted = false;

  PR_CEnterMonitor(this);

  if (m_runningUrl && !TestFlag(IMAP_CLEAN_UP_URL_STATE))
  {
    nsImapAction imapAction;
    m_runningUrl->GetImapAction(&imapAction);

    if (imapAction == nsIImapUrl::nsImapMsgFetch)
    {
      nsresult rv = NS_OK;
      nsCOMPtr<nsIImapUrl> runningImapURL;

      rv = GetRunningImapURL(getter_AddRefs(runningImapURL));
      if (NS_SUCCEEDED(rv) && runningImapURL)
      {
        nsCOMPtr <nsIMsgFolder> runningImapFolder;
        nsCOMPtr <nsIMsgWindow> msgWindow;
        nsCOMPtr <nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(runningImapURL);
        mailnewsUrl->GetMsgWindow(getter_AddRefs(msgWindow));
        mailnewsUrl->GetFolder(getter_AddRefs(runningImapFolder));
        if (aImapFolder == runningImapFolder && msgWindow == aMsgWindow)
        {
          PseudoInterrupt(true);
          *interrupted = true;
        }
        // If we're interrupted, doom any incomplete cache entry.
        DoomCacheEntry(mailnewsUrl);
      }
    }
  }
  PR_CExitMonitor(this);
#ifdef DEBUG_bienvenu
  printf("interrupt msg load : %s\n", (*interrupted) ? "TRUE" : "FALSE");
#endif
  return NS_OK;
}

void
nsImapProtocol::ImapThreadMainLoop()
{
  MOZ_LOG(IMAP, LogLevel::Debug, ("ImapThreadMainLoop entering [this=%p]\n", this));

  PRIntervalTime sleepTime = kImapSleepTime;
  while (!DeathSignalReceived())
  {
    nsresult rv = NS_OK;
    bool readyToRun;

    // wait for an URL to process...
    {
      ReentrantMonitorAutoEnter mon(m_urlReadyToRunMonitor);

      while (NS_SUCCEEDED(rv) && !DeathSignalReceived() &&
             !m_nextUrlReadyToRun && !m_threadShouldDie)
        rv = mon.Wait(sleepTime);

      readyToRun = m_nextUrlReadyToRun;
      m_nextUrlReadyToRun = false;
    }
    // This will happen if the UI thread signals us to die
    if (m_threadShouldDie)
    {
      TellThreadToDie();
      break;
    }

    if (NS_FAILED(rv) && PR_PENDING_INTERRUPT_ERROR == PR_GetError())
    {
      printf("error waiting for monitor\n");
      break;
    }

    if (readyToRun && m_runningUrl)
    {
      if (m_currentServerCommandTagNumber && m_transport)
      {
        bool isAlive;
        rv = m_transport->IsAlive(&isAlive);
        // if the transport is not alive, and we've ever sent a command with this connection, kill it.
        // otherwise, we've probably just not finished setting it so don't kill it!
        if (NS_FAILED(rv) || !isAlive)
        {
          // This says we never started running the url, which is the case.
          m_runningUrl->SetRerunningUrl(false);
          RetryUrl();
          return;
        }
      }
      //
      // NOTE: Though we cleared m_nextUrlReadyToRun above, it may have been
      //       set by LoadImapUrl, which runs on the main thread.  Because of this,
      //       we must not try to clear m_nextUrlReadyToRun here.
      //
      if (ProcessCurrentURL())
      {
        m_nextUrlReadyToRun = true;
        m_imapMailFolderSink = nullptr;
      }
      else
      {
        // see if we want to go into idle mode. Might want to check a pref here too.
        if (m_useIdle && !m_urlInProgress && GetServerStateParser().GetCapabilityFlag() & kHasIdleCapability
          && GetServerStateParser().GetIMAPstate()
                == nsImapServerResponseParser::kFolderSelected)
        {
          Idle(); // for now, lets just do it. We'll probably want to use a timer
        }
        else // if not idle, don't need to remember folder sink
          m_imapMailFolderSink = nullptr;
      }
    }
    else if (m_idle && !m_threadShouldDie)
    {
      HandleIdleResponses();
    }
    if (!GetServerStateParser().Connected())
      break;
#ifdef DEBUG_bienvenu
    else
      printf("ready to run but no url and not idle\n");
#endif
    // This can happen if the UI thread closes cached connections in the
    // OnStopRunningUrl notification.
    if (m_threadShouldDie)
      TellThreadToDie();
  }
  m_imapThreadIsRunning = false;

  MOZ_LOG(IMAP, LogLevel::Debug, ("ImapThreadMainLoop leaving [this=%p]\n", this));
}

void nsImapProtocol::HandleIdleResponses()
{
  // int32_t oldRecent = GetServerStateParser().NumberOfRecentMessages();
  nsAutoCString commandBuffer(GetServerCommandTag());
  commandBuffer.AppendLiteral(" IDLE" CRLF);

  do
  {
    ParseIMAPandCheckForNewMail(commandBuffer.get());
  }
  while (m_inputStreamBuffer->NextLineAvailable() && GetServerStateParser().Connected());

  //  if (oldRecent != GetServerStateParser().NumberOfRecentMessages())
  //  We might check that something actually changed, but for now we can
  // just assume it. OnNewIdleMessages must run a url, so that
  // we'll go back into asyncwait mode.
  if (GetServerStateParser().Connected() && m_imapMailFolderSink)
    m_imapMailFolderSink->OnNewIdleMessages();
}

void nsImapProtocol::EstablishServerConnection()
{
#define ESC_LENGTH(x) (sizeof(x) - 1)
#define ESC_OK                      "* OK"
#define ESC_OK_LEN                  ESC_LENGTH(ESC_OK)
#define ESC_PREAUTH                 "* PREAUTH"
#define ESC_PREAUTH_LEN             ESC_LENGTH(ESC_PREAUTH)
#define ESC_CAPABILITY_STAR         "* "
#define ESC_CAPABILITY_STAR_LEN     ESC_LENGTH(ESC_CAPABILITY_STAR)
#define ESC_CAPABILITY_OK           "* OK ["
#define ESC_CAPABILITY_OK_LEN       ESC_LENGTH(ESC_CAPABILITY_OK)
#define ESC_CAPABILITY_GREETING     (ESC_CAPABILITY_OK "CAPABILITY")
#define ESC_CAPABILITY_GREETING_LEN ESC_LENGTH(ESC_CAPABILITY_GREETING)

  char * serverResponse = CreateNewLineFromSocket(); // read in the greeting

  // record the fact that we've received a greeting for this connection so we don't ever
  // try to do it again..
  if (serverResponse)
    SetFlag(IMAP_RECEIVED_GREETING);

  if (!PL_strncasecmp(serverResponse, ESC_OK, ESC_OK_LEN))
  {
    SetConnectionStatus(NS_OK);

    if (!PL_strncasecmp(serverResponse, ESC_CAPABILITY_GREETING, ESC_CAPABILITY_GREETING_LEN))
    {
      nsAutoCString tmpstr(serverResponse);
      int32_t endIndex = tmpstr.FindChar(']', ESC_CAPABILITY_GREETING_LEN);
      if (endIndex >= 0)
      {
        // Allocate the new buffer here. This buffer will be passed to
        // ParseIMAPServerResponse() where it will be used to fill the
        // fCurrentLine field and will be freed by the next call to
        // ResetLexAnalyzer().
        char *fakeServerResponse = (char*)PR_Malloc(PL_strlen(serverResponse));
        // Munge the greeting into something that would pass for an IMAP
        // server's response to a "CAPABILITY" command.
        strcpy(fakeServerResponse, ESC_CAPABILITY_STAR);
        strcat(fakeServerResponse, serverResponse + ESC_CAPABILITY_OK_LEN);
        fakeServerResponse[endIndex - ESC_CAPABILITY_OK_LEN + ESC_CAPABILITY_STAR_LEN] = '\0';
        // Tell the response parser that we just issued a "CAPABILITY" and
        // got the following back.
        GetServerStateParser().ParseIMAPServerResponse("1 CAPABILITY", true, fakeServerResponse);
      }
    }
  }
  else if (!PL_strncasecmp(serverResponse, ESC_PREAUTH, ESC_PREAUTH_LEN))
  {
    // we've been pre-authenticated.
    // we can skip the whole password step, right into the
    // kAuthenticated state
    GetServerStateParser().PreauthSetAuthenticatedState();

    if (GetServerStateParser().GetCapabilityFlag() == kCapabilityUndefined)
      Capability();

    if ( !(GetServerStateParser().GetCapabilityFlag() &
          (kIMAP4Capability | kIMAP4rev1Capability | kIMAP4other) ) )
    {
      // AlertUserEvent_UsingId(MK_MSG_IMAP_SERVER_NOT_IMAP4);
      SetConnectionStatus(NS_ERROR_FAILURE);        // stop netlib
    }
    else
    {
      // let's record the user as authenticated.
      m_imapServerSink->SetUserAuthenticated(true);

      ProcessAfterAuthenticated();
      // the connection was a success
      SetConnectionStatus(NS_OK);
     }
  }

  PR_Free(serverResponse); // we don't care about the greeting yet...

#undef ESC_LENGTH
#undef ESC_OK
#undef ESC_OK_LEN
#undef ESC_PREAUTH
#undef ESC_PREAUTH_LEN
#undef ESC_CAPABILITY_STAR
#undef ESC_CAPABILITY_STAR_LEN
#undef ESC_CAPABILITY_OK
#undef ESC_CAPABILITY_OK_LEN
#undef ESC_CAPABILITY_GREETING
#undef ESC_CAPABILITY_GREETING_LEN
}

// This can get called from the UI thread or an imap thread.
// It makes sure we don't get left with partial messages in
// the memory cache.
static void DoomCacheEntry(nsIMsgMailNewsUrl *url)
{
  bool readingFromMemCache = false;
  nsCOMPtr<nsIImapUrl> imapUrl = do_QueryInterface(url);
  imapUrl->GetMsgLoadingFromCache(&readingFromMemCache);
  if (!readingFromMemCache)
  {
    nsCOMPtr<nsICacheEntry> cacheEntry;
    url->GetMemCacheEntry(getter_AddRefs(cacheEntry));
    if (cacheEntry)
      cacheEntry->AsyncDoom(nullptr);
  }
}

// returns true if another url was run, false otherwise.
bool nsImapProtocol::ProcessCurrentURL()
{
  nsresult rv = NS_OK;
  if (m_idle)
    EndIdle();

  if (m_retryUrlOnError)
  {
    // we clear this flag if we're re-running immediately, because that
    // means we never sent a start running url notification, and later we
    // don't send start running notification if we think we're rerunning
    // the url (see first call to SetUrlState below). This means we won't
    // send a start running notification, which means our stop running
    // notification will be ignored because we don't think we were running.
    m_runningUrl->SetRerunningUrl(false);
    return RetryUrl();
  }
  Log("ProcessCurrentURL", nullptr, "entering");
  (void) GetImapHostName(); // force m_hostName to get set.


  bool    logonFailed = false;
  bool anotherUrlRun = false;
  bool rerunningUrl = false;
  bool isExternalUrl;
  bool validUrl = true;

  PseudoInterrupt(false);  // clear this if left over from previous url.

  m_runningUrl->GetRerunningUrl(&rerunningUrl);
  m_runningUrl->GetExternalLinkUrl(&isExternalUrl);
  m_runningUrl->GetValidUrl(&validUrl);
  m_runningUrl->GetImapAction(&m_imapAction);

  if (isExternalUrl)
  {
    if (m_imapAction == nsIImapUrl::nsImapSelectFolder)
    {
      // we need to send a start request so that the doc loader
      // will call HandleContent on the imap service so we
      // can abort this url, and run a new url in a new msg window
      // to run the folder load url and get off this crazy merry-go-round.
      if (m_channelListener)
      {
        m_channelListener->OnStartRequest(m_mockChannel, m_channelContext);
      }
      return false;
    }
  }

  if (!m_imapMailFolderSink && m_imapProtocolSink)
  {
    // This occurs when running another URL in the main thread loop
    rv = m_imapProtocolSink->SetupMainThreadProxies();
    NS_ENSURE_SUCCESS(rv, false);
  }

  // Reinitialize the parser
  GetServerStateParser().InitializeState();
  GetServerStateParser().SetConnected(true);

  // acknowledge that we are running the url now..
  nsCOMPtr<nsIMsgMailNewsUrl> mailnewsurl = do_QueryInterface(m_runningUrl, &rv);
  nsAutoCString urlSpec;
  rv = mailnewsurl->GetSpec(urlSpec);
  NS_ENSURE_SUCCESS(rv, false);
  Log("ProcessCurrentURL", urlSpec.get(), (validUrl) ? " = currentUrl\n" : " is not valid\n");
  if (!validUrl)
    return false;

  if (NS_SUCCEEDED(rv) && mailnewsurl && m_imapMailFolderSink && !rerunningUrl)
    m_imapMailFolderSink->SetUrlState(this, mailnewsurl, true, false,
                                      NS_OK);

  // if we are set up as a channel, we should notify our channel listener that we are starting...
  // so pass in ourself as the channel and not the underlying socket or file channel the protocol
  // happens to be using
  if (m_channelListener) // ### not sure we want to do this if rerunning url...
  {
    m_channelListener->OnStartRequest(m_mockChannel, m_channelContext);
  }
  // If we haven't received the greeting yet, we need to make sure we strip
  // it out of the input before we start to do useful things...
  if (!TestFlag(IMAP_RECEIVED_GREETING))
    EstablishServerConnection();

  // Step 1: If we have not moved into the authenticated state yet then do so
  // by attempting to logon.
  if (!DeathSignalReceived() && NS_SUCCEEDED(GetConnectionStatus()) &&
      (GetServerStateParser().GetIMAPstate() ==
       nsImapServerResponseParser::kNonAuthenticated))
  {
      /* if we got here, the server's greeting should not have been PREAUTH */
      if (GetServerStateParser().GetCapabilityFlag() == kCapabilityUndefined)
          Capability();

      if ( !(GetServerStateParser().GetCapabilityFlag() & (kIMAP4Capability | kIMAP4rev1Capability |
             kIMAP4other) ) )
      {
        if (!DeathSignalReceived() && NS_SUCCEEDED(GetConnectionStatus()))
          AlertUserEventUsingName("imapServerNotImap4");

        SetConnectionStatus(NS_ERROR_FAILURE);        // stop netlib
      }
      else
      {
        if ((m_connectionType.EqualsLiteral("starttls")
             && (m_socketType == nsMsgSocketType::trySTARTTLS
             && (GetServerStateParser().GetCapabilityFlag() & kHasStartTLSCapability)))
            || m_socketType == nsMsgSocketType::alwaysSTARTTLS)
        {
          StartTLS();
          if (GetServerStateParser().LastCommandSuccessful())
          {
            nsCOMPtr<nsISupports> secInfo;

            NS_ENSURE_TRUE(m_transport, false);
            rv = m_transport->GetSecurityInfo(getter_AddRefs(secInfo));

            if (NS_SUCCEEDED(rv) && secInfo)
            {
              nsCOMPtr<nsISSLSocketControl> sslControl = do_QueryInterface(secInfo, &rv);

              if (NS_SUCCEEDED(rv) && sslControl)
              {
                rv = sslControl->StartTLS();
                if (NS_SUCCEEDED(rv))
                {
                  if (m_socketType == nsMsgSocketType::trySTARTTLS)
                    m_imapServerSink->UpdateTrySTARTTLSPref(true);
                  // force re-issue of "capability", because servers may
                  // enable other auth features (e.g. remove LOGINDISABLED
                  // and add AUTH=PLAIN) after we upgraded to SSL.
                  Capability();
                  eIMAPCapabilityFlags capabilityFlag = GetServerStateParser().GetCapabilityFlag();
                  // Courier imap doesn't return STARTTLS capability if we've done
                  // a STARTTLS! But we need to remember this capability so we'll
                  // try to use STARTTLS next time.
                  if (!(capabilityFlag & kHasStartTLSCapability))
                  {
                    capabilityFlag |= kHasStartTLSCapability;
                    GetServerStateParser().SetCapabilityFlag(capabilityFlag);
                    CommitCapability();
                  }
                }
              }
            }
            if (NS_FAILED(rv))
            {
              nsAutoCString logLine("STARTTLS negotiation failed. Error 0x");
              logLine.AppendInt(static_cast<uint32_t>(rv), 16);
              Log("ProcessCurrentURL", nullptr, logLine.get());
              if (m_socketType == nsMsgSocketType::alwaysSTARTTLS)
              {
                SetConnectionStatus(rv);        // stop netlib
                m_transport->Close(rv);
              }
              else if (m_socketType == nsMsgSocketType::trySTARTTLS)
                m_imapServerSink->UpdateTrySTARTTLSPref(false);
            }
          }
          else if (m_socketType == nsMsgSocketType::alwaysSTARTTLS)
          {
            SetConnectionStatus(NS_ERROR_FAILURE);        // stop netlib
            if (m_transport)
              m_transport->Close(rv);
          }
          else if (m_socketType == nsMsgSocketType::trySTARTTLS)
          {
            // STARTTLS failed, so downgrade socket type
            m_imapServerSink->UpdateTrySTARTTLSPref(false);
          }
        }
        else if (m_socketType == nsMsgSocketType::trySTARTTLS)
        {
          // we didn't know the server supported TLS when we created
          // the socket, so we're going to retry with a STARTTLS socket
          if (GetServerStateParser().GetCapabilityFlag() & kHasStartTLSCapability)
          {
            ClearFlag(IMAP_CONNECTION_IS_OPEN);
            TellThreadToDie();
            SetConnectionStatus(NS_ERROR_FAILURE);
            return RetryUrl();
          }
          // trySTARTTLS set, but server doesn't have TLS capability,
          // so downgrade socket type
          m_imapServerSink->UpdateTrySTARTTLSPref(false);
          m_socketType = nsMsgSocketType::plain;
        }
        logonFailed = !TryToLogon();
        if (m_retryUrlOnError)
          return RetryUrl();
      }
  } // if death signal not received

  if (!DeathSignalReceived() && (NS_SUCCEEDED(GetConnectionStatus())))
  {
    // if the server supports a language extension then we should
    // attempt to issue the language extension.
    if ( GetServerStateParser().GetCapabilityFlag() & kHasLanguageCapability)
      Language();

    if (m_runningUrl)
      FindMailboxesIfNecessary();

    nsImapState imapState = nsIImapUrl::ImapStatusNone;
    if (m_runningUrl)
      m_runningUrl->GetRequiredImapState(&imapState);

    if (imapState == nsIImapUrl::nsImapAuthenticatedState)
      ProcessAuthenticatedStateURL();
    else   // must be a url that requires us to be in the selected state
      ProcessSelectedStateURL();

    if (m_retryUrlOnError)
      return RetryUrl();

  // The URL has now been processed
    if ((!logonFailed && NS_FAILED(GetConnectionStatus())) ||
         DeathSignalReceived())
         HandleCurrentUrlError();

  }
  else if (!logonFailed)
      HandleCurrentUrlError();

// if we are set up as a channel, we should notify our channel listener that we are stopping...
// so pass in ourself as the channel and not the underlying socket or file channel the protocol
// happens to be using
  if (m_channelListener)
  {
      NS_ASSERTION(m_mockChannel, "no request");
      if (m_mockChannel) {
        nsresult status;
        m_mockChannel->GetStatus(&status);
        if (!GetServerStateParser().LastCommandSuccessful() && NS_SUCCEEDED(status))
          status = NS_MSG_ERROR_IMAP_COMMAND_FAILED;
        rv = m_channelListener->OnStopRequest(m_mockChannel, m_channelContext, status);
      }
  }
  bool suspendUrl = false;
  m_runningUrl->GetMoreHeadersToDownload(&suspendUrl);
  if (mailnewsurl && m_imapMailFolderSink)
  {
    if (logonFailed)
      rv = NS_ERROR_FAILURE;
    else if (GetServerStateParser().CommandFailed())
      rv = NS_MSG_ERROR_IMAP_COMMAND_FAILED;
    else
      rv = GetConnectionStatus();
    // we are done with this url.
    m_imapMailFolderSink->SetUrlState(this, mailnewsurl, false, suspendUrl,
                                      rv);
     // doom the cache entry
    if (NS_FAILED(rv) && DeathSignalReceived() && m_mockChannel)
      DoomCacheEntry(mailnewsurl);
  }
  else
  {
    // That's seen at times in debug sessions.
    NS_WARNING("missing url or sink");
  }

  // disable timeouts before caching connection.
  if (m_transport)
    m_transport->SetTimeout(nsISocketTransport::TIMEOUT_READ_WRITE, PR_UINT32_MAX);

  SetFlag(IMAP_CLEAN_UP_URL_STATE);

  nsCOMPtr <nsISupports> copyState;
  if (m_runningUrl)
    m_runningUrl->GetCopyState(getter_AddRefs(copyState));
  // this is so hokey...we MUST clear any local references to the url
  // BEFORE calling ReleaseUrlState
  mailnewsurl = nullptr;

  if (suspendUrl)
    m_imapServerSink->SuspendUrl(m_runningUrl);
  // save the imap folder sink since we need it to do the CopyNextStreamMessage
  RefPtr<ImapMailFolderSinkProxy> imapMailFolderSink = m_imapMailFolderSink;
  // release the url as we are done with it...
  ReleaseUrlState(false);
  ResetProgressInfo();

  ClearFlag(IMAP_CLEAN_UP_URL_STATE);

  if (imapMailFolderSink)
  {
    if (copyState)
    {
      rv = imapMailFolderSink->CopyNextStreamMessage(GetServerStateParser().LastCommandSuccessful() &&
                                                NS_SUCCEEDED(GetConnectionStatus()),
                                                copyState);
      if (NS_FAILED(rv))
        MOZ_LOG(IMAP, LogLevel::Info, ("CopyNextStreamMessage failed: %" PRIx32 "\n", static_cast<uint32_t>(rv)));

      NS_ReleaseOnMainThreadSystemGroup("nsImapProtocol, copyState", copyState.forget());
    }
    // we might need this to stick around for IDLE support
    m_imapMailFolderSink = imapMailFolderSink;
    imapMailFolderSink = nullptr;
  }
  else
    MOZ_LOG(IMAP, LogLevel::Info, ("null imapMailFolderSink\n"));

  // now try queued urls, now that we've released this connection.
  if (m_imapServerSink)
  {
    if (NS_SUCCEEDED(GetConnectionStatus()))
      rv = m_imapServerSink->LoadNextQueuedUrl(this, &anotherUrlRun);
    else // if we don't do this, they'll just sit and spin until
          // we run some other url on this server.
    {
      Log("ProcessCurrentURL", nullptr, "aborting queued urls");
      rv = m_imapServerSink->AbortQueuedUrls();
    }
  }

  // if we didn't run another url, release the server sink to
  // cut circular refs.
  if (!anotherUrlRun)
      m_imapServerSink = nullptr;

  if (NS_FAILED(GetConnectionStatus()) || !GetServerStateParser().Connected()
    || GetServerStateParser().SyntaxError())
  {
    if (m_imapServerSink)
      m_imapServerSink->RemoveServerConnection(this);

    if (!DeathSignalReceived())
    {
        TellThreadToDie();
    }
  }
  else
  {
    if (m_imapServerSink)
    {
      bool shuttingDown;
      m_imapServerSink->GetServerShuttingDown(&shuttingDown);
      if (shuttingDown)
        m_useIdle = false;
    }
  }
  return anotherUrlRun;
}

bool nsImapProtocol::RetryUrl()
{
  nsCOMPtr <nsIImapUrl> kungFuGripImapUrl = m_runningUrl;
  nsCOMPtr <nsIImapMockChannel> saveMockChannel;

  // the mock channel might be null - that's OK.
  if (m_imapServerSink)
    (void) m_imapServerSink->PrepareToRetryUrl(kungFuGripImapUrl, getter_AddRefs(saveMockChannel));

  ReleaseUrlState(true);
  if (m_imapServerSink)
  {
    m_imapServerSink->RemoveServerConnection(this);
    m_imapServerSink->RetryUrl(kungFuGripImapUrl, saveMockChannel);
  }
  return (m_imapServerSink != nullptr); // we're running a url (the same url)
}

// ignoreBadAndNOResponses --> don't throw a error dialog if this command results in a NO or Bad response
// from the server..in other words the command is "exploratory" and we don't really care if it succeeds or fails.
void nsImapProtocol::ParseIMAPandCheckForNewMail(const char* commandString, bool aIgnoreBadAndNOResponses)
{
    if (commandString)
        GetServerStateParser().ParseIMAPServerResponse(commandString, aIgnoreBadAndNOResponses);
    else
        GetServerStateParser().ParseIMAPServerResponse(m_currentCommand.get(), aIgnoreBadAndNOResponses);
    // **** fix me for new mail biff state *****
}

/////////////////////////////////////////////////////////////////////////////////////////////
// End of nsIStreamListenerSupport
//////////////////////////////////////////////////////////////////////////////////////////////

NS_IMETHODIMP
nsImapProtocol::GetRunningUrl(nsIURI **result)
{
    if (result && m_runningUrl)
        return m_runningUrl->QueryInterface(NS_GET_IID(nsIURI), (void**)
                                            result);
    return NS_ERROR_NULL_POINTER;
}


NS_IMETHODIMP nsImapProtocol::GetRunningImapURL(nsIImapUrl **aImapUrl)
{
  if (aImapUrl && m_runningUrl)
     return m_runningUrl->QueryInterface(NS_GET_IID(nsIImapUrl), (void**) aImapUrl);
  return NS_ERROR_NULL_POINTER;
}

/*
 * Writes the data contained in dataBuffer into the current output stream. It also informs
 * the transport layer that this data is now available for transmission.
 * Returns a positive number for success, 0 for failure (not all the bytes were written to the
 * stream, etc). We need to make another pass through this file to install an error system (mscott)
 */

nsresult nsImapProtocol::SendData(const char * dataBuffer, bool aSuppressLogging)
{
  nsresult rv = NS_ERROR_NULL_POINTER;

  if (!m_transport)
  {
      Log("SendData", nullptr, "clearing IMAP_CONNECTION_IS_OPEN");
      // the connection died unexpectedly! so clear the open connection flag
      ClearFlag(IMAP_CONNECTION_IS_OPEN);
      TellThreadToDie();
      SetConnectionStatus(NS_ERROR_FAILURE);
      return NS_ERROR_FAILURE;
  }

  if (dataBuffer && m_outputStream)
  {
    m_currentCommand = dataBuffer;
    if (!aSuppressLogging)
      Log("SendData", nullptr, dataBuffer);
    else
      Log("SendData", nullptr, "Logging suppressed for this command (it probably contained authentication information)");

    {
      // don't allow someone to close the stream/transport out from under us
      // this can happen when the ui thread calls TellThreadToDie.
      PR_CEnterMonitor(this);
      uint32_t n;
      if (m_outputStream)
        rv = m_outputStream->Write(dataBuffer, PL_strlen(dataBuffer), &n);
      PR_CExitMonitor(this);
    }
    if (NS_FAILED(rv))
    {
      Log("SendData", nullptr, "clearing IMAP_CONNECTION_IS_OPEN");
      // the connection died unexpectedly! so clear the open connection flag
      ClearFlag(IMAP_CONNECTION_IS_OPEN);
      TellThreadToDie();
      SetConnectionStatus(rv);
      if (m_runningUrl && !m_retryUrlOnError)
      {
        bool alreadyRerunningUrl;
        m_runningUrl->GetRerunningUrl(&alreadyRerunningUrl);
        if (!alreadyRerunningUrl)
        {
          m_runningUrl->SetRerunningUrl(true);
          m_retryUrlOnError = true;
        }
      }
    }
  }

  return rv;
}

/////////////////////////////////////////////////////////////////////////////////////////////
// Begin protocol state machine functions...
//////////////////////////////////////////////////////////////////////////////////////////////

  // ProcessProtocolState - we override this only so we'll link - it should never get called.

nsresult nsImapProtocol::ProcessProtocolState(nsIURI * url, nsIInputStream * inputStream,
                                              uint64_t sourceOffset, uint32_t length)
{
  return NS_OK;
}

class UrlListenerNotifierEvent : public mozilla::Runnable
{
public:
  UrlListenerNotifierEvent(nsIMsgMailNewsUrl *aUrl, nsIImapProtocol *aProtocol)
    : mozilla::Runnable("UrlListenerNotifierEvent")
    , mUrl(aUrl), mProtocol(aProtocol)
  {}

  NS_IMETHOD Run()
  {
    if (mUrl)
    {
      nsCOMPtr<nsIMsgFolder> folder;
      mUrl->GetFolder(getter_AddRefs(folder));
      NS_ENSURE_TRUE(folder, NS_OK);
      nsCOMPtr<nsIImapMailFolderSink> folderSink(do_QueryInterface(folder));
      // This causes the url listener to get OnStart and Stop notifications.
      folderSink->SetUrlState(mProtocol, mUrl, true, false, NS_OK);
      folderSink->SetUrlState(mProtocol, mUrl, false, false, NS_OK);
    }
    return NS_OK;
  }

private:
  nsCOMPtr<nsIMsgMailNewsUrl> mUrl;
  nsCOMPtr<nsIImapProtocol> mProtocol;
};


bool nsImapProtocol::TryToRunUrlLocally(nsIURI *aURL, nsISupports *aConsumer)
{
  nsresult rv;
  nsCOMPtr<nsIImapUrl> imapUrl(do_QueryInterface(aURL, &rv));
  NS_ENSURE_SUCCESS(rv, false);
  nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(aURL);
  nsCString messageIdString;
  imapUrl->GetListOfMessageIds(messageIdString);
  bool useLocalCache = false;
  if (!messageIdString.IsEmpty() && !HandlingMultipleMessages(messageIdString))
  {
    nsImapAction action;
    imapUrl->GetImapAction(&action);
    nsCOMPtr <nsIMsgFolder> folder;
    mailnewsUrl->GetFolder(getter_AddRefs(folder));
    NS_ENSURE_TRUE(folder, false);

    folder->HasMsgOffline(strtoul(messageIdString.get(), nullptr, 10), &useLocalCache);
    mailnewsUrl->SetMsgIsInLocalCache(useLocalCache);
    // We're downloading a single message for offline use, and it's
    // already offline. So we shouldn't do anything, but we do
    // need to notify the url listener.
    if (useLocalCache && action == nsIImapUrl::nsImapMsgDownloadForOffline)
    {
      nsCOMPtr<nsIRunnable> event = new UrlListenerNotifierEvent(mailnewsUrl,
                                                                 this);
      // Post this as an event because it can lead to re-entrant calls to
      // LoadNextQueuedUrl if the listener runs a new url.
      if (event)
        NS_DispatchToCurrentThread(event);
      return true;
    }
  }
  if (!useLocalCache)
    return false;

  nsCOMPtr<nsIImapMockChannel> mockChannel;
  imapUrl->GetMockChannel(getter_AddRefs(mockChannel));
  if (!mockChannel)
    return false;

  nsImapMockChannel *imapChannel = static_cast<nsImapMockChannel *>(mockChannel.get());
  if (!imapChannel)
    return false;

  nsCOMPtr <nsILoadGroup> loadGroup;
  imapChannel->GetLoadGroup(getter_AddRefs(loadGroup));
  if (!loadGroup) // if we don't have one, the url will snag one from the msg window...
    mailnewsUrl->GetLoadGroup(getter_AddRefs(loadGroup));

  if (loadGroup)
    loadGroup->RemoveRequest((nsIRequest *) mockChannel, nullptr /* context isupports */, NS_OK);

  if (imapChannel->ReadFromLocalCache())
  {
    (void) imapChannel->NotifyStartEndReadFromCache(true);
    return true;
  }
  return false;
}


// LoadImapUrl takes a url, initializes all of our url specific data by calling SetupUrl.
// If we don't have a connection yet, we open the connection. Finally, we signal the
// url to run monitor to let the imap main thread loop process the current url (it is waiting
// on this monitor). There is a contract that the imap thread has already been started b4 we
// attempt to load a url....
NS_IMETHODIMP nsImapProtocol::LoadImapUrl(nsIURI * aURL, nsISupports * aConsumer)
{
  nsresult rv = NS_ERROR_FAILURE;
  if (aURL)
  {
#ifdef DEBUG_bienvenu
    printf("loading url %s\n", aURL->GetSpecOrDefault().get());
#endif
    if (TryToRunUrlLocally(aURL, aConsumer))
      return NS_OK;
    m_urlInProgress = true;
    m_imapMailFolderSink = nullptr;
    rv = SetupWithUrl(aURL, aConsumer);
    NS_ASSERTION(NS_SUCCEEDED(rv), "error setting up imap url");

    m_lastActiveTime = PR_Now();
  }
  return rv;
}

nsresult nsImapProtocol::LoadImapUrlInternal()
{
  nsresult rv = NS_ERROR_FAILURE;

  if (m_transport && m_mockChannel)
  {
    m_transport->SetTimeout(nsISocketTransport::TIMEOUT_CONNECT, gResponseTimeout + 60);
    int32_t readWriteTimeout = gResponseTimeout;
    if (m_runningUrl)
    {
      m_runningUrl->GetImapAction(&m_imapAction);
      // this is a silly hack, but the default of 100 seconds is way too long
      // for things like APPEND, which should come back immediately.
      if (m_imapAction == nsIImapUrl::nsImapAppendMsgFromFile ||
        m_imapAction == nsIImapUrl::nsImapAppendDraftFromFile)
      {
        readWriteTimeout = 20;
      }
      else if (m_imapAction == nsIImapUrl::nsImapOnlineMove ||
        m_imapAction == nsIImapUrl::nsImapOnlineCopy)
      {
        nsCString messageIdString;
        m_runningUrl->GetListOfMessageIds(messageIdString);
        uint32_t copyCount = CountMessagesInIdString(messageIdString.get());
        // If we're move/copying a large number of messages,
        // which should be rare, increase the timeout based on number
        // of messages. 40 messages per second should be sufficiently slow.
        if (copyCount > 2400) // 40 * 60, 60 is default read write timeout
          readWriteTimeout = std::max(readWriteTimeout, (int32_t)copyCount / 40);
      }
    }
    m_transport->SetTimeout(nsISocketTransport::TIMEOUT_READ_WRITE, readWriteTimeout);
    // set the security info for the mock channel to be the security status for our underlying transport.
    nsCOMPtr<nsISupports> securityInfo;
    m_transport->GetSecurityInfo(getter_AddRefs(securityInfo));
    m_mockChannel->SetSecurityInfo(securityInfo);

    SetSecurityCallbacksFromChannel(m_transport, m_mockChannel);

    nsCOMPtr<nsITransportEventSink> sink = do_QueryInterface(m_mockChannel);
    if (sink) {
      nsCOMPtr<nsIThread> thread = do_GetMainThread();
      m_transport->SetEventSink(sink, thread);
    }

    // and if we have a cache entry that we are saving the message to, set the security info on it too.
    // since imap only uses the memory cache, passing this on is the right thing to do.
    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl);
    if (mailnewsUrl)
    {
      nsCOMPtr<nsICacheEntry> cacheEntry;
      mailnewsUrl->GetMemCacheEntry(getter_AddRefs(cacheEntry));
      if (cacheEntry)
        cacheEntry->SetSecurityInfo(securityInfo);
    }
  }

  rv = SetupSinkProxy(); // generate proxies for all of the event sinks in the url
  if (NS_FAILED(rv)) // URL can be invalid.
    return rv;

  if (m_transport && m_runningUrl)
  {
    nsImapAction imapAction;
    m_runningUrl->GetImapAction(&imapAction);
    // if we're shutting down, and not running the kinds of urls we run at
    // shutdown, then this should fail because running urls during
    // shutdown will very likely fail and potentially hang.
    nsCOMPtr<nsIMsgAccountManager> accountMgr = do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv);
    NS_ENSURE_SUCCESS(rv, rv);
    bool shuttingDown = false;
    (void) accountMgr->GetShutdownInProgress(&shuttingDown);
    if (shuttingDown && imapAction != nsIImapUrl::nsImapExpungeFolder &&
        imapAction != nsIImapUrl::nsImapDeleteAllMsgs &&
        imapAction != nsIImapUrl::nsImapDeleteFolder)
      return NS_ERROR_FAILURE;

    // if we're running a select or delete all, do a noop first.
    // this should really be in the connection cache code when we know
    // we're pulling out a selected state connection, but maybe we
    // can get away with this.
    m_needNoop = (imapAction == nsIImapUrl::nsImapSelectFolder || imapAction == nsIImapUrl::nsImapDeleteAllMsgs);

    // We now have a url to run so signal the monitor for url ready to be processed...
    ReentrantMonitorAutoEnter urlReadyMon(m_urlReadyToRunMonitor);
    m_nextUrlReadyToRun = true;
    urlReadyMon.Notify();

  } // if we have an imap url and a transport
  else {
    NS_ASSERTION(false, "missing channel or running url");
  }

  return rv;
}

NS_IMETHODIMP nsImapProtocol::IsBusy(bool *aIsConnectionBusy,
                                     bool *isInboxConnection)
{
  if (!aIsConnectionBusy || !isInboxConnection)
    return NS_ERROR_NULL_POINTER;
  nsresult rv = NS_OK;
  *aIsConnectionBusy = false;
  *isInboxConnection = false;
  if (!m_transport)
  {
    // this connection might not be fully set up yet.
    rv = NS_ERROR_FAILURE;
  }
  else
  {
    if (m_urlInProgress) // do we have a url? That means we're working on it...
      *aIsConnectionBusy = true;

    if (GetServerStateParser().GetIMAPstate() ==
        nsImapServerResponseParser::kFolderSelected && GetServerStateParser().GetSelectedMailboxName() &&
        PL_strcasecmp(GetServerStateParser().GetSelectedMailboxName(),
                      "Inbox") == 0)
      *isInboxConnection = true;

  }
  return rv;
}

#define IS_SUBSCRIPTION_RELATED_ACTION(action) (action == nsIImapUrl::nsImapSubscribe\
|| action == nsIImapUrl::nsImapUnsubscribe || action == nsIImapUrl::nsImapDiscoverAllBoxesUrl || action == nsIImapUrl::nsImapListFolder)


// canRunUrl means the connection is not busy, and is in the selected state
// for the desired folder (or authenticated).
// has to wait means it's in the right selected state, but busy.
NS_IMETHODIMP nsImapProtocol::CanHandleUrl(nsIImapUrl * aImapUrl,
                                           bool * aCanRunUrl,
                                           bool * hasToWait)
{
  if (!aCanRunUrl || !hasToWait || !aImapUrl)
    return NS_ERROR_NULL_POINTER;
  nsresult rv = NS_OK;
  MutexAutoLock mon(mLock);

  *aCanRunUrl = false; // assume guilty until proven otherwise...
  *hasToWait = false;

  if (DeathSignalReceived())
    return NS_ERROR_FAILURE;

  bool isBusy = false;
  bool isInboxConnection = false;

  if (!m_transport)
  {
    // this connection might not be fully set up yet.
    return NS_ERROR_FAILURE;
  }
  IsBusy(&isBusy, &isInboxConnection);
  bool inSelectedState = GetServerStateParser().GetIMAPstate() ==
    nsImapServerResponseParser::kFolderSelected;

  nsAutoCString curSelectedUrlFolderName;
  nsAutoCString pendingUrlFolderName;
  if (inSelectedState)
    curSelectedUrlFolderName = GetServerStateParser().GetSelectedMailboxName();

  if (isBusy)
  {
    nsImapState curUrlImapState;
    NS_ASSERTION(m_runningUrl,"isBusy, but no running url.");
    if (m_runningUrl)
    {
      m_runningUrl->GetRequiredImapState(&curUrlImapState);
      if (curUrlImapState == nsIImapUrl::nsImapSelectedState)
      {
        char *folderName = GetFolderPathString();
        if (!curSelectedUrlFolderName.Equals(folderName))
          pendingUrlFolderName.Assign(folderName);
        inSelectedState = true;
        PR_Free(folderName);
      }
    }
  }

  nsImapState imapState;
  nsImapAction actionForProposedUrl;
  aImapUrl->GetImapAction(&actionForProposedUrl);
  aImapUrl->GetRequiredImapState(&imapState);

  // OK, this is a bit of a hack - we're going to pretend that
  // these types of urls requires a selected state connection on
  // the folder in question. This isn't technically true,
  // but we would much rather use that connection for several reasons,
  // one is that some UW servers require us to use that connection
  // the other is that we don't want to leave a connection dangling in
  // the selected state for the deleted folder.
  // If we don't find a connection in that selected state,
  // we'll fall back to the first free connection.
  bool isSelectedStateUrl = imapState == nsIImapUrl::nsImapSelectedState
    || actionForProposedUrl == nsIImapUrl::nsImapDeleteFolder || actionForProposedUrl == nsIImapUrl::nsImapRenameFolder
    || actionForProposedUrl == nsIImapUrl::nsImapMoveFolderHierarchy
    || actionForProposedUrl == nsIImapUrl::nsImapAppendDraftFromFile
    || actionForProposedUrl == nsIImapUrl::nsImapAppendMsgFromFile
    || actionForProposedUrl == nsIImapUrl::nsImapFolderStatus;

  nsCOMPtr<nsIMsgMailNewsUrl> msgUrl = do_QueryInterface(aImapUrl);
  nsCOMPtr<nsIMsgIncomingServer> server;
  rv = msgUrl->GetServer(getter_AddRefs(server));
  if (NS_SUCCEEDED(rv))
  {
    // compare host/user between url and connection.
    nsCString urlHostName;
    nsCString urlUserName;
    rv = server->GetHostName(urlHostName);
    NS_ENSURE_SUCCESS(rv, rv);
    rv = server->GetUsername(urlUserName);
    NS_ENSURE_SUCCESS(rv, rv);

    if ((GetImapHostName().IsEmpty() ||
      urlHostName.Equals(GetImapHostName(), nsCaseInsensitiveCStringComparator())) &&
      (GetImapUserName().IsEmpty() ||
      urlUserName.Equals(GetImapUserName(), nsCaseInsensitiveCStringComparator())))
    {
      if (isSelectedStateUrl)
      {
        if (inSelectedState)
        {
          // *** jt - in selected state can only run url with
          // matching foldername
          char *folderNameForProposedUrl = nullptr;
          rv = aImapUrl->CreateServerSourceFolderPathString(
            &folderNameForProposedUrl);
          if (NS_SUCCEEDED(rv) && folderNameForProposedUrl)
          {
            bool isInbox =
              PL_strcasecmp("Inbox", folderNameForProposedUrl) == 0;
            if (!curSelectedUrlFolderName.IsEmpty() || !pendingUrlFolderName.IsEmpty())
            {
              bool matched = isInbox ?
                PL_strcasecmp(curSelectedUrlFolderName.get(),
                folderNameForProposedUrl) == 0 :
              PL_strcmp(curSelectedUrlFolderName.get(),
                folderNameForProposedUrl) == 0;
              if (!matched && !pendingUrlFolderName.IsEmpty())
              {
                matched = isInbox ?
                  PL_strcasecmp(pendingUrlFolderName.get(),
                  folderNameForProposedUrl) == 0 :
                PL_strcmp(pendingUrlFolderName.get(),
                  folderNameForProposedUrl) == 0;
              }
              if (matched)
              {
                if (isBusy)
                  *hasToWait = true;
                else
                  *aCanRunUrl = true;
              }
            }
          }
          MOZ_LOG(IMAP, LogLevel::Debug,
                 ("proposed url = %s folder for connection %s has To Wait = %s can run = %s",
                  folderNameForProposedUrl, curSelectedUrlFolderName.get(),
                  (*hasToWait) ? "TRUE" : "FALSE", (*aCanRunUrl) ? "TRUE" : "FALSE"));
          PR_FREEIF(folderNameForProposedUrl);
        }
      }
      else // *** jt - an authenticated state url can be run in either
        // authenticated or selected state
      {
        nsImapAction actionForRunningUrl;

        // If proposed url is subscription related, and we are currently running
        // a subscription url, then we want to queue the proposed url after the current url.
        // Otherwise, we can run this url if we're not busy.
        // If we never find a running subscription-related url, the caller will
        // just use whatever free connection it can find, which is what we want.
        if (IS_SUBSCRIPTION_RELATED_ACTION(actionForProposedUrl))
        {
          if (isBusy && m_runningUrl)
          {
            m_runningUrl->GetImapAction(&actionForRunningUrl);
            if (IS_SUBSCRIPTION_RELATED_ACTION(actionForRunningUrl))
            {
              *aCanRunUrl = false;
              *hasToWait = true;
            }
          }
        }
        else
        {
          if (!isBusy)
            *aCanRunUrl = true;
        }
      }
    }
  }
  return rv;
}


// Command tag handling stuff
void nsImapProtocol::IncrementCommandTagNumber()
{
  sprintf(m_currentServerCommandTag, "%u", ++m_currentServerCommandTagNumber);
}

const char *nsImapProtocol::GetServerCommandTag()
{
    return m_currentServerCommandTag;
}

void nsImapProtocol::ProcessSelectedStateURL()
{
  nsCString mailboxName;
  bool bMessageIdsAreUids = true;
  bool moreHeadersToDownload;
  imapMessageFlagsType  msgFlags = 0;
  nsCString       urlHost;

  // this can't fail, can it?
  nsresult res;
  res = m_runningUrl->GetImapAction(&m_imapAction);
  m_runningUrl->MessageIdsAreUids(&bMessageIdsAreUids);
  m_runningUrl->GetMsgFlags(&msgFlags);
  m_runningUrl->GetMoreHeadersToDownload(&moreHeadersToDownload);

  res = CreateServerSourceFolderPathString(getter_Copies(mailboxName));
  if (NS_FAILED(res))
    Log("ProcessSelectedStateURL", nullptr, "error getting source folder path string");

  if (NS_SUCCEEDED(res) && !DeathSignalReceived())
  {
    bool selectIssued = false;
    if (GetServerStateParser().GetIMAPstate() == nsImapServerResponseParser::kFolderSelected)
    {
      if (GetServerStateParser().GetSelectedMailboxName() &&
        PL_strcmp(GetServerStateParser().GetSelectedMailboxName(),
        mailboxName.get()))
      {       // we are selected in another folder
        if (m_closeNeededBeforeSelect)
          Close();
        if (GetServerStateParser().LastCommandSuccessful())
        {
          selectIssued = true;
          SelectMailbox(mailboxName.get());
        }
      }
      else if (!GetServerStateParser().GetSelectedMailboxName())
      {       // why are we in the selected state with no box name?
        SelectMailbox(mailboxName.get());
        selectIssued = true;
      }
      else if (moreHeadersToDownload && m_imapMailFolderSink) // we need to fetch older headers
      {
        nsMsgKey *msgIdList = nullptr;
        uint32_t msgCount = 0;
        bool more;
        m_imapMailFolderSink->GetMsgHdrsToDownload(&more,
                                                   &m_progressExpectedNumber,
                                                   &msgCount,
                                                   &msgIdList);
        if (msgIdList)
        {
          FolderHeaderDump(msgIdList, msgCount);
          free(msgIdList);
          m_runningUrl->SetMoreHeadersToDownload(more);
          // We're going to be re-running this url.
          if (more)
            m_runningUrl->SetRerunningUrl(true);
        }
        HeaderFetchCompleted();
      }
      else
      {
        // get new message counts, if any, from server
        if (m_needNoop)
        {
          // For some IMAP servers, to detect new email we must send imap
          // SELECT even if already SELECTed on the same mailbox. For other
          // servers that simply don't support IDLE, doing select here will
          // cause emails to be properly marked "read" after they have been
          // read in another email client.
          if (m_forceSelect)
          {
            SelectMailbox(mailboxName.get());
            selectIssued = true;
          }

          m_noopCount++;
          if ((gPromoteNoopToCheckCount > 0 && (m_noopCount % gPromoteNoopToCheckCount) == 0) ||
            CheckNeeded())
            Check();
          else
            Noop(); // I think this is needed when we're using a cached connection
          m_needNoop = false;
        }
      }
    }
    else
    {
      // go to selected state
      SelectMailbox(mailboxName.get());
      selectIssued = GetServerStateParser().LastCommandSuccessful();
    }

    if (selectIssued)
      RefreshACLForFolderIfNecessary(mailboxName.get());

    bool uidValidityOk = true;
    if (GetServerStateParser().LastCommandSuccessful() && selectIssued &&
      (m_imapAction != nsIImapUrl::nsImapSelectFolder) && (m_imapAction != nsIImapUrl::nsImapLiteSelectFolder))
    {

      // error on the side of caution, if the fe event fails to set uidStruct->returnValidity, then assume that UIDVALIDITY
      // did not roll.  This is a common case event for attachments that are fetched within a browser context.
      if (!DeathSignalReceived())
        uidValidityOk = m_uidValidity == kUidUnknown ||
                        m_uidValidity == GetServerStateParser().FolderUID();
    }

    if (!uidValidityOk)
      Log("ProcessSelectedStateURL", nullptr, "uid validity not ok");
    if (GetServerStateParser().LastCommandSuccessful() && !DeathSignalReceived() && (uidValidityOk || m_imapAction == nsIImapUrl::nsImapDeleteAllMsgs))
    {
      if (GetServerStateParser().CurrentFolderReadOnly())
      {
        Log("ProcessSelectedStateURL", nullptr, "current folder read only");
        if (m_imapAction == nsIImapUrl::nsImapAddMsgFlags ||
          m_imapAction == nsIImapUrl::nsImapSubtractMsgFlags)
        {
          bool canChangeFlag = false;
          if (GetServerStateParser().ServerHasACLCapability() && m_imapMailFolderSink)
          {
            uint32_t aclFlags = 0;

            if (NS_SUCCEEDED(m_imapMailFolderSink->GetAclFlags(&aclFlags))
                  && aclFlags != 0) // make sure we have some acl flags
              canChangeFlag = ((msgFlags & kImapMsgSeenFlag) && (aclFlags & IMAP_ACL_STORE_SEEN_FLAG));
          }
          else
            canChangeFlag = (GetServerStateParser().SettablePermanentFlags() & msgFlags) == msgFlags;
          if (!canChangeFlag)
            return;
        }
        if (m_imapAction == nsIImapUrl::nsImapExpungeFolder || m_imapAction == nsIImapUrl::nsImapDeleteMsg ||
          m_imapAction == nsIImapUrl::nsImapDeleteAllMsgs)
          return;
      }
      switch (m_imapAction)
      {
      case nsIImapUrl::nsImapLiteSelectFolder:
        if (GetServerStateParser().LastCommandSuccessful() &&
            m_imapMailFolderSink && !moreHeadersToDownload)
        {
          m_imapMailFolderSink->SetUidValidity(GetServerStateParser().FolderUID());
          ProcessMailboxUpdate(false); // handle uidvalidity change
        }
        break;
      case nsIImapUrl::nsImapSaveMessageToDisk:
      case nsIImapUrl::nsImapMsgFetch:
      case nsIImapUrl::nsImapMsgFetchPeek:
      case nsIImapUrl::nsImapMsgDownloadForOffline:
      case nsIImapUrl::nsImapMsgPreview:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);
          // we don't want to send the flags back in a group
          if (HandlingMultipleMessages(messageIdString) || m_imapAction == nsIImapUrl::nsImapMsgDownloadForOffline
             || m_imapAction == nsIImapUrl::nsImapMsgPreview)
          {
            // multiple messages, fetch them all
            SetProgressString(IMAP_MESSAGES_STRING_INDEX);

            m_progressCurrentNumber[m_stringIndex] = 0;
            m_progressExpectedNumber = CountMessagesInIdString(messageIdString.get());

            // we need to set this so we'll get the msg from the memory cache.
            if (m_imapAction == nsIImapUrl::nsImapMsgFetchPeek)
              SetContentModified(IMAP_CONTENT_NOT_MODIFIED);

            FetchMessage(messageIdString,
              (m_imapAction == nsIImapUrl::nsImapMsgPreview)
              ? kBodyStart : kEveryThingRFC822Peek);
            if (m_imapAction == nsIImapUrl::nsImapMsgPreview)
              HeaderFetchCompleted();
            SetProgressString(IMAP_EMPTY_STRING_INDEX);
          }
          else
          {
            // A single message ID
            nsIMAPeFetchFields whatToFetch = kEveryThingRFC822;
            if(m_imapAction == nsIImapUrl::nsImapMsgFetchPeek)
              whatToFetch = kEveryThingRFC822Peek;

            // First, let's see if we're requesting a specific MIME part
            char *imappart = nullptr;
            m_runningUrl->GetImapPartToFetch(&imappart);
            if (imappart)
            {
              if (bMessageIdsAreUids)
              {
                // We actually want a specific MIME part of the message.
                // The Body Shell will generate it, even though we haven't downloaded it yet.

                IMAP_ContentModifiedType modType = GetShowAttachmentsInline() ?
                  IMAP_CONTENT_MODIFIED_VIEW_INLINE :
                  IMAP_CONTENT_MODIFIED_VIEW_AS_LINKS ;

                nsCString messageIdValString(messageIdString);
                messageIdValString.AppendInt(m_uidValidity);
                RefPtr<nsIMAPBodyShell> foundShell;
                res = m_hostSessionList->FindShellInCacheForHost(GetImapServerKey(),
                  GetServerStateParser().GetSelectedMailboxName(),
                  messageIdValString.get(), modType, getter_AddRefs(foundShell));
                if (!foundShell)
                {
                  // The shell wasn't in the cache.  Deal with this case later.
                  Log("SHELL",NULL,"Loading part, shell not found in cache!");
                  //MOZ_LOG(IMAP, out, ("BODYSHELL: Loading part, shell not found in cache!"));
                  // The parser will extract the part number from the current URL.
                  SetContentModified(modType);
                  Bodystructure(messageIdString, bMessageIdsAreUids);
                }
                else
                {
                  Log("SHELL", NULL, "Loading Part, using cached shell.");
                  //MOZ_LOG(IMAP, out, ("BODYSHELL: Loading part, using cached shell."));
                  SetContentModified(modType);
                  foundShell->SetConnection(this);
                  GetServerStateParser().UseCachedShell(foundShell);
                  //Set the current uid in server state parser (in case it was used for new mail msgs earlier).
                  GetServerStateParser().SetCurrentResponseUID(strtoul(messageIdString.get(), nullptr, 10));
                  foundShell->Generate(imappart);
                  GetServerStateParser().UseCachedShell(NULL);
                }
              }
              else
              {
                // Message IDs are not UIDs.
                NS_ASSERTION(false, "message ids aren't uids");
              }
              PR_Free(imappart);
            }
            else
            {
              // downloading a single message: try to do it by bodystructure, and/or do it by chunks
              uint32_t messageSize = GetMessageSize(messageIdString.get(), bMessageIdsAreUids);
              // We need to check the format_out bits to see if we are allowed to leave out parts,
              // or if we are required to get the whole thing.  Some instances where we are allowed
              // to do it by parts:  when viewing a message, replying to a message, or viewing its source
              // Some times when we're NOT allowed:  when forwarding a message, saving it, moving it, etc.
              // need to set a flag in the url, I guess, equiv to allow_content_changed.
              bool allowedToBreakApart = true; // (ce  && !DeathSignalReceived()) ? ce->URL_s->allow_content_change : false;
              bool mimePartSelectorDetected;
              bool urlOKToFetchByParts = false;
              m_runningUrl->GetMimePartSelectorDetected(&mimePartSelectorDetected);
              m_runningUrl->GetFetchPartsOnDemand(&urlOKToFetchByParts);

              {
                nsCOMPtr<nsIMsgMailNewsUrl> mailnewsurl = do_QueryInterface(m_runningUrl);
                nsAutoCString urlSpec;
                if (mailnewsurl)
                  urlSpec = mailnewsurl->GetSpecOrDefault();
                MOZ_LOG(IMAP, LogLevel::Debug,
                       ("SHELL: URL %s, OKToFetchByParts %d, allowedToBreakApart %d, ShouldFetchAllParts %d",
                        urlSpec.get(), urlOKToFetchByParts, allowedToBreakApart,
                        GetShouldFetchAllParts()));
              }

              if (urlOKToFetchByParts &&
                  allowedToBreakApart &&
                  !GetShouldFetchAllParts() &&
                  GetServerStateParser().ServerHasIMAP4Rev1Capability() /* &&
                !mimePartSelectorDetected */)  // if a ?part=, don't do BS.
              {
                // OK, we're doing bodystructure

                // Before fetching the bodystructure, let's check our body shell cache to see if
                // we already have it around.
                RefPtr<nsIMAPBodyShell> foundShell;
                IMAP_ContentModifiedType modType = GetShowAttachmentsInline() ?
                  IMAP_CONTENT_MODIFIED_VIEW_INLINE :
                  IMAP_CONTENT_MODIFIED_VIEW_AS_LINKS ;

                bool wasStoringMsgOffline;
                m_runningUrl->GetStoreResultsOffline(&wasStoringMsgOffline);
                m_runningUrl->SetStoreOfflineOnFallback(wasStoringMsgOffline);
                m_runningUrl->SetStoreResultsOffline(false);
                SetContentModified(modType);  // This will be looked at by the cache
                if (bMessageIdsAreUids)
                {
                  nsCString messageIdValString(messageIdString);
                  messageIdValString.AppendInt(m_uidValidity);
                  res = m_hostSessionList->FindShellInCacheForHost(GetImapServerKey(),
                    GetServerStateParser().GetSelectedMailboxName(),
                    messageIdValString.get(), modType, getter_AddRefs(foundShell));
                  if (foundShell)
                  {
                    Log("SHELL",NULL,"Loading message, using cached shell.");
                    //MOZ_LOG(IMAP, out, ("BODYSHELL: Loading message, using cached shell."));
                    foundShell->SetConnection(this);
                    GetServerStateParser().UseCachedShell(foundShell);
                    //Set the current uid in server state parser (in case it was used for new mail msgs earlier).
                    GetServerStateParser().SetCurrentResponseUID(strtoul(messageIdString.get(), nullptr, 10));
                    foundShell->Generate(NULL);
                    GetServerStateParser().UseCachedShell(NULL);
                  }
                }

                if (!foundShell)
                  Bodystructure(messageIdString, bMessageIdsAreUids);
              }
              else
              {
                // Not doing bodystructure.  Fetch the whole thing, and try to do
                // it in chunks.
                SetContentModified(IMAP_CONTENT_NOT_MODIFIED);
                FetchTryChunking(messageIdString, whatToFetch,
                  bMessageIdsAreUids, NULL, messageSize, true);
              }
            }
            if (GetServerStateParser().LastCommandSuccessful()
                && m_imapAction != nsIImapUrl::nsImapMsgPreview
                && m_imapAction != nsIImapUrl::nsImapMsgFetchPeek)
            {
              uint32_t uid = strtoul(messageIdString.get(), nullptr, 10);
              int32_t index;
              bool foundIt;
              imapMessageFlagsType flags = m_flagState->GetMessageFlagsFromUID(uid, &foundIt, &index);
              if (foundIt)
              {
                flags |= kImapMsgSeenFlag;
                m_flagState->SetMessageFlags(index, flags);
              }
            }
          }
        }
        break;
      case nsIImapUrl::nsImapExpungeFolder:
        Expunge();
        // note fall through to next cases.
        MOZ_FALLTHROUGH;
      case nsIImapUrl::nsImapSelectFolder:
      case nsIImapUrl::nsImapSelectNoopFolder:
        if (!moreHeadersToDownload)
          ProcessMailboxUpdate(true);
        break;
      case nsIImapUrl::nsImapMsgHeader:
        {
          nsCString messageIds;
          m_runningUrl->GetListOfMessageIds(messageIds);

          FetchMessage(messageIds,
            kHeadersRFC822andUid);
          // if we explicitly ask for headers, as opposed to getting them as a result
          // of selecting the folder, or biff, send the headerFetchCompleted notification
          // to flush out the header cache.
          HeaderFetchCompleted();
        }
        break;
      case nsIImapUrl::nsImapSearch:
        {
          nsAutoCString searchCriteriaString;
          m_runningUrl->CreateSearchCriteriaString(getter_Copies(searchCriteriaString));
          Search(searchCriteriaString.get(), bMessageIdsAreUids);
          // drop the results on the floor for now
        }
        break;
      case nsIImapUrl::nsImapUserDefinedMsgCommand:
        {
          nsCString messageIdString;
          nsCString command;

          m_runningUrl->GetCommand(command);
          m_runningUrl->GetListOfMessageIds(messageIdString);
          IssueUserDefinedMsgCommand(command.get(), messageIdString.get());
        }
        break;
      case nsIImapUrl::nsImapUserDefinedFetchAttribute:
        {
          nsCString messageIdString;
          nsCString attribute;

          m_runningUrl->GetCustomAttributeToFetch(attribute);
          m_runningUrl->GetListOfMessageIds(messageIdString);
          FetchMsgAttribute(messageIdString, attribute);
        }
        break;
      case nsIImapUrl::nsImapMsgStoreCustomKeywords:
        {
          // if the server doesn't support user defined flags, don't try to set them.
          uint16_t userFlags;
          GetSupportedUserFlags(&userFlags);
          if (! (userFlags & kImapMsgSupportUserFlag))
            break;
          nsCString messageIdString;
          nsCString addFlags;
          nsCString subtractFlags;

          m_runningUrl->GetListOfMessageIds(messageIdString);
          m_runningUrl->GetCustomAddFlags(addFlags);
          m_runningUrl->GetCustomSubtractFlags(subtractFlags);
          if (!addFlags.IsEmpty())
          {
            nsAutoCString storeString("+FLAGS (");
            storeString.Append(addFlags);
            storeString.Append(')');
            Store(messageIdString, storeString.get(), true);
          }
          if (!subtractFlags.IsEmpty())
          {
            nsAutoCString storeString("-FLAGS (");
            storeString.Append(subtractFlags);
            storeString.Append(')');
            Store(messageIdString, storeString.get(), true);
          }
        }
        break;
      case nsIImapUrl::nsImapDeleteMsg:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);

          ProgressEventFunctionUsingName(HandlingMultipleMessages(messageIdString) ?
                                         "imapDeletingMessages" :
                                         "imapDeletingMessage");

          Store(messageIdString, "+FLAGS (\\Deleted)",  bMessageIdsAreUids);

          if (GetServerStateParser().LastCommandSuccessful())
          {
            //delete_message_struct *deleteMsg = (delete_message_struct *) PR_Malloc (sizeof(delete_message_struct));
            // convert name back from utf7
            nsCString canonicalName;
            const char *selectedMailboxName = GetServerStateParser().GetSelectedMailboxName();
            if (selectedMailboxName)
            {
              m_runningUrl->AllocateCanonicalPath(selectedMailboxName,
                kOnlineHierarchySeparatorUnknown, getter_Copies(canonicalName));
            }

            if (m_imapMessageSink)
              m_imapMessageSink->NotifyMessageDeleted(canonicalName.get(), false, messageIdString.get());
            // notice we don't wait for this to finish...
          }
          else
            HandleMemoryFailure();
        }
        break;
      case nsIImapUrl::nsImapDeleteFolderAndMsgs:
        DeleteFolderAndMsgs(mailboxName.get());
        break;
      case nsIImapUrl::nsImapDeleteAllMsgs:
        {
          uint32_t numberOfMessages = GetServerStateParser().NumberOfMessages();
          if (numberOfMessages)
          {
            Store(NS_LITERAL_CSTRING("1:*"), "+FLAGS.SILENT (\\Deleted)",
              false);  // use sequence #'s

            if (GetServerStateParser().LastCommandSuccessful())
              Expunge();      // expunge messages with deleted flag
            if (GetServerStateParser().LastCommandSuccessful())
            {
              // convert name back from utf7
              nsCString canonicalName;
              const char *selectedMailboxName = GetServerStateParser().GetSelectedMailboxName();
              if (selectedMailboxName )
              {
                m_runningUrl->AllocateCanonicalPath(selectedMailboxName,
                  kOnlineHierarchySeparatorUnknown, getter_Copies(canonicalName));
              }

              if (m_imapMessageSink)
                m_imapMessageSink->NotifyMessageDeleted(canonicalName.get(), true, nullptr);
            }

          }
          bool deleteSelf = false;
          DeleteSubFolders(mailboxName.get(), deleteSelf);  // don't delete self
        }
        break;
      case nsIImapUrl::nsImapAppendDraftFromFile:
        {
          OnAppendMsgFromFile();
        }
        break;
      case nsIImapUrl::nsImapAddMsgFlags:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);

          ProcessStoreFlags(messageIdString, bMessageIdsAreUids,
            msgFlags, true);
        }
        break;
      case nsIImapUrl::nsImapSubtractMsgFlags:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);

          ProcessStoreFlags(messageIdString, bMessageIdsAreUids,
            msgFlags, false);
        }
        break;
      case nsIImapUrl::nsImapSetMsgFlags:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);

          ProcessStoreFlags(messageIdString, bMessageIdsAreUids,
            msgFlags, true);
          ProcessStoreFlags(messageIdString, bMessageIdsAreUids,
            ~msgFlags, false);
        }
        break;
      case nsIImapUrl::nsImapBiff:
        PeriodicBiff();
        break;
      case nsIImapUrl::nsImapOnlineCopy:
      case nsIImapUrl::nsImapOnlineMove:
        {
          nsCString messageIdString;
          m_runningUrl->GetListOfMessageIds(messageIdString);
          char *destinationMailbox = OnCreateServerDestinationFolderPathString();

          if (destinationMailbox)
          {
            if (m_imapAction == nsIImapUrl::nsImapOnlineMove)
            {
              if (HandlingMultipleMessages(messageIdString))
                ProgressEventFunctionUsingNameWithString("imapMovingMessages", destinationMailbox);
              else
                ProgressEventFunctionUsingNameWithString("imapMovingMessage", destinationMailbox);
            }
            else {
              if (HandlingMultipleMessages(messageIdString))
                ProgressEventFunctionUsingNameWithString("imapCopyingMessages", destinationMailbox);
              else
                ProgressEventFunctionUsingNameWithString("imapCopyingMessage", destinationMailbox);
            }
            Copy(messageIdString.get(), destinationMailbox, bMessageIdsAreUids);
            PR_FREEIF( destinationMailbox);
            ImapOnlineCopyState copyState;
            if (DeathSignalReceived())
              copyState = ImapOnlineCopyStateType::kInterruptedState;
            else
              copyState = GetServerStateParser().LastCommandSuccessful() ?
              (ImapOnlineCopyState) ImapOnlineCopyStateType::kSuccessfulCopy :
            (ImapOnlineCopyState) ImapOnlineCopyStateType::kFailedCopy;
            if (m_imapMailFolderSink)
              m_imapMailFolderSink->OnlineCopyCompleted(this, copyState);
            // Don't mark message 'Deleted' for AOL servers or standard imap servers
            // that support MOVE since we already issued an 'xaol-move' or 'move' command.
            if (GetServerStateParser().LastCommandSuccessful() &&
              (m_imapAction == nsIImapUrl::nsImapOnlineMove) &&
              !(GetServerStateParser().ServerIsAOLServer() ||
                GetServerStateParser().GetCapabilityFlag() & kHasMoveCapability))
            {
              // Simulate MOVE for servers that don't support MOVE: do COPY-DELETE-EXPUNGE.
              Store(messageIdString, "+FLAGS (\\Deleted \\Seen)",
                bMessageIdsAreUids);
              bool storeSuccessful = GetServerStateParser().LastCommandSuccessful();
              if (storeSuccessful)
              {
                if(gExpungeAfterDelete)
                {
                  // This will expunge all emails marked as deleted in mailbox,
                  // not just the ones marked as deleted above.
                  Expunge();
                }
                else
                {
                  // Check if UIDPLUS capable so we can just expunge emails we just
                  // copied and marked as deleted. This prevents expunging emails
                  // that other clients may have marked as deleted in the mailbox
                  // and don't want them to disappear.
                  // Only do UidExpunge() when user selected delete method is "Move
                  // it to this folder" or "Remove it immediately", not when the
                  // delete method is "Just mark it as deleted".
                  if (!GetShowDeletedMessages() &&
                      (GetServerStateParser().GetCapabilityFlag() & kUidplusCapability))
                  {
                    UidExpunge(messageIdString);
                  }
                }
              }
              if (m_imapMailFolderSink)
              {
                copyState = storeSuccessful ? (ImapOnlineCopyState) ImapOnlineCopyStateType::kSuccessfulDelete
                  : (ImapOnlineCopyState) ImapOnlineCopyStateType::kFailedDelete;
                m_imapMailFolderSink->OnlineCopyCompleted(this, copyState);
              }
            }
          }
          else
            HandleMemoryFailure();
        }
        break;
      case nsIImapUrl::nsImapOnlineToOfflineCopy:
      case nsIImapUrl::nsImapOnlineToOfflineMove:
        {
          nsCString messageIdString;
          nsresult rv = m_runningUrl->GetListOfMessageIds(messageIdString);
          if (NS_SUCCEEDED(rv))
          {
            SetProgressString(IMAP_MESSAGES_STRING_INDEX);
            m_progressCurrentNumber[m_stringIndex] = 0;
            m_progressExpectedNumber = CountMessagesInIdString(messageIdString.get());

            FetchMessage(messageIdString, kEveryThingRFC822Peek);

            SetProgressString(IMAP_EMPTY_STRING_INDEX);
            if (m_imapMailFolderSink)
            {
              ImapOnlineCopyState copyStatus;
              copyStatus = GetServerStateParser().LastCommandSuccessful() ?
                ImapOnlineCopyStateType::kSuccessfulCopy : ImapOnlineCopyStateType::kFailedCopy;

              m_imapMailFolderSink->OnlineCopyCompleted(this, copyStatus);
              if (GetServerStateParser().LastCommandSuccessful() &&
                (m_imapAction == nsIImapUrl::nsImapOnlineToOfflineMove))
              {
                Store(messageIdString, "+FLAGS (\\Deleted \\Seen)",bMessageIdsAreUids);
                if (GetServerStateParser().LastCommandSuccessful())
                {
                  copyStatus = ImapOnlineCopyStateType::kSuccessfulDelete;
                  if (gExpungeAfterDelete)
                    Expunge();
                }
                else
                  copyStatus = ImapOnlineCopyStateType::kFailedDelete;

                m_imapMailFolderSink->OnlineCopyCompleted(this,  copyStatus);
              }
            }
          }
          else
            HandleMemoryFailure();
        }
        break;
      default:
        if (GetServerStateParser().LastCommandSuccessful() && !uidValidityOk)
          ProcessMailboxUpdate(false); // handle uidvalidity change
        break;
      }
    }
  }
  else if (!DeathSignalReceived())
    HandleMemoryFailure();
}

nsresult nsImapProtocol::BeginMessageDownLoad(
                                              uint32_t total_message_size, // for user, headers and body
                                              const char *content_type)
{
  nsresult rv = NS_OK;
  char *sizeString = PR_smprintf("OPEN Size: %ld", total_message_size);
  Log("STREAM",sizeString,"Begin Message Download Stream");
  PR_Free(sizeString);
  // start counting how many bytes we see in this message after all transformations
  m_bytesToChannel = 0;

  if (content_type)
  {
    m_fromHeaderSeen = false;
    if (GetServerStateParser().GetDownloadingHeaders())
    {
      // if we get multiple calls to BeginMessageDownload w/o intervening
      // calls to NormalEndMessageDownload or Abort, then we're just
      // going to fake a NormalMessageEndDownload. This will most likely
      // cause an empty header to get written to the db, and the user
      // will have to delete the empty header themselves, which
      // should remove the message from the server as well.
      if (m_curHdrInfo)
        NormalMessageEndDownload();
      if (!m_curHdrInfo)
        m_curHdrInfo = m_hdrDownloadCache->StartNewHdr();
      if (m_curHdrInfo)
        m_curHdrInfo->SetMsgSize(total_message_size);
      return NS_OK;
    }
    // if we have a mock channel, that means we have a channel listener who wants the
    // message. So set up a pipe. We'll write the message into one end of the pipe
    // and they will read it out of the other end.
    if (m_channelListener)
    {
      // create a pipe to pump the message into...the output will go to whoever
      // is consuming the message display
      // we create an "infinite" pipe in case we get extremely long lines from the imap server,
      // and the consumer is waiting for a whole line
      nsCOMPtr<nsIPipe> pipe = do_CreateInstance("@mozilla.org/pipe;1");
      rv = pipe->Init(false, false, 4096, PR_UINT32_MAX);
      NS_ENSURE_SUCCESS(rv, rv);

      // These always succeed because the pipe is initialized above.
      MOZ_ALWAYS_SUCCEEDS(pipe->GetInputStream(getter_AddRefs(m_channelInputStream)));
      MOZ_ALWAYS_SUCCEEDS(pipe->GetOutputStream(getter_AddRefs(m_channelOutputStream)));
    }
    // else, if we are saving the message to disk!
    else if (m_imapMessageSink /* && m_imapAction == nsIImapUrl::nsImapSaveMessageToDisk */)
    {
      // we get here when download the inbox for offline use
      nsCOMPtr<nsIFile> file;
      bool addDummyEnvelope = true;
      nsCOMPtr<nsIMsgMessageUrl> msgurl = do_QueryInterface(m_runningUrl);
      msgurl->GetMessageFile(getter_AddRefs(file));
      msgurl->GetAddDummyEnvelope(&addDummyEnvelope);
      if (file)
        rv = m_imapMessageSink->SetupMsgWriteStream(file, addDummyEnvelope);
    }
    if (m_imapMailFolderSink && m_runningUrl)
    {
      nsCOMPtr <nsISupports> copyState;
      if (m_runningUrl)
      {
        m_runningUrl->GetCopyState(getter_AddRefs(copyState));
        if (copyState) // only need this notification during copy
        {
          nsCOMPtr<nsIMsgMailNewsUrl> mailurl = do_QueryInterface(m_runningUrl);
          m_imapMailFolderSink->StartMessage(mailurl);
        }
      }
    }

  }
  else
    HandleMemoryFailure();
  return rv;
}

void
nsImapProtocol::GetShouldDownloadAllHeaders(bool *aResult)
{
  if (m_imapMailFolderSink)
    m_imapMailFolderSink->GetShouldDownloadAllHeaders(aResult);
}

void
nsImapProtocol::GetArbitraryHeadersToDownload(nsCString &aResult)
{
  if (m_imapServerSink)
    m_imapServerSink->GetArbitraryHeaders(aResult);
}

void
nsImapProtocol::AdjustChunkSize()
{
  int32_t deltaInSeconds;
  PRTime2Seconds(m_endTime - m_startTime, &deltaInSeconds);
  m_trackingTime = false;
  if (deltaInSeconds < 0)
    return;            // bogus for some reason

  if (deltaInSeconds <= m_tooFastTime && m_curFetchSize >= m_chunkSize)
  {
    m_chunkSize += m_chunkAddSize;
    m_chunkThreshold = m_chunkSize + (m_chunkSize / 2);
    // we used to have a max for the chunk size - I don't think that's needed.
  }
  else if (deltaInSeconds <= m_idealTime)
    return;
  else
  {
    if (m_chunkSize > m_chunkStartSize)
      m_chunkSize = m_chunkStartSize;
    else if (m_chunkSize > (m_chunkAddSize * 2))
      m_chunkSize -= m_chunkAddSize;
    m_chunkThreshold = m_chunkSize + (m_chunkSize / 2);
  }
  // remember these new values globally so new connections
  // can take advantage of them.
  if (gChunkSize != m_chunkSize)
  {
    // will cause chunk size pref to be written in CloseStream.
    gChunkSizeDirty = true;
    gChunkSize = m_chunkSize;
    gChunkThreshold = m_chunkThreshold;
  }
}

// authenticated state commands

// escape any backslashes or quotes.  Backslashes are used a lot with our NT server
void nsImapProtocol::CreateEscapedMailboxName(const char *rawName, nsCString &escapedName)
{
  escapedName.Assign(rawName);

  for (int32_t strIndex = 0; *rawName; strIndex++)
  {
    char currentChar = *rawName++;
    if ((currentChar == '\\') || (currentChar == '\"'))
      escapedName.Insert('\\', strIndex++);
  }
}
void nsImapProtocol::SelectMailbox(const char *mailboxName)
{
  ProgressEventFunctionUsingNameWithString("imapStatusSelectingMailbox", mailboxName);
  IncrementCommandTagNumber();

  m_closeNeededBeforeSelect = false;   // initial value
  GetServerStateParser().ResetFlagInfo();
  nsCString escapedName;
  CreateEscapedMailboxName(mailboxName, escapedName);
  nsCString commandBuffer(GetServerCommandTag());
  commandBuffer.AppendLiteral(" select \"");
  commandBuffer.Append(escapedName.get());
  commandBuffer.Append('"');
  if (UseCondStore())
    commandBuffer.AppendLiteral(" (CONDSTORE)");
  commandBuffer.Append(CRLF);

  nsresult res;
  res = SendData(commandBuffer.get());
  if (NS_FAILED(res)) return;
  ParseIMAPandCheckForNewMail();

  int32_t numOfMessagesInFlagState = 0;
  nsImapAction imapAction;
  m_flagState->GetNumberOfMessages(&numOfMessagesInFlagState);
  res = m_runningUrl->GetImapAction(&imapAction);
  // if we've selected a mailbox, and we're not going to do an update because of the
  // url type, but don't have the flags, go get them!
  if (GetServerStateParser().LastCommandSuccessful() && NS_SUCCEEDED(res) &&
    imapAction != nsIImapUrl::nsImapSelectFolder && imapAction != nsIImapUrl::nsImapExpungeFolder
    && imapAction != nsIImapUrl::nsImapLiteSelectFolder &&
    imapAction != nsIImapUrl::nsImapDeleteAllMsgs &&
    ((GetServerStateParser().NumberOfMessages() != numOfMessagesInFlagState) && (numOfMessagesInFlagState == 0)))
  {
      ProcessMailboxUpdate(false);
  }
}

// Please call only with a single message ID
void nsImapProtocol::Bodystructure(const nsCString &messageId, bool idIsUid)
{
  IncrementCommandTagNumber();

  nsCString commandString(GetServerCommandTag());
  if (idIsUid)
    commandString.AppendLiteral(" UID");
  commandString.AppendLiteral(" fetch ");

  commandString.Append(messageId);
  commandString.AppendLiteral(" (BODYSTRUCTURE)" CRLF);

  nsresult rv = SendData(commandString.get());
  if (NS_SUCCEEDED(rv))
      ParseIMAPandCheckForNewMail(commandString.get());
}

void nsImapProtocol::PipelinedFetchMessageParts(const char *uid, nsIMAPMessagePartIDArray *parts)
{
  // assumes no chunking

  // build up a string to fetch
  nsCString stringToFetch, what;
  uint32_t currentPartNum = 0;
  while ((parts->GetNumParts() > currentPartNum) && !DeathSignalReceived())
  {
    nsIMAPMessagePartID *currentPart = parts->GetPart(currentPartNum);
    if (currentPart)
    {
      // Do things here depending on the type of message part
      // Append it to the fetch string
      if (currentPartNum > 0)
        stringToFetch.Append(' ');

      switch (currentPart->GetFields())
      {
      case kMIMEHeader:
        what = "BODY.PEEK[";
        what.Append(currentPart->GetPartNumberString());
        what.AppendLiteral(".MIME]");
        stringToFetch.Append(what);
        break;
      case kRFC822HeadersOnly:
        if (currentPart->GetPartNumberString())
        {
          what = "BODY.PEEK[";
          what.Append(currentPart->GetPartNumberString());
          what.AppendLiteral(".HEADER]");
          stringToFetch.Append(what);
        }
        else
        {
          // headers for the top-level message
          stringToFetch.AppendLiteral("BODY.PEEK[HEADER]");
        }
        break;
      default:
        NS_ASSERTION(false, "we should only be pipelining MIME headers and Message headers");
        break;
      }
    }
    currentPartNum++;
  }

  // Run the single, pipelined fetch command
  if ((parts->GetNumParts() > 0) && !DeathSignalReceived() && !GetPseudoInterrupted() && stringToFetch.get())
  {
      IncrementCommandTagNumber();

    nsCString commandString(GetServerCommandTag());
    commandString.AppendLiteral(" UID fetch ");
    commandString.Append(uid, 10);
    commandString.AppendLiteral(" (");
    commandString.Append(stringToFetch);
    commandString.AppendLiteral(")" CRLF);
    nsresult rv = SendData(commandString.get());
        if (NS_SUCCEEDED(rv))
            ParseIMAPandCheckForNewMail(commandString.get());
  }
}

void nsImapProtocol::FetchMsgAttribute(const nsCString &messageIds, const nsCString &attribute)
{
    IncrementCommandTagNumber();

    nsAutoCString commandString (GetServerCommandTag());
    commandString.AppendLiteral(" UID fetch ");
    commandString.Append(messageIds);
    commandString.AppendLiteral(" (");
    commandString.Append(attribute);
    commandString.AppendLiteral(")" CRLF);
    nsresult rv = SendData(commandString.get());

    if (NS_SUCCEEDED(rv))
       ParseIMAPandCheckForNewMail(commandString.get());
    GetServerStateParser().SetFetchingFlags(false);
    // Always clear this flag after every fetch.
    m_fetchingWholeMessage = false;
}

// this routine is used to fetch a message or messages, or headers for a
// message...

void nsImapProtocol::FallbackToFetchWholeMsg(const nsCString &messageId, uint32_t messageSize)
{
  if (m_imapMessageSink && m_runningUrl)
  {
    bool shouldStoreMsgOffline;
    m_runningUrl->GetStoreOfflineOnFallback(&shouldStoreMsgOffline);
    m_runningUrl->SetStoreResultsOffline(shouldStoreMsgOffline);
  }
  FetchTryChunking(messageId,
                   m_imapAction == nsIImapUrl::nsImapMsgFetchPeek ?
                     kEveryThingRFC822Peek : kEveryThingRFC822,
                   true, nullptr, messageSize, true);
}

void
nsImapProtocol::FetchMessage(const nsCString &messageIds,
                             nsIMAPeFetchFields whatToFetch,
                             const char *fetchModifier,
                             uint32_t startByte, uint32_t numBytes,
                             char *part)
{
  IncrementCommandTagNumber();

  nsCString commandString;
  commandString = "%s UID fetch";

  switch (whatToFetch) {
  case kEveryThingRFC822:
    m_flagChangeCount++;
    m_fetchingWholeMessage = true;
    if (m_trackingTime)
      AdjustChunkSize();      // we started another segment
    m_startTime = PR_Now();     // save start of download time
    m_trackingTime = true;
    MOZ_LOG(IMAP, LogLevel::Debug, ("FetchMessage everything: curFetchSize %u numBytes %u",
                                m_curFetchSize, numBytes));
    if (numBytes > 0)
      m_curFetchSize = numBytes;

    if (GetServerStateParser().ServerHasIMAP4Rev1Capability())
    {
      if (GetServerStateParser().GetCapabilityFlag() & kHasXSenderCapability)
        commandString.AppendLiteral(" %s (XSENDER UID RFC822.SIZE BODY[]");
      else
        commandString.AppendLiteral(" %s (UID RFC822.SIZE BODY[]");
    }
    else
    {
      if (GetServerStateParser().GetCapabilityFlag() & kHasXSenderCapability)
        commandString.AppendLiteral(" %s (XSENDER UID RFC822.SIZE RFC822");
      else
        commandString.AppendLiteral(" %s (UID RFC822.SIZE RFC822");
    }
    if (numBytes > 0)
    {
      // if we are retrieving chunks
      char *byterangeString = PR_smprintf("<%ld.%ld>",startByte, numBytes);
      if (byterangeString)
      {
        commandString.Append(byterangeString);
        PR_Free(byterangeString);
      }
    }
    commandString.Append(')');

    break;

  case kEveryThingRFC822Peek:
    {
      MOZ_LOG(IMAP, LogLevel::Debug, ("FetchMessage peek: curFetchSize %u numBytes %u",
                                  m_curFetchSize, numBytes));
      if (numBytes > 0)
        m_curFetchSize = numBytes;
      const char *formatString = "";
      eIMAPCapabilityFlags server_capabilityFlags = GetServerStateParser().GetCapabilityFlag();

      m_fetchingWholeMessage = true;
      if (server_capabilityFlags & kIMAP4rev1Capability)
      {
        // use body[].peek since rfc822.peek is not in IMAP4rev1
        if (server_capabilityFlags & kHasXSenderCapability)
          formatString = " %s (XSENDER UID RFC822.SIZE BODY.PEEK[]";
        else
          formatString = " %s (UID RFC822.SIZE BODY.PEEK[]";
      }
      else
      {
        if (server_capabilityFlags & kHasXSenderCapability)
          formatString = " %s (XSENDER UID RFC822.SIZE RFC822.peek";
        else
          formatString = " %s (UID RFC822.SIZE RFC822.peek";
      }

      commandString.Append(formatString);
      if (numBytes > 0)
      {
        // if we are retrieving chunks
        char *byterangeString = PR_smprintf("<%ld.%ld>",startByte, numBytes);
        if (byterangeString)
        {
          commandString.Append(byterangeString);
          PR_Free(byterangeString);
        }
      }
      commandString.Append(')');
    }
    break;
  case kHeadersRFC822andUid:
    if (GetServerStateParser().ServerHasIMAP4Rev1Capability())
    {
      eIMAPCapabilityFlags server_capabilityFlags = GetServerStateParser().GetCapabilityFlag();
      bool aolImapServer = ((server_capabilityFlags & kAOLImapCapability) != 0);
      bool downloadAllHeaders = false;
      // checks if we're filtering on "any header" or running a spam filter requiring all headers
      GetShouldDownloadAllHeaders(&downloadAllHeaders);

      if (!downloadAllHeaders)  // if it's ok -- no filters on any header, etc.
      {
        char *headersToDL = nullptr;
        char *what = nullptr;
        const char *dbHeaders = (gUseEnvelopeCmd) ? IMAP_DB_HEADERS : IMAP_ENV_AND_DB_HEADERS;
        nsCString arbitraryHeaders;
        GetArbitraryHeadersToDownload(arbitraryHeaders);
        for (uint32_t i = 0; i < mCustomDBHeaders.Length(); i++)
        {
          if (arbitraryHeaders.Find(mCustomDBHeaders[i], /* ignoreCase = */ true) == kNotFound)
          {
            if (!arbitraryHeaders.IsEmpty())
              arbitraryHeaders.Append(' ');
            arbitraryHeaders.Append(mCustomDBHeaders[i]);
          }
        }
        for (uint32_t i = 0; i < mCustomHeaders.Length(); i++)
        {
           if (arbitraryHeaders.Find(mCustomHeaders[i], /* ignoreCase = */ true) == kNotFound)
          {
            if (!arbitraryHeaders.IsEmpty())
              arbitraryHeaders.Append(' ');
            arbitraryHeaders.Append(mCustomHeaders[i]);
          }
        }
        if (arbitraryHeaders.IsEmpty())
          headersToDL = strdup(dbHeaders);
        else
          headersToDL = PR_smprintf("%s %s",dbHeaders, arbitraryHeaders.get());

        if (gUseEnvelopeCmd)
          what = PR_smprintf(" ENVELOPE BODY.PEEK[HEADER.FIELDS (%s)])", headersToDL);
        else
          what = PR_smprintf(" BODY.PEEK[HEADER.FIELDS (%s)])",headersToDL);
        free(headersToDL);
        if (what)
        {
          commandString.AppendLiteral(" %s (UID ");
           if (m_isGmailServer)
            commandString.AppendLiteral("X-GM-MSGID X-GM-THRID X-GM-LABELS ");
          if (aolImapServer)
            commandString.AppendLiteral(" XAOL.SIZE") ;
          else
            commandString.AppendLiteral("RFC822.SIZE");
          commandString.AppendLiteral(" FLAGS");
          commandString.Append(what);
          PR_Free(what);
        }
        else
        {
          commandString.AppendLiteral(" %s (UID RFC822.SIZE BODY.PEEK[HEADER] FLAGS)");
        }
      }
      else
        commandString.AppendLiteral(" %s (UID RFC822.SIZE BODY.PEEK[HEADER] FLAGS)");
    }
    else
      commandString.AppendLiteral(" %s (UID RFC822.SIZE RFC822.HEADER FLAGS)");
    break;
  case kUid:
    commandString.AppendLiteral(" %s (UID)");
    break;
  case kFlags:
    GetServerStateParser().SetFetchingFlags(true);
    commandString.AppendLiteral(" %s (FLAGS)");
    break;
  case kRFC822Size:
    commandString.AppendLiteral(" %s (RFC822.SIZE)");
    break;
  case kBodyStart:
    {
      int32_t numBytesToFetch;
      m_runningUrl->GetNumBytesToFetch(&numBytesToFetch);

      commandString.AppendLiteral(" %s (UID BODY.PEEK[HEADER.FIELDS (Content-Type Content-Transfer-Encoding)] BODY.PEEK[TEXT]<0.");
      commandString.AppendInt(numBytesToFetch);
      commandString.AppendLiteral(">)");
    }
    break;
  case kRFC822HeadersOnly:
    if (GetServerStateParser().ServerHasIMAP4Rev1Capability())
    {
      if (part)
      {
        commandString.AppendLiteral(" %s (BODY[");
        char *what = PR_smprintf("%s.HEADER])", part);
        if (what)
        {
          commandString.Append(what);
          PR_Free(what);
        }
        else
          HandleMemoryFailure();
      }
      else
      {
        // headers for the top-level message
        commandString.AppendLiteral(" %s (BODY[HEADER])");
      }
    }
    else
      commandString.AppendLiteral(" %s (RFC822.HEADER)");
    break;
  case kMIMEPart:
    commandString.AppendLiteral(" %s (BODY.PEEK[%s]");
    if (numBytes > 0)
    {
      // if we are retrieving chunks
      char *byterangeString = PR_smprintf("<%ld.%ld>",startByte, numBytes);
      if (byterangeString)
      {
        commandString.Append(byterangeString);
        PR_Free(byterangeString);
      }
    }
    commandString.Append(')');
    break;
  case kMIMEHeader:
    commandString.AppendLiteral(" %s (BODY[%s.MIME])");
    break;
  }

  if (fetchModifier)
    commandString.Append(fetchModifier);

  commandString.Append(CRLF);

  // since messageIds can be infinitely long, use a dynamic buffer rather than the fixed one
  const char *commandTag = GetServerCommandTag();
  int protocolStringSize = commandString.Length() + messageIds.Length() + PL_strlen(commandTag) + 1 +
    (part ? PL_strlen(part) : 0);
  char *protocolString = (char *) PR_CALLOC( protocolStringSize );

  if (protocolString)
  {
    char *cCommandStr = ToNewCString(commandString);
    if ((whatToFetch == kMIMEPart) ||
      (whatToFetch == kMIMEHeader))
    {
      PR_snprintf(protocolString,                                      // string to create
        protocolStringSize,                                      // max size
        cCommandStr,                                   // format string
        commandTag,                          // command tag
        messageIds.get(),
        part);
    }
    else
    {
      PR_snprintf(protocolString,                                      // string to create
        protocolStringSize,                                      // max size
        cCommandStr,                                   // format string
        commandTag,                          // command tag
        messageIds.get());
    }

    nsresult rv = SendData(protocolString);

    free(cCommandStr);
    if (NS_SUCCEEDED(rv))
      ParseIMAPandCheckForNewMail(protocolString);
    PR_Free(protocolString);
    GetServerStateParser().SetFetchingFlags(false);
    // Always clear this flag after every fetch.
    m_fetchingWholeMessage = false;
    if (GetServerStateParser().LastCommandSuccessful() && CheckNeeded())
      Check();
  }
  else
    HandleMemoryFailure();
}

void nsImapProtocol::FetchTryChunking(const nsCString &messageIds,
                                      nsIMAPeFetchFields whatToFetch,
                                      bool idIsUid,
                                      char *part,
                                      uint32_t downloadSize,
                                      bool tryChunking)
{
  GetServerStateParser().SetTotalDownloadSize(downloadSize);
  MOZ_LOG(IMAP, LogLevel::Debug, ("FetchTryChunking: curFetchSize %u", downloadSize));
  m_curFetchSize = downloadSize; // we'll change this if chunking.
  if (m_fetchByChunks && tryChunking &&
        GetServerStateParser().ServerHasIMAP4Rev1Capability() &&
    (downloadSize > (uint32_t) m_chunkThreshold))
  {
    uint32_t startByte = 0;
    m_curFetchSize = m_chunkSize;
    GetServerStateParser().ClearLastFetchChunkReceived();
    while (!DeathSignalReceived() && !GetPseudoInterrupted() &&
      !GetServerStateParser().GetLastFetchChunkReceived() &&
      GetServerStateParser().ContinueParse())
    {
      FetchMessage(messageIds,
             whatToFetch,
             nullptr,
             startByte, m_chunkSize,
             part);
      startByte += m_chunkSize;
    }

    // Only abort the stream if this is a normal message download
    // Otherwise, let the body shell abort the stream.
    if ((whatToFetch == kEveryThingRFC822)
      &&
      ((startByte > 0 && (startByte < downloadSize) &&
      (DeathSignalReceived() || GetPseudoInterrupted())) ||
      !GetServerStateParser().ContinueParse()))
    {
      AbortMessageDownLoad();
      PseudoInterrupt(false);
    }
  }
  else
  {
    // small message, or (we're not chunking and not doing bodystructure),
    // or the server is not rev1.
    // Just fetch the whole thing.
    FetchMessage(messageIds, whatToFetch, nullptr, 0, 0, part);
  }
}


void nsImapProtocol::PipelinedFetchMessageParts(nsCString &uid, nsIMAPMessagePartIDArray *parts)
{
  // assumes no chunking

  // build up a string to fetch
  nsCString stringToFetch;
  nsCString what;

  uint32_t currentPartNum = 0;
  while ((parts->GetNumParts() > currentPartNum) && !DeathSignalReceived())
  {
    nsIMAPMessagePartID *currentPart = parts->GetPart(currentPartNum);
    if (currentPart)
    {
      // Do things here depending on the type of message part
      // Append it to the fetch string
      if (currentPartNum > 0)
        stringToFetch += " ";

      switch (currentPart->GetFields())
      {
      case kMIMEHeader:
        what = "BODY.PEEK[";
        what += currentPart->GetPartNumberString();
        what += ".MIME]";
        stringToFetch += what;
        break;
      case kRFC822HeadersOnly:
        if (currentPart->GetPartNumberString())
        {
          what = "BODY.PEEK[";
          what += currentPart->GetPartNumberString();
          what += ".HEADER]";
          stringToFetch += what;
        }
        else
        {
          // headers for the top-level message
          stringToFetch += "BODY.PEEK[HEADER]";
        }
        break;
      default:
        NS_ASSERTION(false, "we should only be pipelining MIME headers and Message headers");
        break;
      }

    }
    currentPartNum++;
  }

  // Run the single, pipelined fetch command
  if ((parts->GetNumParts() > 0) && !DeathSignalReceived() && !GetPseudoInterrupted() && stringToFetch.get())
  {
      IncrementCommandTagNumber();

    char *commandString = PR_smprintf("%s UID fetch %s (%s)%s",
                                          GetServerCommandTag(), uid.get(),
                                          stringToFetch.get(), CRLF);

    if (commandString)
    {
      nsresult rv = SendData(commandString);
            if (NS_SUCCEEDED(rv))
                ParseIMAPandCheckForNewMail(commandString);
      PR_Free(commandString);
    }
    else
      HandleMemoryFailure();
  }
}


void
nsImapProtocol::PostLineDownLoadEvent(const char *line, uint32_t uidOfMessage)
{
  if (!GetServerStateParser().GetDownloadingHeaders())
  {
    uint32_t byteCount = PL_strlen(line);
    bool echoLineToMessageSink = false;
    // if we have a channel listener, then just spool the message
    // directly to the listener
    if (m_channelListener)
    {
      uint32_t count = 0;
      if (m_channelOutputStream)
      {
        nsresult rv = m_channelOutputStream->Write(line, byteCount, &count);
        NS_ASSERTION(count == byteCount, "IMAP channel pipe couldn't buffer entire write");
        if (NS_SUCCEEDED(rv))
        {
          m_channelListener->OnDataAvailable(m_mockChannel, m_channelContext, m_channelInputStream, 0, count);
        }
        // else some sort of explosion?
      }
    }
    if (m_runningUrl)
      m_runningUrl->GetStoreResultsOffline(&echoLineToMessageSink);

    m_bytesToChannel += byteCount;
    if (m_imapMessageSink && line && echoLineToMessageSink && !GetPseudoInterrupted())
      m_imapMessageSink->ParseAdoptedMsgLine(line, uidOfMessage, m_runningUrl);
  }
  // ***** We need to handle the pseudo interrupt here *****
}

// Handle a line seen by the parser.
// * The argument |lineCopy| must be nullptr or should contain the same string as
//   |line|.  |lineCopy| will be modified.
// * A line may be passed by parts, e.g., "part1 part2\r\n" may be passed as
//     HandleMessageDownLoadLine("part 1 ", 1);
//     HandleMessageDownLoadLine("part 2\r\n", 0);
//   However, it is assumed that a CRLF or a CRCRLF is never split (i.e., this is
//   ensured *before* invoking this method).
void nsImapProtocol::HandleMessageDownLoadLine(const char *line, bool isPartialLine,
                                               char *lineCopy)
{
  NS_ASSERTION(lineCopy == nullptr || !PL_strcmp(line, lineCopy),
               "line and lineCopy must contain the same string");
  const char *messageLine = line;
  uint32_t lineLength = strlen(messageLine);
  const char *cEndOfLine = messageLine + lineLength;
  char *localMessageLine = nullptr;

  // If we obtain a partial line (due to fetching by chunks), we do not
  // add/modify the end-of-line terminator.
  if (!isPartialLine)
  {
    // Change this line to native line termination, duplicate if necessary.
    // Do not assume that the line really ends in CRLF
    // to start with, even though it is supposed to be RFC822

    // normalize line endings to CRLF unless we are saving the message to disk
    bool canonicalLineEnding = true;
    nsCOMPtr<nsIMsgMessageUrl> msgUrl = do_QueryInterface(m_runningUrl);

    if (m_imapAction == nsIImapUrl::nsImapSaveMessageToDisk && msgUrl)
      msgUrl->GetCanonicalLineEnding(&canonicalLineEnding);

    NS_ASSERTION(MSG_LINEBREAK_LEN == 1 ||
                    (MSG_LINEBREAK_LEN == 2 && !PL_strcmp(CRLF, MSG_LINEBREAK)),
                    "violated assumptions on MSG_LINEBREAK");
    if (MSG_LINEBREAK_LEN == 1 && !canonicalLineEnding)
    {
      bool lineEndsWithCRorLF = lineLength >= 1 &&
        (cEndOfLine[-1] == '\r' || cEndOfLine[-1] == '\n');
      char *endOfLine;
      if (lineCopy && lineEndsWithCRorLF)  // true for most lines
      {
        endOfLine = lineCopy + lineLength;
        messageLine = lineCopy;
      }
      else
      {
        // leave enough room for one more char, MSG_LINEBREAK[0]
        localMessageLine = (char *) PR_MALLOC(lineLength + 2);
        if (!localMessageLine) // memory failure
          return;
        PL_strcpy(localMessageLine, line);
        endOfLine = localMessageLine + lineLength;
        messageLine = localMessageLine;
      }

      if (lineLength >= 2 &&
        endOfLine[-2] == '\r' &&
        endOfLine[-1] == '\n')
      {
        if(lineLength>=3 && endOfLine[-3] == '\r') // CRCRLF
        {
          endOfLine--;
          lineLength--;
        }
        /* CRLF -> CR or LF */
        endOfLine[-2] = MSG_LINEBREAK[0];
        endOfLine[-1] = '\0';
        lineLength--;
      }
      else if (lineLength >= 1 &&
        ((endOfLine[-1] == '\r') || (endOfLine[-1] == '\n')))
      {
        /* CR -> LF or LF -> CR */
        endOfLine[-1] = MSG_LINEBREAK[0];
      }
      else // no eol characters at all
      {
        endOfLine[0] = MSG_LINEBREAK[0]; // CR or LF
        endOfLine[1] = '\0';
        lineLength++;
      }
    }
    else  // enforce canonical CRLF linebreaks
    {
      if (lineLength==0 || (lineLength == 1 && cEndOfLine[-1] == '\n'))
      {
        messageLine = CRLF;
        lineLength = 2;
      }
      else if (cEndOfLine[-1] != '\n' || cEndOfLine[-2] != '\r' ||
               (lineLength >=3 && cEndOfLine[-3] == '\r'))
      {
        // The line does not end in CRLF (or it ends in CRCRLF).
        // Copy line and leave enough room for two more chars (CR and LF).
        localMessageLine = (char *) PR_MALLOC(lineLength + 3);
        if (!localMessageLine) // memory failure
            return;
        PL_strcpy(localMessageLine, line);
        char *endOfLine = localMessageLine + lineLength;
        messageLine = localMessageLine;

        if (lineLength>=3 && endOfLine[-1] == '\n' &&
            endOfLine[-2] == '\r')
        {
          // CRCRLF -> CRLF
          endOfLine[-2] = '\n';
          endOfLine[-1] = '\0';
          lineLength--;
        }
        else if ((endOfLine[-1] == '\r') || (endOfLine[-1] == '\n'))
        {
          // LF -> CRLF or CR -> CRLF
          endOfLine[-1] = '\r';
          endOfLine[0]  = '\n';
          endOfLine[1]  = '\0';
          lineLength++;
        }
        else // no eol characters at all
        {
          endOfLine[0] = '\r';
          endOfLine[1] = '\n';
          endOfLine[2] = '\0';
          lineLength += 2;
        }
      }
    }
  }
  NS_ASSERTION(lineLength == PL_strlen(messageLine), "lineLength not accurate");

  // check if sender obtained via XSENDER server extension matches "From:" field
  const char *xSenderInfo = GetServerStateParser().GetXSenderInfo();
  if (xSenderInfo && *xSenderInfo && !m_fromHeaderSeen)
  {
    if (!PL_strncmp("From: ", messageLine, 6))
    {
      m_fromHeaderSeen = true;
      if (PL_strstr(messageLine, xSenderInfo) != NULL)
          // Adding a X-Mozilla-Status line here is not very elegant but it
          // works.  Another X-Mozilla-Status line is added to the message when
          // downloading to a local folder; this new line will also contain the
          // 'authed' flag we are adding here.  (If the message is again
          // uploaded to the server, this flag is lost.)
          // 0x0200 == nsMsgMessageFlags::SenderAuthed
          HandleMessageDownLoadLine("X-Mozilla-Status: 0200\r\n", false);
      GetServerStateParser().FreeXSenderInfo();
    }
  }

  if (GetServerStateParser().GetDownloadingHeaders())
  {
    if (!m_curHdrInfo)
      BeginMessageDownLoad(GetServerStateParser().SizeOfMostRecentMessage(), MESSAGE_RFC822);
    if (m_curHdrInfo)
      m_curHdrInfo->CacheLine(messageLine, GetServerStateParser().CurrentResponseUID());
    PR_Free(localMessageLine);
    return;
  }
  // if this line is for a different message, or the incoming line is too big
  if (((m_downloadLineCache->CurrentUID() != GetServerStateParser().CurrentResponseUID()) && !m_downloadLineCache->CacheEmpty()) ||
      (m_downloadLineCache->SpaceAvailable() < lineLength + 1) )
    FlushDownloadCache();

  // so now the cache is flushed, but this string might still be to big
  if (m_downloadLineCache->SpaceAvailable() < lineLength + 1)
      PostLineDownLoadEvent(messageLine, GetServerStateParser().CurrentResponseUID());
  else
    m_downloadLineCache->CacheLine(messageLine, GetServerStateParser().CurrentResponseUID());

  PR_Free(localMessageLine);
}

void nsImapProtocol::FlushDownloadCache()
{
  if (!m_downloadLineCache->CacheEmpty())
  {
    msg_line_info *downloadLine = m_downloadLineCache->GetCurrentLineInfo();
    PostLineDownLoadEvent(downloadLine->adoptedMessageLine,
                          downloadLine->uidOfMessage);
    m_downloadLineCache->ResetCache();
  }
}

void nsImapProtocol::NormalMessageEndDownload()
{
  Log("STREAM", "CLOSE", "Normal Message End Download Stream");

  if (m_trackingTime)
    AdjustChunkSize();
  if (m_imapMailFolderSink && m_curHdrInfo && GetServerStateParser().GetDownloadingHeaders())
  {
    m_curHdrInfo->SetMsgSize(GetServerStateParser().SizeOfMostRecentMessage());
    m_curHdrInfo->SetMsgUid(GetServerStateParser().CurrentResponseUID());
    m_hdrDownloadCache->FinishCurrentHdr();
    int32_t numHdrsCached;
    m_hdrDownloadCache->GetNumHeaders(&numHdrsCached);
    if (numHdrsCached == kNumHdrsToXfer)
    {
      m_imapMailFolderSink->ParseMsgHdrs(this, m_hdrDownloadCache);
      m_hdrDownloadCache->ResetAll();
    }
  }
  FlushDownloadCache();

  if (!GetServerStateParser().GetDownloadingHeaders())
  {
    int32_t updatedMessageSize = -1;
    if (m_fetchingWholeMessage)
    {
      updatedMessageSize = m_bytesToChannel;
      if (m_bytesToChannel != GetServerStateParser().SizeOfMostRecentMessage()) {
        MOZ_LOG(IMAP, LogLevel::Debug, ("STREAM:CLOSE Server's RFC822.SIZE %u, actual size %u",
                                    GetServerStateParser().SizeOfMostRecentMessage(),
                                    m_bytesToChannel));
      }
    }
    // need to know if we're downloading for display or not. We'll use action == nsImapMsgFetch for now
    nsImapAction imapAction = nsIImapUrl::nsImapSelectFolder; // just set it to some legal value
    if (m_runningUrl)
      m_runningUrl->GetImapAction(&imapAction);

    if (m_imapMessageSink)
      m_imapMessageSink->NormalEndMsgWriteStream(m_downloadLineCache->CurrentUID(), imapAction == nsIImapUrl::nsImapMsgFetch, m_runningUrl, updatedMessageSize);

    if (m_runningUrl && m_imapMailFolderSink)
    {
      nsCOMPtr <nsISupports> copyState;
      m_runningUrl->GetCopyState(getter_AddRefs(copyState));
      if (copyState) // only need this notification during copy
      {
        nsCOMPtr<nsIMsgMailNewsUrl> mailUrl (do_QueryInterface(m_runningUrl));
        m_imapMailFolderSink->EndMessage(mailUrl, m_downloadLineCache->CurrentUID());
      }
    }
  }
  m_curHdrInfo = nullptr;
}

void nsImapProtocol::AbortMessageDownLoad()
{
  Log("STREAM", "CLOSE", "Abort Message  Download Stream");

  if (m_trackingTime)
    AdjustChunkSize();
  FlushDownloadCache();
  if (GetServerStateParser().GetDownloadingHeaders())
  {
    if (m_imapMailFolderSink)
      m_imapMailFolderSink->AbortHeaderParseStream(this);
  }
  else if (m_imapMessageSink)
        m_imapMessageSink->AbortMsgWriteStream();

  m_curHdrInfo = nullptr;
}


void nsImapProtocol::ProcessMailboxUpdate(bool handlePossibleUndo)
{
  if (DeathSignalReceived())
    return;

  // Update quota information
  char *boxName;
  GetSelectedMailboxName(&boxName);
  GetQuotaDataIfSupported(boxName);
  PR_Free(boxName);

  // fetch the flags and uids of all existing messages or new ones
  if (!DeathSignalReceived() && GetServerStateParser().NumberOfMessages())
  {
    if (handlePossibleUndo)
    {
      // undo any delete flags we may have asked to
      nsCString undoIdsStr;
      nsAutoCString undoIds;

      GetCurrentUrl()->GetListOfMessageIds(undoIdsStr);
      undoIds.Assign(undoIdsStr);
      if (!undoIds.IsEmpty())
      {
        char firstChar = (char) undoIds.CharAt(0);
        undoIds.Cut(0, 1);  // remove first character
        // if this string started with a '-', then this is an undo of a delete
        // if its a '+' its a redo
        if (firstChar == '-')
          Store(undoIds, "-FLAGS (\\Deleted)", true);  // most servers will fail silently on a failure, deal with it?
        else  if (firstChar == '+')
          Store(undoIds, "+FLAGS (\\Deleted)", true);  // most servers will fail silently on a failure, deal with it?
        else
          NS_ASSERTION(false, "bogus undo Id's");
      }
    }

    // make the parser record these flags
    nsCString fetchStr;
    int32_t added = 0, deleted = 0;

    m_flagState->GetNumberOfMessages(&added);
    deleted = m_flagState->NumberOfDeletedMessages();
    bool flagStateEmpty = !added;
    bool useCS = UseCondStore();

    // Figure out if we need to do a full sync (UID Fetch Flags 1:*),
    // a partial sync using CHANGEDSINCE, or a sync from the previous
    // highwater mark.

    // If the folder doesn't know about the highest uid, or the flag state
    // is empty, and we're not using CondStore, we definitely need a full sync.
    //
    // Print to log items affecting needFullFolderSync:
    MOZ_LOG(IMAP_CS, LogLevel::Debug,
            ("Do full sync?: mFolderHighestUID=%" PRIu32 ", added=%" PRId32 ", useCS=%s",
             mFolderHighestUID, added, useCS ? "true" : "false"));
    bool needFullFolderSync = !mFolderHighestUID || (flagStateEmpty && !useCS);
    bool needFolderSync = false;

    if (!needFullFolderSync)
    {
      // Figure out if we need to do a non-highwater mark sync.
      // Set needFolderSync true when at least 1 of these 3 cases is true:
      // 1. Have no uids in flag array or all flag elements are marked deleted AND
      // not using CONDSTORE.
      // 2. Have no uids in flag array or all flag elements are marked deleted AND
      // using "just mark as deleted" and EXISTS response count differs from
      // stored message count for folder.
      // 3. Using CONDSTORE and highest MODSEQ response is not equal to stored
      // mod seq for folder.

      // Print to log items affecting needFolderSync:
      MOZ_LOG(IMAP_CS, LogLevel::Debug,
              ("1. Do a sync?: added=%" PRId32 ", deleted=%" PRId32 ", useCS=%s",
               added, deleted,  useCS ? "true" : "false"));
      MOZ_LOG(IMAP_CS, LogLevel::Debug,
              ("2. Do a sync?: ShowDeletedMsgs=%s, exists=%" PRId32 ", mFolderTotalMsgCount=%" PRId32 "",
               GetShowDeletedMessages() ? "true" : "false", GetServerStateParser().NumberOfMessages(),
               mFolderTotalMsgCount));
      MOZ_LOG(IMAP_CS, LogLevel::Debug,
              ("3. Do a sync?: fHighestModSeq=%" PRIu64 ", mFolderLastModSeq=%" PRIu64 "",
               GetServerStateParser().fHighestModSeq,  mFolderLastModSeq));

      needFolderSync =
        (
          (flagStateEmpty || added == deleted) &&
          (
            !useCS
            ||
            (GetShowDeletedMessages() &&
             GetServerStateParser().NumberOfMessages() != mFolderTotalMsgCount)
          )
        )
        ||
        (useCS && GetServerStateParser().fHighestModSeq != mFolderLastModSeq);
    }
    MOZ_LOG(IMAP_CS, LogLevel::Debug, ("needFullFolderSync=%s, needFolderSync=%s",
            needFullFolderSync ? "true" : "false", needFolderSync ? "true" : "false"));

    if (needFullFolderSync || needFolderSync)
    {
      nsCString idsToFetch("1:*");
      char fetchModifier[40] = "";
      if (!needFullFolderSync && !GetShowDeletedMessages() && useCS)
      {
        m_flagState->StartCapture();
        MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Doing UID fetch 1:* (CHANGEDSINCE %" PRIu64 ")",
                mFolderLastModSeq));
        PR_snprintf(fetchModifier, sizeof(fetchModifier), " (CHANGEDSINCE %llu)",
                    mFolderLastModSeq);
      }
      else
        m_flagState->SetPartialUIDFetch(false);

      FetchMessage(idsToFetch, kFlags, fetchModifier);
      // lets see if we should expunge during a full sync of flags.
      if (GetServerStateParser().LastCommandSuccessful())
      {
        // if we did a CHANGEDSINCE fetch, do a sanity check on the msg counts
        // to see if some other client may have done an expunge.
        if (m_flagState->GetPartialUIDFetch())
        {
          uint32_t numExists = GetServerStateParser().NumberOfMessages();
          uint32_t numPrevExists = mFolderTotalMsgCount;

          if (MOZ_LOG_TEST(IMAP_CS, LogLevel::Debug))
          {
            int32_t addedByPartialFetch;
            m_flagState->GetNumberOfMessages(&addedByPartialFetch);
            MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Sanity, deleted=%" PRId32 ", numPrevExists=%" PRIu32 ", numExists=%" PRIu32 "",
                    m_flagState->NumberOfDeletedMessages(), numPrevExists,  numExists));
            MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Sanity, addedByPartialFetch=%" PRId32 "",
                    addedByPartialFetch));
          }

          // Determine the number of new UIDs just fetched that are greater than
          // the saved highest UID for the folder. numToCheck will contain the
          // number of UIDs just fetched and, of course, not all are new.
          uint32_t numNewUIDs = 0;
          uint32_t numToCheck = m_flagState->GetNumAdded();
          bool flagChangeDetected = false;
          MOZ_LOG(IMAP_CS, LogLevel::Debug, ("numToCheck=%" PRIu32 "", numToCheck));
          if (numToCheck && mFolderHighestUID)
          {
            uint32_t uid;
            int32_t topIndex;
            m_flagState->GetNumberOfMessages(&topIndex);
            do {
              topIndex--;
              m_flagState->GetUidOfMessage(topIndex, &uid);
              if (uid && uid != nsMsgKey_None)
              {
                if (uid > mFolderHighestUID)
                {
                  numNewUIDs++;
                  MOZ_LOG(IMAP_CS, LogLevel::Debug, ("numNewUIDs=%" PRIu32 ", Added new UID=%" PRIu32 "",
                       numNewUIDs ,uid));
                  numToCheck--;
                }
                else
                {
                  // Just a flag change on an existing UID. No more new UIDs
                  // will be found. This does not detect an expunged message.
                  flagChangeDetected = true;
                  MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Not new uid = %" PRIu32 "", uid));
                  break;
                }
              }
            } while(numToCheck);
          }

          // Another client expunged at least one message if the number of new
          // UIDs is not equal to the observed change in the number of messages
          // existing in the folder.
          bool expungeHappened = numNewUIDs != (numExists - numPrevExists);
          if (expungeHappened)
          {
            // Sanity check failed - need full fetch to remove expunged msgs.
            MOZ_LOG(IMAP_CS, LogLevel::Debug,
                    ("Other client expunged msgs, do full fetch to remove expunged msgs"));
            m_flagState->Reset();
            m_flagState->SetPartialUIDFetch(false);
            FetchMessage(NS_LITERAL_CSTRING("1:*"), kFlags);
          }
          else if (numNewUIDs == 0)
          {
            // Nothing has been expunged and no new UIDs, so if just a flag
            // change on existing message(s), avoid unneeded fetch of flags for
            // messages with UIDs at and above uid (see var uid above) when
            // "highwater mark" fetch occurs below.
            if (mFolderHighestUID && flagChangeDetected)
            {
              MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Avoid unneeded fetches after just flag changes"));
              GetServerStateParser().ResetHighestRecordedUID();
            }
          }
        }
        int32_t numDeleted = m_flagState->NumberOfDeletedMessages();
        // Don't do expunge when we are lite selecting folder because we
        // could be doing undo.
        // Expunge if we're always expunging, or the number of deleted messages
        // is over the threshold, and we're either always respecting the
        // threshold, or we're expunging based on the delete model, and
        // the delete model is not the imap delete model.
        if (m_imapAction != nsIImapUrl::nsImapLiteSelectFolder &&
            (gExpungeOption == kAutoExpungeAlways ||
             (numDeleted >= gExpungeThreshold &&
              (gExpungeOption == kAutoExpungeOnThreshold ||
               (gExpungeOption == kAutoExpungeDeleteModel && !GetShowDeletedMessages())))))
          Expunge();
      }
    }
    else
    {
      // Obtain the highest (highwater mark) UID seen since the last UIDVALIDITY
      // response occurred (associated with the most recent SELECT for the folder).
      uint32_t highestRecordedUID = GetServerStateParser().HighestRecordedUID();
      // if we're using CONDSTORE, and the parser hasn't seen any UIDs, use
      // the highest UID previously seen and saved for the folder instead.
      if (useCS && !highestRecordedUID)
        highestRecordedUID = mFolderHighestUID;
      MOZ_LOG(IMAP_CS, LogLevel::Debug, ("Check for new messages above UID=%" PRIu32 "",
              highestRecordedUID));
      AppendUid(fetchStr, highestRecordedUID + 1);
      fetchStr.AppendLiteral(":*");
      FetchMessage(fetchStr, kFlags);      // only new messages please
    }
  }
  else if (GetServerStateParser().LastCommandSuccessful())
  {
    GetServerStateParser().ResetFlagInfo();
    // the flag state is empty, but not partial.
    m_flagState->SetPartialUIDFetch(false);
  }

  if (GetServerStateParser().LastCommandSuccessful())
  {
    nsImapAction imapAction;
    nsresult res = m_runningUrl->GetImapAction(&imapAction);
    if (NS_SUCCEEDED(res) && imapAction == nsIImapUrl::nsImapLiteSelectFolder)
      return;
  }

  bool entered_waitForBodyIdsMonitor = false;

  uint32_t *msgIdList = nullptr;
  uint32_t msgCount = 0;

  RefPtr<nsImapMailboxSpec> new_spec = GetServerStateParser().CreateCurrentMailboxSpec();
  if (new_spec && GetServerStateParser().LastCommandSuccessful())
  {
    nsImapAction imapAction;
    nsresult res = m_runningUrl->GetImapAction(&imapAction);
    if (NS_SUCCEEDED(res) && imapAction == nsIImapUrl::nsImapExpungeFolder)
      new_spec->mBoxFlags |= kJustExpunged;
    m_waitForBodyIdsMonitor.Enter();
    entered_waitForBodyIdsMonitor = true;

    if (m_imapMailFolderSink)
    {
      bool more;
      m_imapMailFolderSink->UpdateImapMailboxInfo(this, new_spec);
      m_imapMailFolderSink->GetMsgHdrsToDownload(&more, &m_progressExpectedNumber,
                                                 &msgCount, &msgIdList);
      // Assert that either it's empty string OR it must be header string.
      MOZ_ASSERT((m_stringIndex == IMAP_EMPTY_STRING_INDEX) || (m_stringIndex == IMAP_HEADERS_STRING_INDEX));
      m_progressCurrentNumber[m_stringIndex] = 0;
      m_runningUrl->SetMoreHeadersToDownload(more);
      // We're going to be re-running this url if there are more headers.
      if (more)
        m_runningUrl->SetRerunningUrl(true);
    }
  }

  if (GetServerStateParser().LastCommandSuccessful())
  {
    if (entered_waitForBodyIdsMonitor)
      m_waitForBodyIdsMonitor.Exit();

    if (msgIdList && !DeathSignalReceived() && GetServerStateParser().LastCommandSuccessful())
    {
      FolderHeaderDump(msgIdList, msgCount);
      free( msgIdList);
    }
    HeaderFetchCompleted();
      // this might be bogus, how are we going to do pane notification and stuff when we fetch bodies without
      // headers!
  }
  else if (entered_waitForBodyIdsMonitor) // need to exit this monitor if death signal received
    m_waitForBodyIdsMonitor.Exit();

  // wait for a list of bodies to fetch.
  if (GetServerStateParser().LastCommandSuccessful())
  {
    WaitForPotentialListOfBodysToFetch(&msgIdList, msgCount);
    if ( msgCount && GetServerStateParser().LastCommandSuccessful())
    {
      // Tell the url that it should store the msg fetch results offline,
      // while we're dumping the messages, and then restore the setting.
      bool wasStoringOffline;
      m_runningUrl->GetStoreResultsOffline(&wasStoringOffline);
      m_runningUrl->SetStoreResultsOffline(true);
      // Assert that either it's empty string OR it must be message string.
      MOZ_ASSERT((m_stringIndex == IMAP_EMPTY_STRING_INDEX) || (m_stringIndex == IMAP_MESSAGES_STRING_INDEX));
      m_progressCurrentNumber[m_stringIndex] = 0;
      m_progressExpectedNumber = msgCount;
      FolderMsgDump(msgIdList, msgCount, kEveryThingRFC822Peek);
      m_runningUrl->SetStoreResultsOffline(wasStoringOffline);
    }
  }
  if (!GetServerStateParser().LastCommandSuccessful())
    GetServerStateParser().ResetFlagInfo();
}

void nsImapProtocol::FolderHeaderDump(uint32_t *msgUids, uint32_t msgCount)
{
  FolderMsgDump(msgUids, msgCount, kHeadersRFC822andUid);
}

void nsImapProtocol::FolderMsgDump(uint32_t *msgUids, uint32_t msgCount, nsIMAPeFetchFields fields)
{
  // lets worry about this progress stuff later.
  switch (fields) {
  case kHeadersRFC822andUid:
    SetProgressString(IMAP_HEADERS_STRING_INDEX);
    break;
  case kFlags:
    SetProgressString(IMAP_FLAGS_STRING_INDEX);
    break;
  default:
    SetProgressString(IMAP_MESSAGES_STRING_INDEX);
    break;
  }

  FolderMsgDumpLoop(msgUids, msgCount, fields);

  SetProgressString(IMAP_EMPTY_STRING_INDEX);
}

void nsImapProtocol::WaitForPotentialListOfBodysToFetch(uint32_t **msgIdList, uint32_t &msgCount)
{
  PRIntervalTime sleepTime = kImapSleepTime;

  ReentrantMonitorAutoEnter fetchListMon(m_fetchBodyListMonitor);
  while(!m_fetchBodyListIsNew && !DeathSignalReceived())
    fetchListMon.Wait(sleepTime);
  m_fetchBodyListIsNew = false;

  *msgIdList = m_fetchBodyIdList;
  msgCount   = m_fetchBodyCount;
}

// libmsg uses this to notify a running imap url about message bodies it should download.
// why not just have libmsg explicitly download the message bodies?
NS_IMETHODIMP nsImapProtocol::NotifyBodysToDownload(uint32_t *keys, uint32_t keyCount)
{
  ReentrantMonitorAutoEnter fetchListMon(m_fetchBodyListMonitor);
  PR_FREEIF(m_fetchBodyIdList);
  m_fetchBodyIdList = (uint32_t *) PR_MALLOC(keyCount * sizeof(uint32_t));
  if (m_fetchBodyIdList)
    memcpy(m_fetchBodyIdList, keys, keyCount * sizeof(uint32_t));
  m_fetchBodyCount    = keyCount;
  m_fetchBodyListIsNew = true;
  fetchListMon.Notify();
  return NS_OK;
}

NS_IMETHODIMP nsImapProtocol::GetFlagsForUID(uint32_t uid, bool *foundIt, imapMessageFlagsType *resultFlags, char **customFlags)
{
  int32_t i;

  imapMessageFlagsType flags = m_flagState->GetMessageFlagsFromUID(uid, foundIt, &i);
  if (*foundIt)
  {
    *resultFlags = flags;
    if ((flags & kImapMsgCustomKeywordFlag) && customFlags)
      m_flagState->GetCustomFlags(uid, customFlags);
  }
  return NS_OK;
}

NS_IMETHODIMP nsImapProtocol::GetFlagAndUidState(nsIImapFlagAndUidState **aFlagState)
{
  NS_ENSURE_ARG_POINTER(aFlagState);
  NS_IF_ADDREF(*aFlagState = m_flagState);
  return NS_OK;
}

NS_IMETHODIMP nsImapProtocol::GetSupportedUserFlags(uint16_t *supportedFlags)
{
  if (!supportedFlags)
    return NS_ERROR_NULL_POINTER;

  *supportedFlags = m_flagState->GetSupportedUserFlags();
  return NS_OK;
}
void nsImapProtocol::FolderMsgDumpLoop(uint32_t *msgUids, uint32_t msgCount, nsIMAPeFetchFields fields)
{
  int32_t msgCountLeft = msgCount;
  uint32_t msgsDownloaded = 0;
  do
  {
    nsCString idString;
    uint32_t msgsToDownload = msgCountLeft;
    AllocateImapUidString(msgUids + msgsDownloaded, msgsToDownload, m_flagState, idString);  // 20 * 200
    FetchMessage(idString, fields);
    msgsDownloaded += msgsToDownload;
    msgCountLeft -= msgsToDownload;
  }
  while (msgCountLeft > 0 && !DeathSignalReceived());
}

void nsImapProtocol::HeaderFetchCompleted()
{
  if (m_imapMailFolderSink)
    m_imapMailFolderSink->ParseMsgHdrs(this, m_hdrDownloadCache);
  m_hdrDownloadCache->ReleaseAll();

  if (m_imapMailFolderSink)
    m_imapMailFolderSink->HeaderFetchCompleted(this);
}


// Use the noop to tell the server we are still here, and therefore we are willing to receive
// status updates. The recent or exists response from the server could tell us that there is
// more mail waiting for us, but we need to check the flags of the mail and the high water mark
// to make sure that we do not tell the user that there is new mail when perhaps they have
// already read it in another machine.

void nsImapProtocol::PeriodicBiff()
{

  nsMsgBiffState startingState = m_currentBiffState;

  if (GetServerStateParser().GetIMAPstate() == nsImapServerResponseParser::kFolderSelected)
  {
    Noop(); // check the latest number of messages
    int32_t numMessages = 0;
    m_flagState->GetNumberOfMessages(&numMessages);
    if (GetServerStateParser().NumberOfMessages() != numMessages)
    {
      uint32_t id = GetServerStateParser().HighestRecordedUID() + 1;
      nsCString fetchStr;           // only update flags
      uint32_t added = 0, deleted = 0;

      deleted = m_flagState->NumberOfDeletedMessages();
      added = numMessages;
      if (!added || (added == deleted)) // empty keys, get them all
        id = 1;

      //sprintf(fetchStr, "%ld:%ld", id, id + GetServerStateParser().NumberOfMessages() - fFlagState->GetNumberOfMessages());
      AppendUid(fetchStr, id);
      fetchStr.AppendLiteral(":*");
      FetchMessage(fetchStr, kFlags);
      if (((uint32_t) m_flagState->GetHighestNonDeletedUID() >= id) && m_flagState->IsLastMessageUnseen())
        m_currentBiffState = nsIMsgFolder::nsMsgBiffState_NewMail;
      else
        m_currentBiffState = nsIMsgFolder::nsMsgBiffState_NoMail;
    }
    else
      m_currentBiffState = nsIMsgFolder::nsMsgBiffState_NoMail;
  }
  else
    m_currentBiffState = nsIMsgFolder::nsMsgBiffState_Unknown;

  if (startingState != m_currentBiffState)
    SendSetBiffIndicatorEvent(m_currentBiffState);
}

void nsImapProtocol::SendSetBiffIndicatorEvent(nsMsgBiffState newState)
{
    if (m_imapMailFolderSink)
      m_imapMailFolderSink->SetBiffStateAndUpdate(newState);
}

/* static */ void nsImapProtocol::LogImapUrl(const char *logMsg, nsIImapUrl *imapUrl)
{
  if (MOZ_LOG_TEST(IMAP, LogLevel::Info))
  {
    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(imapUrl);
    if (mailnewsUrl)
    {
      nsAutoCString urlSpec, unescapedUrlSpec;
      nsresult rv = mailnewsUrl->GetSpec(urlSpec);
      if (NS_FAILED(rv))
        return;
      MsgUnescapeString(urlSpec, 0, unescapedUrlSpec);
      MOZ_LOG(IMAP, LogLevel::Info, ("%s:%s", logMsg, unescapedUrlSpec.get()));
    }
  }
}

// log info including current state...
void nsImapProtocol::Log(const char *logSubName, const char *extraInfo, const char *logData)
{
  if (MOZ_LOG_TEST(IMAP, LogLevel::Info))
  {
    static const char nonAuthStateName[] = "NA";
    static const char authStateName[] = "A";
    static const char selectedStateName[] = "S";
    const nsCString& hostName = GetImapHostName();  // initialize to empty string

    int32_t logDataLen = PL_strlen(logData); // PL_strlen checks for null
    nsCString logDataLines;
    const char *logDataToLog;
    int32_t lastLineEnd;

    const int kLogDataChunkSize = 400; // nspr line length is 512, and we
                                       // allow some space for the log preamble.

    // break up buffers > 400 bytes on line boundaries.
    if (logDataLen > kLogDataChunkSize)
    {
      logDataLines.Assign(logData);
      lastLineEnd = MsgRFindChar(logDataLines, '\n', kLogDataChunkSize);
      // null terminate the last line
      if (lastLineEnd == kNotFound)
        lastLineEnd = kLogDataChunkSize - 1;

      logDataLines.Insert( '\0', lastLineEnd + 1);
      logDataToLog = logDataLines.get();
    }
    else
    {
      logDataToLog = logData;
      lastLineEnd = logDataLen;
    }
    switch (GetServerStateParser().GetIMAPstate())
    {
    case nsImapServerResponseParser::kFolderSelected:
      if (extraInfo)
        MOZ_LOG(IMAP, LogLevel::Info, ("%p:%s:%s-%s:%s:%s: %.400s", this, hostName.get(),
               selectedStateName, GetServerStateParser().GetSelectedMailboxName(),
               logSubName, extraInfo, logDataToLog));
      else
        MOZ_LOG(IMAP, LogLevel::Info, ("%p:%s:%s-%s:%s: %.400s", this, hostName.get(),
               selectedStateName, GetServerStateParser().GetSelectedMailboxName(),
               logSubName, logDataToLog));
      break;
    case nsImapServerResponseParser::kNonAuthenticated:
    case nsImapServerResponseParser::kAuthenticated:
    {
      const char *stateName = (GetServerStateParser().GetIMAPstate() ==
                               nsImapServerResponseParser::kNonAuthenticated)
                               ? nonAuthStateName : authStateName;
      if (extraInfo)
        MOZ_LOG(IMAP, LogLevel::Info, ("%p:%s:%s:%s:%s: %.400s", this,
               hostName.get(),stateName,logSubName,extraInfo,logDataToLog));
      else
        MOZ_LOG(IMAP, LogLevel::Info, ("%p:%s:%s:%s: %.400s", this,
               hostName.get(),stateName,logSubName,logDataToLog));
    }
    }

    // dump the rest of the string in < 400 byte chunks
    while (logDataLen > kLogDataChunkSize)
    {
      logDataLines.Cut(0, lastLineEnd + 2); // + 2 to account for the LF and the '\0' we added
      logDataLen = logDataLines.Length();
      lastLineEnd = (logDataLen > kLogDataChunkSize) ? MsgRFindChar(logDataLines, '\n', kLogDataChunkSize) : kNotFound;
      // null terminate the last line
      if (lastLineEnd == kNotFound)
        lastLineEnd = kLogDataChunkSize - 1;
      logDataLines.Insert( '\0', lastLineEnd + 1);
      logDataToLog = logDataLines.get();
      MOZ_LOG(IMAP, LogLevel::Info, ("%.400s", logDataToLog));
    }
  }
}

// In 4.5, this posted an event back to libmsg and blocked until it got a response.
// We may still have to do this.It would be nice if we could preflight this value,
// but we may not always know when we'll need it.
uint32_t nsImapProtocol::GetMessageSize(const char * messageId, bool idsAreUids)
{
  const char *folderFromParser = GetServerStateParser().GetSelectedMailboxName();
  if (!folderFromParser || !messageId || !m_runningUrl || !m_hostSessionList)
    return 0;

  char *folderName = nullptr;
  uint32_t size;

  nsIMAPNamespace *nsForMailbox = nullptr;
  m_hostSessionList->GetNamespaceForMailboxForHost(GetImapServerKey(),
                                                   folderFromParser,
                                                   nsForMailbox);

  m_runningUrl->AllocateCanonicalPath(folderFromParser,
                                      nsForMailbox
                                        ? nsForMailbox->GetDelimiter()
                                        : kOnlineHierarchySeparatorUnknown,
                                      &folderName);

  if (folderName && m_imapMessageSink)
    m_imapMessageSink->GetMessageSizeFromDB(messageId, &size);

  PR_FREEIF(folderName);

  if (DeathSignalReceived())
    size = 0;

  return size;
}

// message id string utility functions
/* static */bool nsImapProtocol::HandlingMultipleMessages(const nsCString & messageIdString)
{
  return (MsgFindCharInSet(messageIdString, ",:") != kNotFound);
}

uint32_t nsImapProtocol::CountMessagesInIdString(const char *idString)
{
  uint32_t numberOfMessages = 0;
  char *uidString = PL_strdup(idString);

  if (uidString)
  {
    // This is in the form <id>,<id>, or <id1>:<id2>
    char curChar = *uidString;
    bool isRange = false;
    int32_t curToken;
    int32_t saveStartToken=0;

    for (char *curCharPtr = uidString; curChar && *curCharPtr;)
    {
      char *currentKeyToken = curCharPtr;
      curChar = *curCharPtr;
      while (curChar != ':' && curChar != ',' && curChar != '\0')
        curChar = *curCharPtr++;
      *(curCharPtr - 1) = '\0';
      curToken = atol(currentKeyToken);
      if (isRange)
      {
        while (saveStartToken < curToken)
        {
          numberOfMessages++;
          saveStartToken++;
        }
      }

      numberOfMessages++;
      isRange = (curChar == ':');
      if (isRange)
        saveStartToken = curToken + 1;
    }
    PR_Free(uidString);
  }
  return numberOfMessages;
}


// It would be really nice not to have to use this method nearly as much as we did
// in 4.5 - we need to think about this some. Some of it may just go away in the new world order
bool nsImapProtocol::DeathSignalReceived()
{
  // ignore mock channel status if we've been pseudo interrupted
  // ### need to make sure we clear pseudo interrupted status appropriately.
  if (!GetPseudoInterrupted() && m_mockChannel)
  {
    nsresult returnValue;
    m_mockChannel->GetStatus(&returnValue);
    if (NS_FAILED(returnValue))
      return false;
  }

  // Check the other way of cancelling.
  ReentrantMonitorAutoEnter threadDeathMon(m_threadDeathMonitor);
  return m_threadShouldDie;
}

NS_IMETHODIMP nsImapProtocol::ResetToAuthenticatedState()
{
    GetServerStateParser().PreauthSetAuthenticatedState();
    return NS_OK;
}


NS_IMETHODIMP nsImapProtocol::GetSelectedMailboxName(char ** folderName)
{
    if (!folderName) return NS_ERROR_NULL_POINTER;
    if (GetServerStateParser().GetSelectedMailboxName())
        *folderName =
            PL_strdup((GetServerStateParser().GetSelectedMailboxName()));
    return NS_OK;
}

bool nsImapProtocol::GetPseudoInterrupted()
{
  ReentrantMonitorAutoEnter pseudoInterruptMon(m_pseudoInterruptMonitor);
  return m_pseudoInterrupted;
}

void nsImapProtocol::PseudoInterrupt(bool the_interrupt)
{
  ReentrantMonitorAutoEnter pseudoInterruptMon(m_pseudoInterruptMonitor);
  m_pseudoInterrupted = the_interrupt;
  if (the_interrupt)
    Log("CONTROL", NULL, "PSEUDO-Interrupted");
}

void  nsImapProtocol::SetActive(bool active)
{
  ReentrantMonitorAutoEnter dataMemberMon(m_dataMemberMonitor);
  m_active = active;
}

bool    nsImapProtocol::GetActive()
{
  ReentrantMonitorAutoEnter dataMemberMon(m_dataMemberMonitor);
  return m_active;
}

bool nsImapProtocol::GetShowAttachmentsInline()
{
  bool showAttachmentsInline = true;
  if (m_imapServerSink)
    m_imapServerSink->GetShowAttachmentsInline(&showAttachmentsInline);
  return showAttachmentsInline;

}

void nsImapProtocol::SetContentModified(IMAP_ContentModifiedType modified)
{
  if (m_runningUrl && m_imapMessageSink)
    m_imapMessageSink->SetContentModified(m_runningUrl, modified);
}


bool nsImapProtocol::GetShouldFetchAllParts()
{
  if (m_runningUrl  && !DeathSignalReceived())
  {
    nsImapContentModifiedType contentModified;
    if (NS_SUCCEEDED(m_runningUrl->GetContentModified(&contentModified)))
      return (contentModified == IMAP_CONTENT_FORCE_CONTENT_NOT_MODIFIED);
  }
  return true;
}

// Adds a set of rights for a given user on a given mailbox on the current host.
// if userName is NULL, it means "me," or MYRIGHTS.
void nsImapProtocol::AddFolderRightsForUser(const char *mailboxName, const char *userName, const char *rights)
{
  if (!userName)
    userName = "";
  if (m_imapServerSink)
    m_imapServerSink->AddFolderRights(nsDependentCString(mailboxName), nsDependentCString(userName),
                                      nsDependentCString(rights));
}

void nsImapProtocol::SetCopyResponseUid(const char *msgIdString)
{
  if (m_imapMailFolderSink)
    m_imapMailFolderSink->SetCopyResponseUid(msgIdString, m_runningUrl);
}

void nsImapProtocol::CommitNamespacesForHostEvent()
{
    if (m_imapServerSink)
        m_imapServerSink->CommitNamespaces();
}

// notifies libmsg that we have new capability data for the current host
void nsImapProtocol::CommitCapability()
{
    if (m_imapServerSink)
    {
        m_imapServerSink->SetCapability(GetServerStateParser().GetCapabilityFlag());
    }
}

// rights is a single string of rights, as specified by RFC2086, the IMAP ACL extension.
// Clears all rights for a given folder, for all users.
void nsImapProtocol::ClearAllFolderRights()
{
  if (m_imapMailFolderSink)
    m_imapMailFolderSink->ClearFolderRights();
}


char* nsImapProtocol::CreateNewLineFromSocket()
{
  bool needMoreData = false;
  char * newLine = nullptr;
  uint32_t numBytesInLine = 0;
  nsresult rv = NS_OK;
  // we hold a ref to the input stream in case we get cancelled from the
  // ui thread, which releases our ref to the input stream, and can
  // cause the pipe to get deleted before the monitor the read is
  // blocked on gets notified. When that happens, the imap thread
  // will stay blocked.
  nsCOMPtr <nsIInputStream> kungFuGrip = m_inputStream;
  do
  {
    newLine = m_inputStreamBuffer->ReadNextLine(m_inputStream, numBytesInLine, needMoreData, &rv);
    MOZ_LOG(IMAP, LogLevel::Debug, ("ReadNextLine [stream=%p nb=%u needmore=%u]\n",
        m_inputStream.get(), numBytesInLine, needMoreData));

  } while (!newLine && NS_SUCCEEDED(rv) && !DeathSignalReceived()); // until we get the next line and haven't been interrupted

  kungFuGrip = nullptr;

  if (NS_FAILED(rv))
  {
    switch (rv)
    {
        case NS_ERROR_UNKNOWN_HOST:
        case NS_ERROR_UNKNOWN_PROXY_HOST:
            AlertUserEventUsingName("imapUnknownHostError");
            break;
        case NS_ERROR_CONNECTION_REFUSED:
        case NS_ERROR_PROXY_CONNECTION_REFUSED:
            AlertUserEventUsingName("imapConnectionRefusedError");
            break;
        case NS_ERROR_NET_TIMEOUT:
        case NS_ERROR_NET_RESET:
        case NS_BASE_STREAM_CLOSED:
        case NS_ERROR_NET_INTERRUPT:
          // we should retry on RESET, especially for SSL...
          if ((TestFlag(IMAP_RECEIVED_GREETING) || rv == NS_ERROR_NET_RESET) &&
              m_runningUrl && !m_retryUrlOnError)
          {
            bool rerunningUrl;
            nsImapAction imapAction;
            m_runningUrl->GetRerunningUrl(&rerunningUrl);
            m_runningUrl->GetImapAction(&imapAction);
            // don't rerun if we already were rerunning. And don't rerun
            // online move/copies that timeout.
            if (!rerunningUrl && (rv != NS_ERROR_NET_TIMEOUT ||
                                 (imapAction != nsIImapUrl::nsImapOnlineCopy &&
                                  imapAction != nsIImapUrl::nsImapOnlineMove)))
            {
              m_runningUrl->SetRerunningUrl(true);
              m_retryUrlOnError = true;
              break;
            }
          }
          if (rv == NS_ERROR_NET_TIMEOUT)
            AlertUserEventUsingName("imapNetTimeoutError");
          else
            AlertUserEventUsingName(TestFlag(IMAP_RECEIVED_GREETING) ?
                                    "imapServerDisconnected" :
                                    "imapServerDroppedConnection");
          break;
        default:
          break;
    }

    nsAutoCString logMsg("clearing IMAP_CONNECTION_IS_OPEN - rv = ");
    logMsg.AppendInt(static_cast<uint32_t>(rv), 16);
    Log("CreateNewLineFromSocket", nullptr, logMsg.get());
    ClearFlag(IMAP_CONNECTION_IS_OPEN);
    TellThreadToDie();
  }
  Log("CreateNewLineFromSocket", nullptr, newLine);
  SetConnectionStatus(newLine && numBytesInLine ? NS_OK : rv); // set > 0 if string is not null or empty
  return newLine;
}

nsresult
nsImapProtocol::GetConnectionStatus()
{
  return m_connectionStatus;
}

void
nsImapProtocol::SetConnectionStatus(nsresult status)
{
  m_connectionStatus = status;
}

void
nsImapProtocol::NotifyMessageFlags(imapMessageFlagsType flags,
                                   const nsACString &keywords,
                                   nsMsgKey key, uint64_t highestModSeq)
{
    if (m_imapMessageSink)
    {
      // if we're selecting the folder, don't need to report the flags; we've already fetched them.
      if (m_imapAction != nsIImapUrl::nsImapSelectFolder &&
          (m_imapAction != nsIImapUrl::nsImapMsgFetch ||
          (flags & ~kImapMsgRecentFlag) != kImapMsgSeenFlag))
        m_imapMessageSink->NotifyMessageFlags(flags, keywords, key, highestModSeq);
    }
}

void
nsImapProtocol::NotifySearchHit(const char * hitLine)
{
    nsresult rv;
    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl, &rv);
    if (m_imapMailFolderSink)
        m_imapMailFolderSink->NotifySearchHit(mailnewsUrl, hitLine);
}

void nsImapProtocol::SetMailboxDiscoveryStatus(EMailboxDiscoverStatus status)
{
  ReentrantMonitorAutoEnter mon(m_dataMemberMonitor);
  m_discoveryStatus = status;
}

EMailboxDiscoverStatus nsImapProtocol::GetMailboxDiscoveryStatus( )
{
  ReentrantMonitorAutoEnter mon(m_dataMemberMonitor);
  return m_discoveryStatus;
}

bool
nsImapProtocol::GetSubscribingNow()
{
    // ***** code me *****
    return false;// ***** for now
}

void
nsImapProtocol::DiscoverMailboxSpec(nsImapMailboxSpec * adoptedBoxSpec)
{
  nsIMAPNamespace *ns = nullptr;

  NS_ASSERTION (m_hostSessionList, "fatal null host session list");
  if (!m_hostSessionList)
    return;

  m_hostSessionList->GetDefaultNamespaceOfTypeForHost(
    GetImapServerKey(), kPersonalNamespace, ns);
  const char *nsPrefix = ns ? ns->GetPrefix() : 0;

  if (m_specialXListMailboxes.Count() > 0)
  {
    int32_t hashValue = 0;
    nsCString strHashKey(adoptedBoxSpec->mAllocatedPathName);
    m_specialXListMailboxes.Get(strHashKey, &hashValue);
    adoptedBoxSpec->mBoxFlags |= hashValue;
  }

  switch (m_hierarchyNameState)
  {
  case kXListing:
    if (adoptedBoxSpec->mBoxFlags &
        (kImapXListTrash|kImapAllMail|kImapInbox|kImapSent|kImapSpam|kImapDrafts))
    {
      nsCString mailboxName(adoptedBoxSpec->mAllocatedPathName);
      m_specialXListMailboxes.Put(mailboxName, adoptedBoxSpec->mBoxFlags);
      // Remember hierarchy delimiter in case this is the first time we've
      // connected to the server and we need it to be correct for the two-level
      // XLIST we send (INBOX is guaranteed to be in the first response).
      if (adoptedBoxSpec->mBoxFlags & kImapInbox)
        m_runningUrl->SetOnlineSubDirSeparator(adoptedBoxSpec->mHierarchySeparator);

    }
    break;
  case kListingForFolderFlags:
    {
      // store mailbox flags from LIST for use by LSUB
      nsCString mailboxName(adoptedBoxSpec->mAllocatedPathName);
      m_standardListMailboxes.Put(mailboxName, adoptedBoxSpec->mBoxFlags);
    }
    break;
  case kListingForCreate:
  case kNoOperationInProgress:
  case kDiscoverTrashFolderInProgress:
  case kListingForInfoAndDiscovery:
    {
      // standard mailbox specs are stored in m_standardListMailboxes
      // because LSUB does necessarily return all mailbox flags.
      // count should be > 0 only when we are looking at response of LSUB
      if (m_standardListMailboxes.Count() > 0)
      {
        int32_t hashValue = 0;
        nsCString strHashKey(adoptedBoxSpec->mAllocatedPathName);
        if (m_standardListMailboxes.Get(strHashKey, &hashValue))
          adoptedBoxSpec->mBoxFlags |= hashValue;
        else
          // if mailbox is not in hash list, then it is subscribed but does not
          // exist, so we make sure it can't be selected
          adoptedBoxSpec->mBoxFlags |= kNoselect;
      }
      if (ns && nsPrefix) // if no personal namespace, there can be no Trash folder
      {
        bool onlineTrashFolderExists = false;
        if (m_hostSessionList)
        {
          if (adoptedBoxSpec->mBoxFlags & (kImapTrash|kImapXListTrash))
          {
             m_hostSessionList->SetOnlineTrashFolderExistsForHost(GetImapServerKey(), true);
             onlineTrashFolderExists = true;
          }
          else
          {
            m_hostSessionList->GetOnlineTrashFolderExistsForHost(
                                  GetImapServerKey(), onlineTrashFolderExists);
          }
        }

        // Don't set the Trash flag if not using the Trash model
        if (GetDeleteIsMoveToTrash() && !onlineTrashFolderExists &&
            adoptedBoxSpec->mAllocatedPathName.Find(m_trashFolderPath, /* ignoreCase = */ true) != -1)
        {
          bool trashExists = false;
          if (StringBeginsWith(m_trashFolderPath,
                               NS_LITERAL_CSTRING("INBOX/"),
                               nsCaseInsensitiveCStringComparator()))
          {
            nsAutoCString pathName(adoptedBoxSpec->mAllocatedPathName.get() + 6);
            trashExists =
              StringBeginsWith(adoptedBoxSpec->mAllocatedPathName,
                               m_trashFolderPath,
                               nsCaseInsensitiveCStringComparator()) && /* "INBOX/" */
              pathName.Equals(Substring(m_trashFolderPath, 6), nsCaseInsensitiveCStringComparator());
          }
          else
            trashExists = adoptedBoxSpec->mAllocatedPathName.Equals(m_trashFolderPath, nsCaseInsensitiveCStringComparator());

          if (m_hostSessionList)
            m_hostSessionList->SetOnlineTrashFolderExistsForHost(GetImapServerKey(), trashExists);

          if (trashExists)
            adoptedBoxSpec->mBoxFlags |= kImapTrash;
        }
      }

      // Discover the folder (shuttle over to libmsg, yay)
      // Do this only if the folder name is not empty (i.e. the root)
      if (!adoptedBoxSpec->mAllocatedPathName.IsEmpty())
      {
        if (m_hierarchyNameState == kListingForCreate)
          adoptedBoxSpec->mBoxFlags |= kNewlyCreatedFolder;

        if (m_imapServerSink)
        {
          bool newFolder;

          m_imapServerSink->PossibleImapMailbox(adoptedBoxSpec->mAllocatedPathName,
                                                adoptedBoxSpec->mHierarchySeparator,
                                                adoptedBoxSpec->mBoxFlags, &newFolder);
          // if it's a new folder to the server sink, setting discovery status to
          // eContinueNew will cause us to get the ACL for the new folder.
          if (newFolder)
            SetMailboxDiscoveryStatus(eContinueNew);

          bool useSubscription = false;

          if (m_hostSessionList)
            m_hostSessionList->GetHostIsUsingSubscription(GetImapServerKey(),
                                                          useSubscription);

          if ((GetMailboxDiscoveryStatus() != eContinue) &&
              (GetMailboxDiscoveryStatus() != eContinueNew) &&
              (GetMailboxDiscoveryStatus() != eListMyChildren))
          {
            SetConnectionStatus(NS_ERROR_FAILURE);
          }
          else if (!adoptedBoxSpec->mAllocatedPathName.IsEmpty() &&
                   (GetMailboxDiscoveryStatus() == eListMyChildren) &&
                   (!useSubscription || GetSubscribingNow()))
          {
            NS_ASSERTION (false,
              "we should never get here anymore");
            SetMailboxDiscoveryStatus(eContinue);
          }
          else if (GetMailboxDiscoveryStatus() == eContinueNew)
          {
            if (m_hierarchyNameState == kListingForInfoAndDiscovery &&
                !adoptedBoxSpec->mAllocatedPathName.IsEmpty() &&
                !(adoptedBoxSpec->mBoxFlags & kNameSpace))
            {
              // remember the info here also
              nsIMAPMailboxInfo *mb = new nsIMAPMailboxInfo(adoptedBoxSpec->mAllocatedPathName, adoptedBoxSpec->mHierarchySeparator);
              m_listedMailboxList.AppendElement(mb);
            }
            SetMailboxDiscoveryStatus(eContinue);
          }
        }
      }
      }
      break;
    case kDeleteSubFoldersInProgress:
      {
        NS_ASSERTION(m_deletableChildren, "Oops .. null m_deletableChildren\n");
        m_deletableChildren->AppendElement(ToNewCString(adoptedBoxSpec->mAllocatedPathName));
      }
      break;
    case kListingForInfoOnly:
      {
        //UpdateProgressWindowForUpgrade(adoptedBoxSpec->allocatedPathName);
        ProgressEventFunctionUsingNameWithString("imapDiscoveringMailbox",
          adoptedBoxSpec->mAllocatedPathName.get());
        nsIMAPMailboxInfo *mb = new nsIMAPMailboxInfo(adoptedBoxSpec->mAllocatedPathName,
                                                      adoptedBoxSpec->mHierarchySeparator);
        m_listedMailboxList.AppendElement(mb);
      }
      break;
    case kDiscoveringNamespacesOnly:
      {
      }
      break;
    default:
      NS_ASSERTION (false, "we aren't supposed to be here");
      break;
  }
}

void
nsImapProtocol::AlertUserEventUsingName(const char* aMessageName)
{
  if (m_imapServerSink)
  {
    bool suppressErrorMsg = false;

    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl);
    if (mailnewsUrl)
      mailnewsUrl->GetSuppressErrorMsgs(&suppressErrorMsg);

    if (!suppressErrorMsg)
      m_imapServerSink->FEAlertWithName(aMessageName,
                                        mailnewsUrl);
  }
}

void
nsImapProtocol::AlertUserEvent(const char * message)
{
  if (m_imapServerSink)
  {
    bool suppressErrorMsg = false;

    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl);
    if (mailnewsUrl)
      mailnewsUrl->GetSuppressErrorMsgs(&suppressErrorMsg);

    if (!suppressErrorMsg)
      m_imapServerSink->FEAlert(NS_ConvertASCIItoUTF16(message), mailnewsUrl);
  }
}

void
nsImapProtocol::AlertUserEventFromServer(const char * aServerEvent)
{
    if (m_imapServerSink && aServerEvent)
    {
      nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl = do_QueryInterface(m_runningUrl);
      m_imapServerSink->FEAlertFromServer(nsDependentCString(aServerEvent),
                                          mailnewsUrl);
    }
}

void nsImapProtocol::ResetProgressInfo()
{
  m_lastProgressTime = 0;
  m_lastPercent = -1;
  m_lastProgressStringName.Truncate();
}

void nsImapProtocol::SetProgressString(uint32_t aStringIndex)
{
  m_stringIndex = aStringIndex;
  MOZ_ASSERT(m_stringIndex <= IMAP_EMPTY_STRING_INDEX);
  switch (m_stringIndex)
  {
    case IMAP_HEADERS_STRING_INDEX:
      m_progressStringName = "imapReceivingMessageHeaders3";
      break;
    case IMAP_MESSAGES_STRING_INDEX:
      m_progressStringName = "imapFolderReceivingMessageOf3";
      break;
    case IMAP_FLAGS_STRING_INDEX:
      m_progressStringName = "imapReceivingMessageFlags3";
      break;
    case IMAP_EMPTY_STRING_INDEX:
    default:
      break;
  }
}

void
nsImapProtocol::ShowProgress()
{
  if (m_imapServerSink && (m_stringIndex != IMAP_EMPTY_STRING_INDEX))
  {
    nsString progressString;
    const char *mailboxName = GetServerStateParser().GetSelectedMailboxName();
    nsString unicodeMailboxName;
    nsresult rv = CopyMUTF7toUTF16(nsDependentCString(mailboxName),
                                   unicodeMailboxName);
    NS_ENSURE_SUCCESS_VOID(rv);

    int32_t progressCurrentNumber = ++m_progressCurrentNumber[m_stringIndex];
    nsAutoString progressCurrentNumberString;
    progressCurrentNumberString.AppendInt(progressCurrentNumber);

    nsAutoString progressExpectedNumberString;
    progressExpectedNumberString.AppendInt(m_progressExpectedNumber);

    const char16_t *formatStrings[] = {
      progressCurrentNumberString.get(),
      progressExpectedNumberString.get(),
      unicodeMailboxName.get()
    };

    rv = m_bundle->FormatStringFromName(
                    m_progressStringName.get(),
                    formatStrings, 3, progressString);

    if (NS_SUCCEEDED(rv) && !progressString.IsEmpty())
    {
      PercentProgressUpdateEvent(progressString.get(), progressCurrentNumber,
                                m_progressExpectedNumber);
    }
  }
}

void
nsImapProtocol::ProgressEventFunctionUsingName(const char* aMsgName)
{
  if (m_imapMailFolderSink && !m_lastProgressStringName.Equals(aMsgName))
  {
    m_imapMailFolderSink->ProgressStatusString(this, aMsgName, nullptr);
    m_lastProgressStringName.Assign(aMsgName);
    // who's going to free this? Does ProgressStatusString complete synchronously?
  }
}

void
nsImapProtocol::ProgressEventFunctionUsingNameWithString(const char* aMsgName,
                                                         const char * aExtraInfo)
{
  if (m_imapMailFolderSink)
  {
    nsString unicodeStr;
    nsresult rv = CopyMUTF7toUTF16(nsDependentCString(aExtraInfo), unicodeStr);
    if (NS_SUCCEEDED(rv))
      m_imapMailFolderSink->ProgressStatusString(this, aMsgName, unicodeStr.get());
  }
}

void
nsImapProtocol::PercentProgressUpdateEvent(const char16_t *message, int64_t currentProgress, int64_t maxProgress)
{
  int64_t nowMS = 0;
  int32_t percent = (100 * currentProgress) / maxProgress;
  if (percent == m_lastPercent)
    return; // hasn't changed, right? So just return. Do we need to clear this anywhere?

  if (percent < 100)  // always need to do 100%
  {
    nowMS = PR_IntervalToMilliseconds(PR_IntervalNow());
    if (nowMS - m_lastProgressTime < 750)
      return;
  }

  m_lastPercent = percent;
  m_lastProgressTime = nowMS;

  // set our max progress on the running URL
  if (m_runningUrl)
  {
    nsCOMPtr<nsIMsgMailNewsUrl> mailnewsUrl(do_QueryInterface(m_runningUrl));
    mailnewsUrl->SetMaxProgress(maxProgress);
  }

  if (m_imapMailFolderSink)
    m_imapMailFolderSink->PercentProgress(this, message, currentProgress, maxProgress);
}

  // imap commands issued by the parser
void
nsImapProtocol::Store(const nsCString &messageList, const char * messageData,
                      bool idsAreUid)
{

  // turn messageList back into key array and then back into a message id list,
  // but use the flag state to handle ranges correctly.
  nsCString messageIdList;
  nsTArray<nsMsgKey> msgKeys;
  if (idsAreUid)
    ParseUidString(messageList.get(), msgKeys);

  int32_t msgCountLeft = msgKeys.Length();
  uint32_t msgsHandled = 0;
  do
  {
    nsCString idString;

    uint32_t msgsToHandle = msgCountLeft;
    if (idsAreUid)
      AllocateImapUidString(msgKeys.Elements() + msgsHandled, msgsToHandle, m_flagState, idString);  // 20 * 200
    else
      idString.Assign(messageList);


    msgsHandled += msgsToHandle;
    msgCountLeft -= msgsToHandle;

    IncrementCommandTagNumber();
    const char *formatString;
    if (idsAreUid)
        formatString = "%s uid store %s %s\015\012";
    else
        formatString = "%s store %s %s\015\012";

    // we might need to close this mailbox after this
    m_closeNeededBeforeSelect = GetDeleteIsMoveToTrash() &&
        (PL_strcasestr(messageData, "\\Deleted"));

    const char *commandTag = GetServerCommandTag();
    int protocolStringSize = PL_strlen(formatString) +
          messageList.Length() + PL_strlen(messageData) +
          PL_strlen(commandTag) + 1;
    char *protocolString = (char *) PR_CALLOC( protocolStringSize );

    if (protocolString)
    {
      PR_snprintf(protocolString, // string to create
                    protocolStringSize, // max size
                    formatString, // format string
                    commandTag, // command tag
                    idString.get(),
                    messageData);

      nsresult rv = SendData(protocolString);
      if (NS_SUCCEEDED(rv))
      {
        m_flagChangeCount++;
        ParseIMAPandCheckForNewMail(protocolString);
        if (GetServerStateParser().LastCommandSuccessful() && CheckNeeded())
          Check();
      }
      PR_Free(protocolString);
    }
    else
      HandleMemoryFailure();
  }
  while (msgCountLeft > 0 && !DeathSignalReceived());

}

void
nsImapProtocol::IssueUserDefinedMsgCommand(const char *command, const char * messageList)
{
  IncrementCommandTagNumber();

  const char *formatString;
  formatString = "%s uid %s %s\015\012";

  const char *commandTag = GetServerCommandTag();
  int protocolStringSize = PL_strlen(formatString) +
        PL_strlen(messageList) + PL_strlen(command) +
        PL_strlen(commandTag) + 1;
  char *protocolString = (char *) PR_CALLOC( protocolStringSize );

  if (protocolString)
  {
    PR_snprintf(protocolString, // string to create
                  protocolStringSize, // max size
                  formatString, // format string
                  commandTag, // command tag
                  command,
                  messageList);

    nsresult rv = SendData(protocolString);
    if (NS_SUCCEEDED(rv))
      ParseIMAPandCheckForNewMail(protocolString);
    PR_Free(protocolString);
  }
  else
    HandleMemoryFailure();
}

void
nsImapProtocol::UidExpunge(const nsCString &messageSet)
{
    IncrementCommandTagNumber();
    nsCString command(GetServerCommandTag());
    command.AppendLiteral(" uid expunge ");
    command.Append(messageSet);
    command.Append(CRLF);
    nsresult rv = SendData(command.get());
    if (NS_SUCCEEDED(rv))
        ParseIMAPandCheckForNewMail();
}

void
nsImapProtocol::Expunge()
{
  uint32_t aclFlags = 0;
  if (GetServerStateParser().ServerHasACLCapability() && m_imapMailFolderSink)
    m_imapMailFolderSink->GetAclFlags(&aclFlags);

  if (aclFlags && !(aclFlags & IMAP_ACL_EXPUNGE_FLAG))
    return;
  ProgressEventFunctionUsingName("imapStatusExpungingMailbox");

  if(gCheckDeletedBeforeExpunge)
  {
    GetServerStateParser().ResetSearchResultSequence();
    Search("SEARCH DELETED", false, false);
    if (GetServerStateParser().LastCommandSuccessful())
    {
      nsImapSearchResultIterator *search = GetServerStateParser().CreateSearchResultIterator();
      nsMsgKey key = search->GetNextMessageNumber();
      delete search;
      if (key == 0)
        return;  //no deleted messages to expunge (bug 235004)
    }
  }

  IncrementCommandTagNumber();
  nsAutoCString command(GetServerCommandTag());
  command.AppendLiteral(" expunge" CRLF);

  nsresult rv = SendData(command.get());
  if (NS_SUCCEEDED(rv))
    ParseIMAPandCheckForNewMail();
}

void
nsImapProtocol::HandleMemoryFailure()
{
    PR_CEnterMonitor(this);
    // **** jefft fix me!!!!!! ******
    // m_imapThreadIsRunning = false;
    // SetConnectionStatus(-1);
    PR_CExitMonitor(this);
}

void nsImapProtocol::HandleCurrentUrlError()
{
  // This is to handle a move/copy failing, especially because the user
  // cancelled the password prompt.
  (void) m_runningUrl->GetImapAction(&m_imapAction);
    if (m_imapAction == nsIImapUrl::nsImapOfflineToOnlineMove || m_imapAction == nsIImapUrl::nsImapAppendMsgFromFile
      || m_imapAction == nsIImapUrl::nsImapAppendDraftFromFile)
  {
    if (m_imapMailFolderSink)
      m_imapMailFolderSink->OnlineCopyCompleted(this, ImapOnlineCopyStateType::kFailedCopy);
  }
}

void nsImapProtocol::StartTLS()
{
    IncrementCommandTagNumber();
    nsCString command(GetServerCommandTag());

    command.AppendLiteral(" STARTTLS" CRLF);

    nsresult rv = SendData(command.get());
    if (NS_SUCCEEDED(rv))
        ParseIMAPandCheckForNewMail();
}

void nsImapProtocol::Capability()
{

    ProgressEventFunctionUsingName("imapStatusCheckCompat");
    IncrementCommandTagNumber();
    nsCString command(GetServerCommandTag());

    command.AppendLiteral(" capability" CRLF);

    nsresult rv = SendData(command.get());
    if (NS_SUCCEEDED(rv))
        ParseIMAPandCheckForNewMail();
}

void nsImapProtocol::ID()
{
  if (!gAppName[0])
    return;
  IncrementCommandTagNumber();
  nsCString command(GetServerCommandTag());
  command.AppendLiteral(" ID (\"name\" \"");
  command.Append(gAppName);
  command.AppendLiteral("\" \"version\" \"");
  command.Append(gAppVersion);
  command.AppendLiteral("\")" CRLF);

  nsresult rv = SendData(command.get());
  if (NS_SUCCEEDED(rv))
    ParseIMAPandCheckForNewMail();
}

void nsImapProtocol::EnableCondStore()
{
  IncrementCommandTagNumber();
  nsCString command(GetServerCommandTag());

  command.AppendLiteral(" ENABLE CONDSTORE" CRLF);

  nsresult rv = SendData(command.get());
  if (NS_SUCCEEDED(rv))
    ParseIMAPandCheckForNewMail();
}

void nsImapProtocol::StartCompressDeflate()
{
  // only issue a compression request if we haven't already
  if (!TestFlag(IMAP_ISSUED_COMPRESS_REQUEST))
  {
    SetFlag(IMAP_ISSUED_COMPRESS_REQUEST);
    IncrementCommandTagNumber();
    nsCString command(GetServerCommandTag());

    command.AppendLiteral(" COMPRESS DEFLATE" CRLF);

    nsresult rv = SendData(command.get());
    if (NS_SUCCEEDED(rv))
    {
      ParseIMAPandCheckForNewMail();
      if (GetServerStateParser().LastCommandSuccessful())
      {
        rv = BeginCompressing();
        if (NS_FAILED(rv))
        {
          Log("CompressDeflate", nullptr, "failed to enable compression");
          // we can't use this connection without compression any more, so die
          ClearFlag(IMAP_CONNECTION_IS_OPEN);
          TellThreadToDie();
          SetConnectionStatus(rv);
          return;
        }
      }
    }
  }
}

nsresult nsImapProtocol::BeginCompressing()
{
  // wrap the streams in compression layers that compress or decompress
  // all traffic.
  RefPtr<nsMsgCompressIStream> new_in = new nsMsgCompressIStream();
  if (!new_in)
    return NS_ERROR_OUT_OF_MEMORY;

  nsresult rv = new_in->InitInputStream(m_inputStream);
  NS_ENSURE_SUCCESS(rv, rv);

  m_inputStream = new_in;

  RefPtr<nsMsgCompressOStream> new_out = new nsMsgCompressOStream();
  if (!new_out)
    return NS_ERROR_OUT_OF_MEMORY;

  rv = new_out->InitOutputStream(m_outputStream);
  NS_ENSURE_SUCCESS(rv, rv);

  m_outputStream = new_out;
  return rv;
}

void nsImapProtocol::Language()
{
  // only issue the language request if we haven't done so already...
  if (!TestFlag(IMAP_ISSUED_LANGUAGE_REQUEST))
  {
    SetFlag(IMAP_ISSUED_LANGUAGE_REQUEST);
    ProgressEventFunctionUsingName("imapStatusCheckCompat");
    IncrementCommandTagNumber();
    nsCString command(GetServerCommandTag());

    // extract the desired language attribute from prefs
    nsresult rv = NS_OK;

    // we need to parse out the first language out of this comma separated list....
    // i.e if we have en,ja we only want to send en to the server.
    if (mAcceptLanguages.get())
    {
      nsAutoCString extractedLanguage;
      LossyCopyUTF16toASCII(mAcceptLanguages, extractedLanguage);
      int32_t pos = extractedLanguage.FindChar(',');
      if (pos > 0) // we have a comma separated list of languages...
        extractedLanguage.SetLength(pos); // truncate everything after the first comma (including the comma)

      if (extractedLanguage.IsEmpty())
        return;

      command.AppendLiteral(" LANGUAGE ");
      command.Append(extractedLanguage);
      command.Append(CRLF);

      rv = SendData(command.get());
      if (NS_SUCCEEDED(rv))
        ParseIMAPandCheckForNewMail(nullptr, true /* ignore bad or no result from the server for this command */);
    }
  }
}

void nsImapProtocol::EscapeUserNamePasswordString(const char *strToEscape, nsCString *resultStr)
{
  if (strToEscape)
  {
    uint32_t i = 0;
    uint32_t escapeStrlen = strlen(strToEscape);
    for (i=0; i<escapeStrlen; i++)
    {
        if (strToEscape[i] == '\\' || strToEscape[i] == '\"')
        {
            resultStr->Append('\\');
        }
        resultStr->Append(strToEscape[i]);
    }
  }
}

void nsImapProtocol::InitPrefAuthMethods(int32_t authMethodPrefValue,
                                         nsIMsgIncomingServer *aServer)
{
    // for m_prefAuthMethods, using the same flags as server capablities.
    switch (authMethodPrefValue)
    {
      case nsMsgAuthMethod::none:
        m_prefAuthMethods = kHasAuthNoneCapability;
        break;
      case nsMsgAuthMethod::old:
        m_prefAuthMethods = kHasAuthOldLoginCapability;
        break;
      case nsMsgAuthMethod::passwordCleartext:
        m_prefAuthMethods = kHasAuthOldLoginCapability |
            kHasAuthLoginCapability | kHasAuthPlainCapability;
        break;
      case nsMsgAuthMethod::passwordEncrypted:
        m_prefAuthMethods = kHasCRAMCapability;
        break;
      case nsMsgAuthMethod::NTLM:
        m_prefAuthMethods = kHasAuthNTLMCapability | kHasAuthMSNCapability;
        break;
      case nsMsgAuthMethod::GSSAPI:
        m_prefAuthMethods = kHasAuthGssApiCapability;
        break;
      case nsMsgAuthMethod::External:
        m_prefAuthMethods = kHasAuthExternalCapability;
        break;
      case nsMsgAuthMethod::secure:
        m_prefAuthMethods = kHasCRAMCapability |
            kHasAuthGssApiCapability |
            kHasAuthNTLMCapability | kHasAuthMSNCapability;
        break;
      default:
        NS_ASSERTION(false, "IMAP: authMethod pref invalid");
        // TODO log to error console
        MOZ_LOG(IMAP, LogLevel::Error,
            ("IMAP: bad pref authMethod = %d\n", authMethodPrefValue));
        // fall to any
        MOZ_FALLTHROUGH;
      case nsMsgAuthMethod::anything:
        m_prefAuthMethods = kHasAuthOldLoginCapability |
            kHasAuthLoginCapability | kHasAuthPlainCapability |
            kHasCRAMCapability | kHasAuthGssApiCapability |
            kHasAuthNTLMCapability | kHasAuthMSNCapability |
            kHasAuthExternalCapability | kHasXOAuth2Capability;
        break;
      case nsMsgAuthMethod::OAuth2:
        m_prefAuthMethods = kHasXOAuth2Capability;
        break;
    }

    if (m_prefAuthMethods & kHasXOAuth2Capability)
      mOAuth2Support = new mozilla::mailnews::OAuth2ThreadHelper(aServer);

    // Disable OAuth2 support if we don't have the prefs installed.
    if (m_prefAuthMethods & kHasXOAuth2Capability &&
        (!mOAuth2Support || !mOAuth2Support->SupportsOAuth2()))
      m_prefAuthMethods &= ~kHasXOAuth2Capability;

    NS_ASSERTION(m_prefAuthMethods != kCapabilityUndefined,
         "IMAP: InitPrefAuthMethods() didn't work");
}

/**
 * Changes m_currentAuthMethod to pick the best remaining one
 * which is allowed by server and prefs and not marked failed.
 * The order of preference and trying of auth methods is encoded here.
 */
nsresult nsImapProtocol::ChooseAuthMethod()
{
  eIMAPCapabilityFlags serverCaps = GetServerStateParser().GetCapabilityFlag();
  eIMAPCapabilityFlags availCaps = serverCaps & m_prefAuthMethods & ~m_failedAuthMethods;

  MOZ_LOG(IMAP, LogLevel::Debug, ("IMAP auth: server caps 0x%" PRIx64
        ", pref 0x%" PRIx64 ", failed 0x%" PRIx64 ", avail caps 0x%" PRIx64,
        serverCaps, m_prefAuthMethods, m_failedAuthMethods, availCaps));
  MOZ_LOG(IMAP, LogLevel::Debug, ("(GSSAPI = 0x%" PRIx64 ", CRAM = 0x%" PRIx64
        ", NTLM = 0x%" PRIx64 ", MSN = 0x%" PRIx64 ", PLAIN = 0x%" PRIx64
        ", LOGIN = 0x%" PRIx64 ", old-style IMAP login = 0x%" PRIx64
        ", auth external IMAP login = 0x%" PRIx64 ", OAUTH2 = 0x%" PRIx64 ")",
        kHasAuthGssApiCapability, kHasCRAMCapability, kHasAuthNTLMCapability,
        kHasAuthMSNCapability, kHasAuthPlainCapability, kHasAuthLoginCapability,
        kHasAuthOldLoginCapability, kHasAuthExternalCapability,
        kHasXOAuth2Capability));

  if (kHasAuthExternalCapability & availCaps)
    m_currentAuthMethod = kHasAuthExternalCapability;
  else if (kHasAuthGssApiCapability & availCaps)
    m_currentAuthMethod = kHasAuthGssApiCapability;
  else if (kHasCRAMCapability & availCaps)
    m_currentAuthMethod = kHasCRAMCapability;
  else if (kHasAuthNTLMCapability & availCaps)
    m_currentAuthMethod = kHasAuthNTLMCapability;
  else if (kHasAuthMSNCapability & availCaps)
    m_currentAuthMethod = kHasAuthMSNCapability;
  else if (kHasXOAuth2Capability & availCaps)
    m_currentAuthMethod = kHasXOAuth2Capability;
  else if (kHasAuthPlainCapability & availCaps)
    m_currentAuthMethod = kHasAuthPlainCapability;
  else if (kHasAuthLoginCapability & availCaps)
    m_currentAuthMethod = kHasAuthLoginCapability;
  else if (kHasAuthOldLoginCapability & availCaps)
    m_currentAuthMethod = kHasAuthOldLoginCapability;
  else
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("no remaining auth method"));
    m_currentAuthMethod = kCapabilityUndefined;
    return NS_ERROR_FAILURE;
  }
  MOZ_LOG(IMAP, LogLevel::Debug, ("trying auth method 0x%" PRIx64, m_currentAuthMethod));
  return NS_OK;
}

void nsImapProtocol::MarkAuthMethodAsFailed(eIMAPCapabilityFlags failedAuthMethod)
{
  MOZ_LOG(IMAP, LogLevel::Debug, ("marking auth method 0x%" PRIx64 " failed", failedAuthMethod));
  m_failedAuthMethods |= failedAuthMethod;
}

/**
 * Start over, trying all auth methods again
 */
void nsImapProtocol::ResetAuthMethods()
{
  MOZ_LOG(IMAP, LogLevel::Debug, ("resetting (failed) auth methods"));
  m_currentAuthMethod = kCapabilityUndefined;
  m_failedAuthMethods = 0;
}

nsresult
nsImapProtocol::SendDataParseIMAPandCheckForNewMail(const char *aData, const char *aCommand)
{
  nsresult rv;
  bool isResend = false;
  while (true)
  {
    // Send authentication string (true: suppress logging the string).
    rv = SendData(aData, true);
    if (NS_FAILED(rv))
      break;
    ParseIMAPandCheckForNewMail(aCommand);
    if (!GetServerStateParser().WaitingForMoreClientInput())
      break;

    // The server is asking for the authentication string again. So we send
    // the same string again although we know that it might be rejected again.
    // We do that to get a firm authentication failure instead of a resend
    // request. That keeps things in order before failing authentication and
    // trying another method if capable.
    if (isResend)
    {
      rv = NS_ERROR_FAILURE;
      break;
    }
    isResend = true;
  }

  return rv;
}

nsresult nsImapProtocol::AuthLogin(const char *userName, const nsString &aPassword, eIMAPCapabilityFlag flag)
{
  ProgressEventFunctionUsingName("imapStatusSendingAuthLogin");
  IncrementCommandTagNumber();

  char * currentCommand=nullptr;
  nsresult rv;
  NS_ConvertUTF16toUTF8 password(aPassword);
  MOZ_LOG(IMAP, LogLevel::Debug, ("IMAP: trying auth method 0x%" PRIx64, m_currentAuthMethod));

  if (flag & kHasAuthExternalCapability)
  {
      char *base64UserName = PL_Base64Encode(userName, strlen(userName), nullptr);
      nsAutoCString command (GetServerCommandTag());
      command.AppendLiteral(" authenticate EXTERNAL " );
      command.Append(base64UserName);
      command.Append(CRLF);
      PR_Free(base64UserName);
      rv = SendData(command.get());
      ParseIMAPandCheckForNewMail();
      nsImapServerResponseParser &parser = GetServerStateParser();
      if (parser.LastCommandSuccessful())
        return NS_OK;
      parser.SetCapabilityFlag(parser.GetCapabilityFlag() & ~kHasAuthExternalCapability);
  }
  else if (flag & kHasCRAMCapability)
  {
    NS_ENSURE_TRUE(m_imapServerSink, NS_ERROR_NULL_POINTER);
    MOZ_LOG(IMAP, LogLevel::Debug, ("MD5 auth"));
    // inform the server that we want to begin a CRAM authentication procedure...
    nsAutoCString command (GetServerCommandTag());
    command.AppendLiteral(" authenticate CRAM-MD5" CRLF);
    rv = SendData(command.get());
    NS_ENSURE_SUCCESS(rv, rv);
    ParseIMAPandCheckForNewMail();
    if (GetServerStateParser().LastCommandSuccessful())
    {
      char *digest = nullptr;
      char *cramDigest = GetServerStateParser().fAuthChallenge;
      char *decodedChallenge = PL_Base64Decode(cramDigest,
                                                strlen(cramDigest), nullptr);
      rv = m_imapServerSink->CramMD5Hash(decodedChallenge, password.get(), &digest);
      PR_Free(decodedChallenge);
      NS_ENSURE_SUCCESS(rv, rv);
      NS_ENSURE_TRUE(digest, NS_ERROR_NULL_POINTER);
      // The encoded digest is the hexadecimal representation of
      // DIGEST_LENGTH characters, so it will be twice that length.
      nsAutoCStringN<2 * DIGEST_LENGTH> encodedDigest;

      for (uint32_t j = 0; j < DIGEST_LENGTH; j++)
      {
        char hexVal[3];
        PR_snprintf (hexVal, 3, "%.2x", 0x0ff & (unsigned short)(digest[j]));
        encodedDigest.Append(hexVal);
      }

      PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%.255s %s", userName, encodedDigest.get());
      char *base64Str = PL_Base64Encode(m_dataOutputBuf, strlen(m_dataOutputBuf), nullptr);
      PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s" CRLF, base64Str);
      PR_Free(base64Str);
      PR_Free(digest);
      rv = SendData(m_dataOutputBuf);
      NS_ENSURE_SUCCESS(rv, rv);
      ParseIMAPandCheckForNewMail(command.get());
    }
  } // if CRAM response was received
  else if (flag & kHasAuthGssApiCapability)
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("MD5 auth"));

    // Only try GSSAPI once - if it fails, its going to be because we don't
    // have valid credentials
    //MarkAuthMethodAsFailed(kHasAuthGssApiCapability);

    // We do step1 first, so we don't try GSSAPI against a server which
    // we can't get credentials for.
    nsAutoCString response;

    nsAutoCString service("imap@");
    service.Append(m_realHostName);
    rv = DoGSSAPIStep1(service.get(), userName, response);
    NS_ENSURE_SUCCESS(rv, rv);

    nsAutoCString command (GetServerCommandTag());
    command.AppendLiteral(" authenticate GSSAPI" CRLF);
    rv = SendData(command.get());
    NS_ENSURE_SUCCESS(rv, rv);

    ParseIMAPandCheckForNewMail("AUTH GSSAPI");
    if (GetServerStateParser().LastCommandSuccessful())
    {
      response += CRLF;
      rv = SendData(response.get());
      NS_ENSURE_SUCCESS(rv, rv);
      ParseIMAPandCheckForNewMail(command.get());
      nsresult gssrv = NS_OK;

      while (GetServerStateParser().LastCommandSuccessful() &&
             NS_SUCCEEDED(gssrv) && gssrv != NS_SUCCESS_AUTH_FINISHED)
      {
        nsCString challengeStr(GetServerStateParser().fAuthChallenge);
        gssrv = DoGSSAPIStep2(challengeStr, response);
        if (NS_SUCCEEDED(gssrv))
        {
          response += CRLF;
          rv = SendData(response.get());
        }
        else
          rv = SendData("*" CRLF);

        NS_ENSURE_SUCCESS(rv, rv);
        ParseIMAPandCheckForNewMail(command.get());
      }
      // TODO: whether it worked or not is shown by LastCommandSuccessful(), not gssrv, right?
    }
  }
  else if (flag & (kHasAuthNTLMCapability | kHasAuthMSNCapability))
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("NTLM auth"));
    nsAutoCString command (GetServerCommandTag());
    command.Append((flag & kHasAuthNTLMCapability) ? " authenticate NTLM" CRLF
                                                   : " authenticate MSN" CRLF);
    rv = SendData(command.get());
    ParseIMAPandCheckForNewMail("AUTH NTLM"); // this just waits for ntlm step 1
    if (GetServerStateParser().LastCommandSuccessful())
    {
      nsAutoCString cmd;
      rv = DoNtlmStep1(nsDependentCString(userName), aPassword, cmd);
      NS_ENSURE_SUCCESS(rv, rv);
      cmd += CRLF;
      rv = SendData(cmd.get());
      NS_ENSURE_SUCCESS(rv, rv);
      ParseIMAPandCheckForNewMail(command.get());
      if (GetServerStateParser().LastCommandSuccessful())
      {
        nsCString challengeStr(GetServerStateParser().fAuthChallenge);
        nsCString response;
        rv = DoNtlmStep2(challengeStr, response);
        NS_ENSURE_SUCCESS(rv, rv);
        response += CRLF;
        rv = SendData(response.get());
        ParseIMAPandCheckForNewMail(command.get());
      }
    }
  }
  else if (flag & kHasAuthPlainCapability)
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("PLAIN auth"));
    PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s authenticate PLAIN" CRLF, GetServerCommandTag());
    rv = SendData(m_dataOutputBuf);
    NS_ENSURE_SUCCESS(rv, rv);
    currentCommand = PL_strdup(m_dataOutputBuf); /* StrAllocCopy(currentCommand, GetOutputBuffer()); */
    ParseIMAPandCheckForNewMail();
    if (GetServerStateParser().LastCommandSuccessful())
    {
      // RFC 4616
      char plain_string[513];
      memset(plain_string, 0, 513);
      PR_snprintf(&plain_string[1], 256, "%.255s", userName);
      uint32_t len = std::min(PL_strlen(userName), 255u) + 2;  // We include two <NUL> characters.
      PR_snprintf(&plain_string[len], 256, "%.255s", password.get());
      len += std::min(password.Length(), 255u);
      char *base64Str = PL_Base64Encode(plain_string, len, nullptr);
      PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s" CRLF, base64Str);
      PR_Free(base64Str);

      rv = SendDataParseIMAPandCheckForNewMail(m_dataOutputBuf, currentCommand);
    } // if the last command succeeded
  } // if auth plain capability
  else if (flag & kHasAuthLoginCapability)
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("LOGIN auth"));
    PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s authenticate LOGIN" CRLF, GetServerCommandTag());
    rv = SendData(m_dataOutputBuf);
    NS_ENSURE_SUCCESS(rv, rv);
    currentCommand = PL_strdup(m_dataOutputBuf);
    ParseIMAPandCheckForNewMail();

    if (GetServerStateParser().LastCommandSuccessful())
    {
      char *base64Str = PL_Base64Encode(userName, std::min(PL_strlen(userName), 255u), nullptr);
      PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s" CRLF, base64Str);
      PR_Free(base64Str);
      rv = SendData(m_dataOutputBuf, true /* suppress logging */);
      if (NS_SUCCEEDED(rv))
      {
        ParseIMAPandCheckForNewMail(currentCommand);
        if (GetServerStateParser().LastCommandSuccessful())
        {
          base64Str = PL_Base64Encode(password.get(), std::min(password.Length(), 255u), nullptr);
          PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE, "%s" CRLF, base64Str);
          PR_Free(base64Str);
          rv = SendData(m_dataOutputBuf, true /* suppress logging */);
          if (NS_SUCCEEDED(rv))
            ParseIMAPandCheckForNewMail(currentCommand);
        } // if last command successful
      } // if last command successful
    } // if last command successful
  } // if has auth login capability
  else if (flag & kHasAuthOldLoginCapability)
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("old-style auth"));
    ProgressEventFunctionUsingName("imapStatusSendingLogin");
    IncrementCommandTagNumber();
    nsCString command (GetServerCommandTag());
    nsAutoCString escapedUserName;
    command.AppendLiteral(" login \"");
    EscapeUserNamePasswordString(userName, &escapedUserName);
    command.Append(escapedUserName);
    command.AppendLiteral("\" \"");

    // if the password contains a \, login will fail
    // turn foo\bar into foo\\bar
    nsAutoCString correctedPassword;
    // We're assuming old style login doesn't want UTF-8
    EscapeUserNamePasswordString(NS_LossyConvertUTF16toASCII(aPassword).get(),
                                 &correctedPassword);
    command.Append(correctedPassword);
    command.AppendLiteral("\"" CRLF);
    rv = SendData(command.get(), true /* suppress logging */);
    NS_ENSURE_SUCCESS(rv, rv);
    ParseIMAPandCheckForNewMail();
  }
  else if (flag & kHasXOAuth2Capability)
  {
    MOZ_LOG(IMAP, LogLevel::Debug, ("XOAUTH2 auth"));

    // Get the XOAuth2 base64 string.
    NS_ASSERTION(mOAuth2Support,
      "What are we doing here without OAuth2 helper?");
    if (!mOAuth2Support)
      return NS_ERROR_UNEXPECTED;
    nsAutoCString base64Str;
    mOAuth2Support->GetXOAuth2String(base64Str);
    mOAuth2Support = nullptr; // Its purpose has been served.
    if (base64Str.IsEmpty())
    {
      MOZ_LOG(IMAP, LogLevel::Debug, ("OAuth2 failed"));
      return NS_ERROR_FAILURE;
    }

    // Send the data on the network.
    nsAutoCString command (GetServerCommandTag());
    command += " AUTHENTICATE XOAUTH2 ";
    command += base64Str;
    command += CRLF;

    rv = SendDataParseIMAPandCheckForNewMail(command.get(), nullptr);
  }
  else if (flag & kHasAuthNoneCapability)
  {
    // TODO What to do? "login <username>" like POP?
    return NS_ERROR_NOT_IMPLEMENTED;
  }
  else
  {
    MOZ_LOG(IMAP, LogLevel::Error, ("flags param has no auth scheme selected"));
    return NS_ERROR_ILLEGAL_VALUE;
  }

  PR_Free(currentCommand);
  NS_ENSURE_SUCCESS(rv, rv);
  return GetServerStateParser().LastCommandSuccessful() ?
           NS_OK : NS_ERROR_FAILURE;
}

void nsImapProtocol::OnLSubFolders()
{
  // **** use to find out whether Drafts, Sent, & Templates folder
  // exists or not even the user didn't subscribe to it
  char *mailboxName = OnCreateServerSourceFolderPathString();
  if (mailboxName)
  {
    ProgressEventFunctionUsingName("imapStatusLookingForMailbox");
    IncrementCommandTagNumber();
    PR_snprintf(m_dataOutputBuf, OUTPUT_BUFFER_SIZE,"%s list \"\" \"%s\"" CRLF, GetServerCommandTag(), mailboxName);
    nsresult rv = SendData(m_dataOutputBuf);
    if (NS_SUCCEEDED(rv))
      ParseIMAPandCheckForNewMail();
    PR_Free(mailboxName);
  }
  else
  {
    HandleMemoryFailure();
  }

}

void nsImapProtocol::OnAppendMsgFromFile()
{
  nsCOMPtr<nsIFile> file;
  nsresult rv = NS_OK;
  rv = m_runningUrl->GetMsgFile(getter_AddRefs(file));
  if (NS_SUCCEEDED(rv) && file)
  {
    char *mailboxName =  OnCreateServerSourceFolderPathString();
    if (mailboxName)
    {
      imapMessageFlagsType flagsToSet = 0;
      uint32_t msgFlags = 0;
      PRTime date = 0;
      nsCString keywords;
      if (m_imapMessageSink)
        m_imapMessageSink->GetCurMoveCopyMessageInfo(m_runningUrl, &date,
                                                     keywords, &msgFlags);

      if (msgFlags & nsMsgMessageFlags::Read)
        flagsToSet |= kImapMsgSeenFlag;
      if (msgFlags & nsMsgMessageFlags::MDNReportSent)
        flagsToSet |= kImapMsgMDNSentFlag;
      // convert msg flag label (0xE000000) to imap flag label (0x0E00)
      if (msgFlags & nsMsgMessageFlags::Labels)
        flagsToSet |= (msgFlags & nsMsgMessageFlags::Labels) >> 16;
      if (msgFlags & nsMsgMessageFlags::Marked)
        flagsToSet |= kImapMsgFlaggedFlag;
      if (msgFlags & nsMsgMessageFlags::Replied)
        flagsToSet |= kImapMsgAnsweredFlag;
      if (msgFlags & nsMsgMessageFlags::Forwarded)
        flagsToSet |= kImapMsgForwardedFlag;

      // If the message copied was a draft, flag it as such
      nsImapAction imapAction;
      rv = m_runningUrl->GetImapAction(&imapAction);