Fix bug 469767 - Very slow etags parsing. r=dbo
authorPhilipp Kewisch <mozilla@kewis.ch>
Fri, 30 Jan 2009 15:04:10 +0100
changeset 1799 50e56618c6ec425aa344a9901f807c9e59b19ba9
parent 1798 1360769469ea343ad94b751ddde24f9dfb131627
child 1800 5edf99040107ab20b5858d2106ec9b613aa341ca
push idunknown
push userunknown
push dateunknown
reviewersdbo
bugs469767
Fix bug 469767 - Very slow etags parsing. r=dbo
calendar/installer/windows/packages-static
calendar/providers/caldav/Makefile.in
calendar/providers/caldav/calDavCalendar.js
calendar/providers/caldav/calDavCalendarModule.js
calendar/providers/caldav/calDavRequestHandlers.js
--- a/calendar/installer/windows/packages-static
+++ b/calendar/installer/windows/packages-static
@@ -222,16 +222,17 @@ bin\calendar-js\calAlarmMonitor.js
 bin\calendar-js\calAlarmService.js
 bin\calendar-js\calAttachment.js
 bin\calendar-js\calAttendee.js
 bin\calendar-js\calCachedCalendar.js
 bin\calendar-js\calCalendarManager.js
 bin\calendar-js\calCalendarSearchService.js
 bin\calendar-js\calDateTimeFormatter.js
 bin\calendar-js\calDavCalendar.js
+bin\calendar-js\calDavRequestHandlers.js
 bin\calendar-js\calEvent.js
 bin\calendar-js\calFilter.js
 bin\calendar-js\calFreeBusyService.js
 bin\calendar-js\calHtmlExport.js
 bin\calendar-js\calICSCalendar.js
 bin\calendar-js\calIcsImportExport.js
 bin\calendar-js\calIcsParser.js
 bin\calendar-js\calIcsSerializer.js
--- a/calendar/providers/caldav/Makefile.in
+++ b/calendar/providers/caldav/Makefile.in
@@ -43,17 +43,17 @@ VPATH		= @srcdir@
 
 include $(DEPTH)/config/autoconf.mk
 
 MODULE = caldav
 
 DIRS = public
 
 EXTRA_COMPONENTS = calDavCalendarModule.js
-EXTRA_SCRIPTS = calDavCalendar.js
+EXTRA_SCRIPTS = calDavCalendar.js calDavRequestHandlers.js
 
 libs:: $(EXTRA_SCRIPTS)
 	if test ! -d $(FINAL_TARGET)/calendar-js; then $(NSINSTALL) -D $(FINAL_TARGET)/calendar-js; fi
 	$(INSTALL) $^ $(FINAL_TARGET)/calendar-js
 
 # The install target must use SYSINSTALL, which is NSINSTALL in copy mode.
 install:: $(EXTRA_SCRIPTS)
 	$(SYSINSTALL) $(IFLAGS1) $^ $(DESTDIR)$(mozappdir)/calendar-js
--- a/calendar/providers/caldav/calDavCalendar.js
+++ b/calendar/providers/caldav/calDavCalendar.js
@@ -926,210 +926,42 @@ calDavCalendar.prototype = {
 
     getUpdatedItems: function caldav_getUpdatedItems(aUri, aChangeLogListener) {
         if (this.mDisabled) {
             // check if maybe our calendar has become available
             this.checkDavResourceType(aChangeLogListener);
             return;
         }
 
-        var itemsReported = {};
-        var itemsNeedFetching = [];
-        unhandledErrors = 0;
-
-        var C = new Namespace("C", "urn:ietf:params:xml:ns:caldav");
-        var D = new Namespace("D", "DAV:");
+        let C = new Namespace("C", "urn:ietf:params:xml:ns:caldav");
+        let D = new Namespace("D", "DAV:");
         default xml namespace = C;
 
-        var queryXml = <D:propfind xmlns:D="DAV:">
+        let queryXml = <D:propfind xmlns:D="DAV:">
                         <D:prop>
                             <D:getcontenttype/>
                             <D:getetag/>
                         </D:prop>
                        </D:propfind>;
 
-        var queryString = xmlHeader + queryXml.toXMLString();
-
-        var multigetQueryXml =
-          <calendar-multiget xmlns:D={D} xmlns={C}>
-            <D:prop>
-              <D:getetag/>
-              <calendar-data/>
-            </D:prop>
-          </calendar-multiget>;
-
-        var thisCalendar = this;
-
-        var etagListener = {};
-
-        etagListener.onStreamComplete =
-            function getUpdatedItems_getetag_onStreamComplete(aLoader, aContext, aStatus,
-                                                              aResultLength, aResult) {
-            let responseStatus;
-            let request = aLoader.request.QueryInterface(Components.interfaces.nsIHttpChannel);
-            try {
-                LOG("CalDAV: Status " + request.responseStatus +
-                    " on getetag for calendar " + thisCalendar.name);
-                responseStatus = request.responseStatus;
-            } catch (ex) {
-                LOG("CalDAV: Error without status on getetag for calendar " +
-                    thisCalendar.name);
-                responseStatus = "none";
-            }
-
-            if (responseStatus == 207) {
-                // We only need to parse 207's, anything else is probably a
-                // server error (i.e 50x).
-                let str = cal.convertByteArray(aResult, aResultLength);
-                if (!str) {
-                    LOG("CAlDAV: Failed to parse getetag PROPFIND");
-                } else if (thisCalendar.verboseLogging()) {
-                    LOG("CalDAV: recv: " + str);
-                }
-
-                let multistatus = cal.safeNewXML(str);
-                for each (let response in multistatus.*::response) {
-                    var etag = response..D::["getetag"];
-                    if (etag.length() == 0) {
-                        continue;
-                    }
-                    var contenttype = null;
-                    contenttype = response..D::["getcontenttype"];
-                    // workaround for a Scalix bug which causes incorrect
-                    // contenttype to be returned. Remove when that's fixed
-                    if (contenttype == "message/rfc822") {
-                        contenttype = "text/calendar";
-                    }
-                    // end Scalix workaround
-                    if (contenttype.toString().substr(0, 13) != "text/calendar") {
-                        continue;
-                    }
-                    var href = response..D::["href"];
-                    var resourcePath = thisCalendar.ensurePath(href);
-                    itemsReported[resourcePath.toString()] = true;
-
-                    var itemuid = thisCalendar.mHrefIndex[resourcePath];
-                    if (!itemuid || etag != thisCalendar.mItemInfoCache[itemuid].etag) {
-                        itemsNeedFetching.push(resourcePath);
-                    }
-                }
-            } else if (Math.floor(responseStatus / 100) == 4) {
-                // A 4xx error probably means the server doesn't support this
-                // type of query. Disable it for this session. This doesn't
-                // really match the spec (which requires a 207), but it works
-                // around bugs in Google and eGroupware 1.6.
-                LOG("CalDAV: Server doesn't support " + itemType + "s");
-                var itemTypeIndex = thisCalendar.supportedItemTypes.indexOf(itemType);
-                if (itemTypeIndex > -1) {
-                    thisCalendar.supportedItemTypes.splice(itemTypeIndex, 1);
-                }
-            } else {
-                unhandledErrors++;
-            }
-
-            var needsRefresh = false;
-
-            if (unhandledErrors) {
-                LOG("CalDAV: Error fetching item etags");
-                thisCalendar.reportDavError(Components.interfaces.calIErrors.DAV_REPORT_ERROR);
-                if (thisCalendar.isCached && aChangeLogListener) {
-                    aChangeLogListener.onResult({ status: Components.results.NS_ERROR_FAILURE },
-                                                Components.results.NS_ERROR_FAILURE);
-                }
-                return;
-            }
-            if (thisCalendar.isCached) {
-                thisCalendar.superCalendar.startBatch();
-            }
-            try {
-                for (var path in thisCalendar.mHrefIndex) {
-                    if (path in itemsReported || path.indexOf(aUri.path) != 0) {
-                        // If the item is also on the server, check the next.
-                        continue;
-                    }
-                    // if an item has been deleted from the server, delete it
-                    // here too. Since the target calendar's operations are
-                    // synchronous, we can safely set variables from this
-                    // function (i.e needsRefresh)
-                    var getItemListener = {
-                        onGetResult: function caldav_gUIs_oGR(aCalendar,
-                                                              aStatus,
-                                                              aItemType,
-                                                              aDetail,
-                                                              aCount,
-                                                              aItems) {
-                            var itemToDelete = aItems[0];
-                            var wasInboxItem = thisCalendar.mItemInfoCache[itemToDelete.id].isInboxItem;
-                            if ((wasInboxItem && thisCalendar.isInbox(aUri.spec)) ||
-                                (wasInboxItem === false && !thisCalendar.isInbox(aUri.spec))) {
-                                delete thisCalendar.mItemInfoCache[itemToDelete.id];
-                                thisCalendar.mTargetCalendar.deleteItem(itemToDelete,
-                                                                        getItemListener);
-                            }
-                            delete thisCalendar.mHrefIndex[path];
-                            needsRefresh = true;
-                        },
-                        onOperationComplete: function caldav_gUIs_oOC(aCalendar,
-                                                                      aStatus,
-                                                                      aOperationType,
-                                                                      aId,
-                                                                      aDetail) {}
-                    };
-                    thisCalendar.mTargetCalendar.getItem(thisCalendar.mHrefIndex[path],
-                                                         getItemListener);
-                }
-            } finally {
-                if (thisCalendar.isCached) {
-                    thisCalendar.superCalendar.endBatch();
-                }
-            }
-
-            // Avoid sending empty multiget requests
-            // update views if something has been deleted server-side
-            if (!itemsNeedFetching.length) {
-                if (thisCalendar.isCached && aChangeLogListener) {
-                    aChangeLogListener.onResult({ status: Components.results.NS_OK },
-                                                Components.results.NS_OK);
-                }
-                if (needsRefresh) {
-                    thisCalendar.mObservers.notify("onLoad", [thisCalendar]);
-                }
-                // but do poll the inbox;
-                if (thisCalendar.hasScheduling &&
-                    !thisCalendar.isInbox(aUri.spec)) {
-                    thisCalendar.pollInbox();
-                }
-                return;
-            }
-
-            while (itemsNeedFetching.length > 0) {
-                var locpath = itemsNeedFetching.pop().toString();
-                multigetQueryXml.D::prop += <D:href xmlns:D={D}>{locpath}</D:href>;
-            }
-
-            var multigetQueryString = xmlHeader +
-                                      multigetQueryXml.toXMLString();
-            thisCalendar.getCalendarData(aUri,
-                                         multigetQueryString,
-                                         null,
-                                         null,
-                                         aChangeLogListener);
-        };
-
+        let queryString = xmlHeader + queryXml.toXMLString();
         if (this.verboseLogging()) {
             LOG("CalDAV: send(" + aUri.spec + "): " + queryString);
         }
 
         let httpchannel = cal.prepHttpChannel(aUri,
                                               queryString,
                                               "text/xml; charset=utf-8",
                                               this);
         httpchannel.requestMethod = "PROPFIND";
         httpchannel.setRequestHeader("Depth", "1", false);
-        cal.sendHttpRequest(cal.createStreamLoader(), httpchannel, etagListener);
+
+        // Submit the request
+        let streamListener = new etagsHandler(this, aUri, aChangeLogListener);
+        httpchannel.asyncOpen(streamListener, httpchannel);
     },
 
     getCalendarData: function caldav_getCalendarData(aUri, aQuery, aItem, aListener, aChangeLogListener) {
         this.ensureTargetCalendar();
 
         var thisCalendar = this;
         var caldataListener = {};
         var C = new Namespace("C", "urn:ietf:params:xml:ns:caldav");
--- a/calendar/providers/caldav/calDavCalendarModule.js
+++ b/calendar/providers/caldav/calDavCalendarModule.js
@@ -69,17 +69,17 @@ var calDavCalendarModule = {
 
     mUtilsLoaded: false,
     loadUtils: function cDCM_loadUtils() {
         if (this.mUtilsLoaded) {
             return;
         }
 
         Components.utils.import("resource://calendar/modules/calUtils.jsm");
-        cal.loadScripts(["calUtils.js", "calDavCalendar.js" ], this.__parent__);
+        cal.loadScripts(["calUtils.js", "calDavCalendar.js", "calDavRequestHandlers.js" ], this.__parent__);
 
         this.mUtilsLoaded = true;
     },
 
     registerSelf: function (compMgr, fileSpec, location, type) {
         compMgr = compMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
         compMgr.registerFactoryLocation(this.mCID,
                                         "Calendar CalDAV back-end",
new file mode 100644
--- /dev/null
+++ b/calendar/providers/caldav/calDavRequestHandlers.js
@@ -0,0 +1,326 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Sun Microsystems code.
+ *
+ * The Initial Developer of the Original Code is
+ *   Philipp Kewisch <mozilla@kewis.ch>
+ * Portions created by the Initial Developer are Copyright (C) 2009
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+
+Components.utils.import("resource://calendar/modules/calUtils.jsm");
+
+/**
+ * This is a handler for the etag request in calDavCalendar.js' getUpdatedItem.
+ * It uses the SAX parser to incrementally parse the items and compose the
+ * resulting multiget.
+ *
+ * @param aCalendar             The (unwrapped) calendar this request belongs to
+ * @param aBaseUri              The URI requested (i.e inbox or collection)
+ * @param aChangeLogListener    (optional) for cached calendars, the listener to
+ *                                notify.
+ */
+function etagsHandler(aCalendar, aBaseUri, aChangeLogListener) {
+    this.calendar = aCalendar;
+    this.baseUri = aBaseUri;
+    this.changelogListener = aChangeLogListener;
+    this._reader = Components.classes["@mozilla.org/saxparser/xmlreader;1"]
+                             .createInstance(Components.interfaces.nsISAXXMLReader);
+    this._reader.contentHandler = this;
+    this._reader.errorHandler = this;
+    this._reader.parseAsync(null);
+
+    this.itemsReported = {};
+    this.itemsNeedFetching = [];
+}
+
+etagsHandler.prototype = {
+    skipIndex: -1,
+    currentResponse: null,
+    tag: null,
+    calendar: null,
+    baseUri: null,
+    changelogListener: null,
+    logXML: "",
+
+    itemsReported: null,
+    itemsNeedFetching: null,
+
+    QueryInterface: function QueryInterface(aIID) {
+        return doQueryInterface(this,
+                                etagsHandler.prototype,
+                                aIID,
+                                [Components.interfaces.nsISAXContentHandler,
+                                 Components.interfaces.nsISAXErrorHandler,
+                                 Components.interfaces.nsIRequestObserver,
+                                 Components.interfaces.nsIStreamListener]);
+    },
+
+    /**
+     * @see nsIStreamListener
+     */
+    onStartRequest: function eSL_onStartRequest(request, context) {
+        let httpchannel = request.QueryInterface(Components.interfaces.nsIHttpChannel);
+
+        let responseStatus;
+        try {
+            responseStatus = httpchannel.responseStatus
+        } catch (ex) {
+            cal.WARN("CalDAV: No response status getting etags for calendar " + this.calendar.name);
+        }
+
+        if (responseStatus == 207) {
+            // We only need to parse 207's, anything else is probably a
+            // server error (i.e 50x).
+            httpchannel.contentType = "application/xml";
+            this._reader.onStartRequest(request, context);
+        } else {
+            cal.LOG("CalDAV: Error fetching item etags");
+            this.calendar.reportDavError(Components.interfaces.calIErrors.DAV_REPORT_ERROR);
+            if (this.calendar.isCached && this.changelogListener) {
+                this.changelogListener.onResult({ status: Components.results.NS_ERROR_FAILURE },
+                                                Components.results.NS_ERROR_FAILURE);
+            }
+            this._reader = null;
+        }
+    },
+
+    onStopRequest: function eSL_onStopRequest(request, context, statusCode) {
+        if (this.calendar.verboseLogging()) {
+            cal.LOG("CalDAV: recv: " + this.logXML);
+        }
+        if (!this._reader) {
+            // No reader means there was a request error
+            return;
+        }
+        try {
+            this._reader.onStopRequest(request, context, statusCode);
+        } finally {
+            this._reader = null;
+        }
+
+        // Now that we are done, check which items need fetching.
+        if (this.calendar.isCached) {
+            this.calendar.superCalendar.startBatch();
+        }
+
+        let needsRefresh = false;
+        try {
+            for (let path in this.calendar.mHrefIndex) {
+                if (path in this.itemsReported ||
+                    path.substr(0, this.baseUri.length) == this.baseUri) {
+                    // If the item is also on the server, check the next.
+                    continue;
+                }
+                // If an item has been deleted from the server, delete it here too.
+                // Since the target calendar's operations are synchronous, we can
+                // safely set variables from this function.
+                let foundItem;
+                let getItemListener = {
+                    onGetResult: function etags_getItem_onResult(aCalendar,
+                                                                 aStatus,
+                                                                 aItemType,
+                                                                 aDetail,
+                                                                 aCount,
+                                                                 aItems) {
+                    
+                        foundItem = aItems[0];
+                    },
+                    onOperationComplete: function etags_getItem_onOperationComplete() {}
+                };
+
+                this.calendar.mTargetCalendar.getItem(this.calendar.mHrefIndex[path],
+                                                      getItemListener);
+                if (foundItem) {
+                    let wasInboxItem = this.calendar.mItemInfoCache[foundItem.id].isInboxItem;
+                    if ((wasInboxItem && this.calendar.isInbox(this.baseUri.spec)) ||
+                        (wasInboxItem === false && !this.calendar.isInbox(this.baseUri.spec))) {
+                        delete this.calendar.mHrefIndex[path];
+                        needsRefresh = true;
+                    }
+                }
+            }
+        } finally {
+            if (this.calendar.isCached) {
+                this.calendar.superCalendar.endBatch();
+            }
+        }
+
+        // Avoid sending empty multiget requests update views if something has
+        // been deleted server-side.
+        if (!this.itemsNeedFetching.length) {
+            if (this.calendar.isCached && this.changelogListener) {
+                this.changelogListener.onResult({ status: Components.results.NS_OK },
+                                                Components.results.NS_OK);
+            }
+
+            if (needsRefresh) {
+                this.calendar.mObservers.notify("onLoad", [this.calendar]);
+            }
+
+            // but do poll the inbox
+            if (this.calendar.hasScheduling &&
+                this.calendar.isInbox(this.baseUri.spec)) {
+                this.calendar.pollInbox();
+            }
+        } else {
+            let C = new Namespace("C", "urn:ietf:params:xml:ns:caldav");
+            let D = new Namespace("D", "DAV:");
+            let multigetQueryXml =
+              <calendar-multiget xmlns:D={D} xmlns={C}>
+                <D:prop>
+                  <D:getetag/>
+                  <calendar-data/>
+                </D:prop>
+              </calendar-multiget>;
+
+            while (this.itemsNeedFetching.length) {
+                let locpath = this.itemsNeedFetching.pop();
+                multigetQueryXml.D::prop += <D:href xmlns:D={D}>{locpath}</D:href>;
+            }
+
+            let multigetQueryString = xmlHeader + multigetQueryXml.toXMLString();
+            this.calendar.getCalendarData(this.baseUri,
+                                          multigetQueryString,
+                                          null,
+                                          null,
+                                          this.changelogListener);
+        }
+    },
+
+    onDataAvailable: function eSL_onDataAvailable(request, context, inputStream, offset, count) {
+        if (this._reader) {
+            // No reader means request error
+            this._reader.onDataAvailable(request, context, inputStream, offset, count);
+        }
+    },
+
+
+    /**
+     * @see nsISAXErrorHandler
+     */
+    fatalError: function eH_fatalError() {
+        LOG("CalDAV: Fatal Error parsing etags for " + this.calendar.name);
+    },
+
+
+    /**
+     * @see nsISAXContentHandler
+     */
+    characters: function eH_characters(aValue) {
+        if (this.calendar.verboseLogging()) {
+            this.logXML += aValue;
+        }
+        this.currentResponse[this.tag] += aValue;
+    },
+
+    startDocument: function eH_startDocument() {
+        this.hrefMap = {};
+        this.currentResponse = {};
+        this.tag = null
+    },
+
+    endDocument: function eH_endDocument() { },
+
+    startElement: function eH_startElement(aUri, aLocalName, aQName, aAttributes) {
+        switch (aLocalName) {
+            case "response":
+                this.currentResponse = {};
+                this.tag = null
+                break;
+            case "href":
+            case "getetag":
+            case "status":
+            case "getcontenttype":
+                this.tag = aLocalName;
+                this.currentResponse[aLocalName] = "";
+                break;
+        }
+        if (this.calendar.verboseLogging()) {
+            this.logXML += "<" + aQName + ">";
+        }
+
+    },
+
+    endElement: function eH_endElement(aUri, aLocalName, aQName) {
+        switch (aLocalName) {
+            case "response":
+                this.tag = null;
+                let r = this.currentResponse;
+                if (r.status.indexOf(" 200") > 0 &&
+                    r.getetag && r.getetag.length &&
+                    r.href && r.href.length &&
+                    !r.getcontenttype || r.getcontenttype.length) {
+                    let href;
+                    if (r.getcontenttype == "message/rfc822") {
+                        // workaround for a Scalix bug which causes incorrect
+                        // contenttype to be returned.
+                        r.getcontenttype = "text/calendar";
+                    }
+
+                    if (r.getcontenttype.substr(0,13) == "text/calendar") {
+                        // Only handle calendar items
+
+                        if (this.skipIndex < 0) {
+                            href = this.calendar.ensurePath(r.href);
+                            this.skipIndex = r.href.indexOf(href);
+                        } else {
+                            href = r.href.substr(this.skipIndex);
+                        }
+                        
+                        if (href && href.length) {
+                            this.itemsReported[href] = r.getetag;
+
+                            let itemUid = this.calendar.mHrefIndex[href];
+                            if (!itemUid ||
+                                r.getetag != this.calendar.mItemInfoCache[itemUid].etag) {
+                                this.itemsNeedFetching.push(href);
+                            }
+                        }
+                    }
+                }
+                break;
+            case "href":
+            case "getetag":
+            case "status":
+            case "getcontenttype":
+                this.tag = null;
+                break;
+        }
+        if (this.calendar.verboseLogging()) {
+            this.logXML += "</" + aQName + ">";
+        }
+    },
+    
+    startPrefixMapping: function eH_startPrefixMapping(aPrefix, aUri) { },
+    endPrefixMapping: function eH_endPrefixMapping(aPrefix) { },
+    ignorableWhitespace: function eH_ignorableWhitespace(aWhiteSpace) { },
+    processingInstruction: function eH_processingInstruction(aTarget, aData) { }
+};