Bug 1199395 - FxOS Stumbling gzip the stumbles to store more data. r=jdm
authorGarvan Keeley <gkeeley@mozilla.com>
Tue, 08 Sep 2015 21:20:56 -0400
changeset 294063 82e6e2bfbf9c93f11756a8aaffaf111451036d42
parent 294062 66b14ac1d547d17894caac7c3732580258faea8a
child 294064 9838257c96b139b41678757f0ec118b74f9f174f
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjdm
bugs1199395
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1199395 - FxOS Stumbling gzip the stumbles to store more data. r=jdm
dom/system/gonk/mozstumbler/UploadStumbleRunnable.cpp
dom/system/gonk/mozstumbler/UploadStumbleRunnable.h
dom/system/gonk/mozstumbler/WriteStumbleOnThread.cpp
--- a/dom/system/gonk/mozstumbler/UploadStumbleRunnable.cpp
+++ b/dom/system/gonk/mozstumbler/UploadStumbleRunnable.cpp
@@ -2,24 +2,25 @@
 /* vim: set ts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "UploadStumbleRunnable.h"
 #include "StumblerLogging.h"
 #include "mozilla/dom/Event.h"
+#include "nsIInputStream.h"
 #include "nsIScriptSecurityManager.h"
 #include "nsIURLFormatter.h"
 #include "nsIVariant.h"
 #include "nsIXMLHttpRequest.h"
 #include "nsNetUtil.h"
 
-UploadStumbleRunnable::UploadStumbleRunnable(const nsACString& aUploadData)
-: mUploadData(aUploadData)
+UploadStumbleRunnable::UploadStumbleRunnable(nsIInputStream* aUploadData)
+: mUploadInputStream(aUploadData)
 {
 }
 
 NS_IMETHODIMP
 UploadStumbleRunnable::Run()
 {
   MOZ_ASSERT(NS_IsMainThread());
   nsresult rv = Upload();
@@ -31,17 +32,17 @@ UploadStumbleRunnable::Run()
 
 nsresult
 UploadStumbleRunnable::Upload()
 {
   nsresult rv;
   nsCOMPtr<nsIWritableVariant> variant = do_CreateInstance("@mozilla.org/variant;1", &rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
-  rv = variant->SetAsACString(mUploadData);
+  rv = variant->SetAsISupports(mUploadInputStream);
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsCOMPtr<nsIXMLHttpRequest> xhr = do_CreateInstance(NS_XMLHTTPREQUEST_CONTRACTID, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsCOMPtr<nsIScriptSecurityManager> secman =
     do_GetService(NS_SCRIPTSECURITYMANAGER_CONTRACTID, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
@@ -58,23 +59,23 @@ UploadStumbleRunnable::Upload()
   NS_ENSURE_SUCCESS(rv, rv);
   nsString url;
   rv = formatter->FormatURLPref(NS_LITERAL_STRING("geo.stumbler.url"), url);
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = xhr->Open(NS_LITERAL_CSTRING("POST"), NS_ConvertUTF16toUTF8(url), false, EmptyString(), EmptyString());
   NS_ENSURE_SUCCESS(rv, rv);
 
-  xhr->SetRequestHeader(NS_LITERAL_CSTRING("Content-Type"), NS_LITERAL_CSTRING("application/json"));
+  xhr->SetRequestHeader(NS_LITERAL_CSTRING("Content-Type"), NS_LITERAL_CSTRING("gzip"));
   xhr->SetMozBackgroundRequest(true);
   // 60s timeout
   xhr->SetTimeout(60 * 1000);
 
   nsCOMPtr<EventTarget> target(do_QueryInterface(xhr));
-  nsRefPtr<nsIDOMEventListener> listener = new UploadEventListener(xhr, mUploadData.Length());
+  nsRefPtr<nsIDOMEventListener> listener = new UploadEventListener(xhr);
 
   const char* const sEventStrings[] = {
     // nsIXMLHttpRequestEventTarget event types
     "abort",
     "error",
     "load",
     "timeout"
   };
@@ -88,33 +89,33 @@ UploadStumbleRunnable::Upload()
   rv = xhr->Send(variant);
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
 }
 
 NS_IMPL_ISUPPORTS(UploadEventListener, nsIDOMEventListener)
 
-UploadEventListener::UploadEventListener(nsIXMLHttpRequest* aXHR, int64_t aFileSize)
-: mXHR(aXHR), mFileSize(aFileSize)
+UploadEventListener::UploadEventListener(nsIXMLHttpRequest* aXHR)
+: mXHR(aXHR)
 {
 }
 
 NS_IMETHODIMP
 UploadEventListener::HandleEvent(nsIDOMEvent* aEvent)
 {
   nsString type;
   if (NS_FAILED(aEvent->GetType(type))) {
     STUMBLER_ERR("Failed to get event type");
     WriteStumbleOnThread::UploadEnded(false);
     return NS_ERROR_FAILURE;
   }
 
   if (type.EqualsLiteral("load")) {
-    STUMBLER_DBG("Got load Event : size %lld", mFileSize);
+    STUMBLER_DBG("Got load Event\n");
   } else if (type.EqualsLiteral("error") && mXHR) {
     STUMBLER_ERR("Upload Error");
   } else {
     STUMBLER_DBG("Receive %s Event", NS_ConvertUTF16toUTF8(type).get());
   }
 
   uint32_t statusCode = 0;
   bool doDelete = false;
--- a/dom/system/gonk/mozstumbler/UploadStumbleRunnable.h
+++ b/dom/system/gonk/mozstumbler/UploadStumbleRunnable.h
@@ -6,41 +6,41 @@
 
 
 #ifndef UPLOADSTUMBLERUNNABLE_H
 #define UPLOADSTUMBLERUNNABLE_H
 
 #include "nsIDOMEventListener.h"
 
 class nsIXMLHttpRequest;
+class nsIInputStream;
 
 /*
  This runnable is managed by WriteStumbleOnThread only, see that class
  for how this is scheduled.
  */
 class UploadStumbleRunnable final : public nsRunnable
 {
 public:
-  explicit UploadStumbleRunnable(const nsACString& aUploadData);
+  explicit UploadStumbleRunnable(nsIInputStream* aUploadInputStream);
 
   NS_IMETHOD Run() override;
 private:
   virtual ~UploadStumbleRunnable() {}
-  const nsCString mUploadData;
+  nsCOMPtr<nsIInputStream> mUploadInputStream;
   nsresult Upload();
 };
 
 
 class UploadEventListener : public nsIDOMEventListener
 {
 public:
-  UploadEventListener(nsIXMLHttpRequest* aXHR, int64_t aFileSize);
+  UploadEventListener(nsIXMLHttpRequest* aXHR);
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSIDOMEVENTLISTENER
 
 protected:
   virtual ~UploadEventListener() {}
   nsCOMPtr<nsIXMLHttpRequest> mXHR;
-  int64_t mFileSize;
 };
 
 #endif
--- a/dom/system/gonk/mozstumbler/WriteStumbleOnThread.cpp
+++ b/dom/system/gonk/mozstumbler/WriteStumbleOnThread.cpp
@@ -16,18 +16,18 @@
 #define MAXFILESIZE_KB (15 * 1024)
 #define ONEDAY_IN_MSEC (24 * 60 * 60 * 1000)
 #define MAX_UPLOAD_ATTEMPTS 20
 
 mozilla::Atomic<bool> WriteStumbleOnThread::sIsUploading(false);
 mozilla::Atomic<bool> WriteStumbleOnThread::sIsAlreadyRunning(false);
 WriteStumbleOnThread::UploadFreqGuard WriteStumbleOnThread::sUploadFreqGuard = {0};
 
-#define FILENAME_INPROGRESS NS_LITERAL_CSTRING("stumbles.json")
-#define FILENAME_COMPLETED NS_LITERAL_CSTRING("stumbles.done.json")
+#define FILENAME_INPROGRESS NS_LITERAL_CSTRING("stumbles.json.gz")
+#define FILENAME_COMPLETED NS_LITERAL_CSTRING("stumbles.done.json.gz")
 #define OUTPUT_DIR NS_LITERAL_CSTRING("mozstumbler")
 
 class DeleteRunnable : public nsRunnable
 {
   public:
     DeleteRunnable() {}
 
     NS_IMETHODIMP
@@ -59,59 +59,50 @@ WriteStumbleOnThread::UploadEnded(bool d
   }
 
   nsCOMPtr<nsIEventTarget> target = do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID);
   MOZ_ASSERT(target);
   nsCOMPtr<nsIRunnable> event = new DeleteRunnable();
   target->Dispatch(event, NS_DISPATCH_NORMAL);
 }
 
-#define DUMP(o, s) \
-  do { \
-    const char* s2 = (s); \
-    uint32_t dummy; \
-    nsresult rv = (o)->Write((s2), strlen(s2), &dummy); \
-    if (NS_WARN_IF(NS_FAILED(rv))) \
-    STUMBLER_ERR("write err"); \
-  } while (0)
-
 void
 WriteStumbleOnThread::WriteJSON(Partition aPart)
 {
   MOZ_ASSERT(!NS_IsMainThread());
 
   nsCOMPtr<nsIFile> tmpFile;
   nsresult rv;
   rv = nsDumpUtils::OpenTempFile(FILENAME_INPROGRESS, getter_AddRefs(tmpFile),
                                  OUTPUT_DIR, nsDumpUtils::CREATE);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     STUMBLER_ERR("Open a file for stumble failed");
     return;
   }
 
-  nsCOMPtr<nsIFileOutputStream> ostream = do_CreateInstance("@mozilla.org/network/file-output-stream;1");
-  rv = ostream->Init(tmpFile, PR_WRONLY | PR_APPEND, 0666, 0);
+  nsRefPtr<nsGZFileWriter> gzWriter = new nsGZFileWriter(nsGZFileWriter::Append);
+  rv = gzWriter->Init(tmpFile);
   if (NS_WARN_IF(NS_FAILED(rv))) {
-    STUMBLER_ERR("Open a file for stumble failed");
+    STUMBLER_ERR("gzWriter init failed");
     return;
   }
 
   /*
    The json format is like below.
    {items:[
    {item},
    {item},
    {item}
    ]}
    */
 
   // Need to add "]}" after the last item
   if (aPart == Partition::End) {
-    DUMP(ostream, "]}");
-    rv = ostream->Close();
+    gzWriter->Write("]}");
+    rv = gzWriter->Finish();
     if (NS_WARN_IF(NS_FAILED(rv))) {
       STUMBLER_ERR("ostream finish failed");
     }
 
     nsCOMPtr<nsIFile> targetFile;
     nsresult rv = nsDumpUtils::OpenTempFile(FILENAME_COMPLETED, getter_AddRefs(targetFile),
                                             OUTPUT_DIR, nsDumpUtils::CREATE);
     nsAutoString targetFilename;
@@ -131,24 +122,24 @@ WriteStumbleOnThread::WriteJSON(Partitio
       STUMBLER_ERR("Rename File failed");
       return;
     }
     return;
   }
 
   // Need to add "{items:[" before the first item
   if (aPart == Partition::Begining) {
-    DUMP(ostream, "{\"items\":[{");
+    gzWriter->Write("{\"items\":[{");
   } else if (aPart == Partition::Middle) {
-    DUMP(ostream, ",{");
+    gzWriter->Write(",{");
   }
-  DUMP(ostream, mDesc.get());
+  gzWriter->Write(mDesc.get());
   //  one item is ended with '}' (e.g. {item})
-  DUMP(ostream, "}");
-  rv = ostream->Close();
+  gzWriter->Write("}");
+  rv = gzWriter->Finish();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     STUMBLER_ERR("ostream finish failed");
   }
 
   // check if it is the end of this file
   int64_t fileSize = 0;
   rv = tmpFile->GetFileSize(&fileSize);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -292,26 +283,17 @@ WriteStumbleOnThread::Upload()
 
   if (fileSize <= 0) {
     sIsUploading = false;
     return;
   }
 
   // prepare json into nsIInputStream
   nsCOMPtr<nsIInputStream> inStream;
-  rv = NS_NewLocalFileInputStream(getter_AddRefs(inStream), tmpFile, -1, -1,
-                                  nsIFileInputStream::DEFER_OPEN);
+  rv = NS_NewLocalFileInputStream(getter_AddRefs(inStream), tmpFile);
   if (NS_FAILED(rv)) {
     sIsUploading = false;
     return;
   }
 
-  nsCString bufStr;
-  rv = NS_ReadInputStreamToString(inStream, bufStr, fileSize);
-
-  if (NS_FAILED(rv)) {
-    sIsUploading = false;
-    return;
-  }
-
-  nsRefPtr<nsIRunnable> uploader = new UploadStumbleRunnable(bufStr);
+  nsRefPtr<nsIRunnable> uploader = new UploadStumbleRunnable(inStream);
   NS_DispatchToMainThread(uploader);
 }