Bug 871846 - ICU usage and infrastructure bits. r=janv
authorReuben Morais <reuben.morais@gmail.com>
Mon, 04 Aug 2014 13:59:57 -0700
changeset 293609 7bfd5e3fc601f5b04a7403060cc1dc516823a122
parent 293608 db900d160b8f2215895d760f92526ae4fb1526e0
child 293610 bab596a35486852df8f0406ff8adeeabcf4571ce
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjanv
bugs871846
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 871846 - ICU usage and infrastructure bits. r=janv
dom/indexedDB/IndexedDatabaseManager.cpp
dom/indexedDB/IndexedDatabaseManager.h
dom/indexedDB/Key.cpp
dom/indexedDB/Key.h
dom/indexedDB/moz.build
--- a/dom/indexedDB/IndexedDatabaseManager.cpp
+++ b/dom/indexedDB/IndexedDatabaseManager.cpp
@@ -47,16 +47,21 @@
 #include "mozilla/dom/IDBKeyRangeBinding.h"
 #include "mozilla/dom/IDBMutableFileBinding.h"
 #include "mozilla/dom/IDBObjectStoreBinding.h"
 #include "mozilla/dom/IDBOpenDBRequestBinding.h"
 #include "mozilla/dom/IDBRequestBinding.h"
 #include "mozilla/dom/IDBTransactionBinding.h"
 #include "mozilla/dom/IDBVersionChangeEventBinding.h"
 
+#ifdef ENABLE_INTL_API
+#include "nsCharSeparatedTokenizer.h"
+#include "unicode/locid.h"
+#endif
+
 #define IDB_STR "indexedDB"
 
 // The two possible values for the data argument when receiving the disk space
 // observer notification.
 #define LOW_DISK_SPACE_DATA_FULL "full"
 #define LOW_DISK_SPACE_DATA_FREE "free"
 
 namespace mozilla {
@@ -389,16 +394,37 @@ IndexedDatabaseManager::Init()
                                 kPrefLoggingDetails);
 #ifdef MOZ_ENABLE_PROFILER_SPS
   Preferences::RegisterCallback(LoggingModePrefChangedCallback,
                                 kPrefLoggingProfiler);
 #endif
   Preferences::RegisterCallbackAndCall(LoggingModePrefChangedCallback,
                                        kPrefLoggingEnabled);
 
+#ifdef ENABLE_INTL_API
+  const nsAdoptingCString& acceptLang =
+    Preferences::GetLocalizedCString("intl.accept_languages");
+
+  // Split values on commas.
+  nsCCharSeparatedTokenizer langTokenizer(acceptLang, ',');
+  while (langTokenizer.hasMoreTokens()) {
+    nsAutoCString lang(langTokenizer.nextToken());
+    icu::Locale locale = icu::Locale::createCanonical(lang.get());
+    if (!locale.isBogus()) {
+      // icu::Locale::getBaseName is always ASCII as per BCP 47
+      mLocale.AssignASCII(locale.getBaseName());
+      break;
+    }
+  }
+
+  if (mLocale.IsEmpty()) {
+    mLocale.AssignLiteral("en-US");
+  }
+#endif
+
   return NS_OK;
 }
 
 void
 IndexedDatabaseManager::Destroy()
 {
   // Setting the closed flag prevents the service from being recreated.
   // Don't set it though if there's no real instance created.
@@ -948,16 +974,28 @@ IndexedDatabaseManager::LoggingModePrefC
     sLoggingMode = logDetails ?
                    Logging_DetailedProfilerMarks :
                    Logging_ConciseProfilerMarks;
   } else {
     sLoggingMode = logDetails ? Logging_Detailed : Logging_Concise;
   }
 }
 
+#ifdef ENABLE_INTL_API
+// static
+const nsCString&
+IndexedDatabaseManager::GetLocale()
+{
+  IndexedDatabaseManager* idbManager = Get();
+  MOZ_ASSERT(idbManager, "IDBManager is not ready!");
+
+  return idbManager->mLocale;
+}
+#endif
+
 NS_IMPL_ADDREF(IndexedDatabaseManager)
 NS_IMPL_RELEASE_WITH_DESTROY(IndexedDatabaseManager, Destroy())
 NS_IMPL_QUERY_INTERFACE(IndexedDatabaseManager, nsIObserver, nsITimerCallback)
 
 NS_IMETHODIMP
 IndexedDatabaseManager::Observe(nsISupports* aSubject, const char* aTopic,
                                 const char16_t* aData)
 {
--- a/dom/indexedDB/IndexedDatabaseManager.h
+++ b/dom/indexedDB/IndexedDatabaseManager.h
@@ -153,16 +153,21 @@ public:
                             int32_t* aRefCnt,
                             int32_t* aDBRefCnt,
                             int32_t* aSliceRefCnt,
                             bool* aResult);
 
   nsresult
   FlushPendingFileDeletions();
 
+#ifdef ENABLE_INTL_API
+  static const nsCString&
+  GetLocale();
+#endif
+
   static mozilla::Mutex&
   FileMutex()
   {
     IndexedDatabaseManager* mgr = Get();
     NS_ASSERTION(mgr, "Must have a manager here!");
 
     return mgr->mFileMutex;
   }
@@ -195,16 +200,20 @@ private:
   nsClassHashtable<nsRefPtrHashKey<FileManager>,
                    nsTArray<int64_t>> mPendingDeleteInfos;
 
   // Lock protecting FileManager.mFileInfos and BlobImplBase.mFileInfos
   // It's s also used to atomically update FileInfo.mRefCnt, FileInfo.mDBRefCnt
   // and FileInfo.mSliceRefCnt
   mozilla::Mutex mFileMutex;
 
+#ifdef ENABLE_INTL_API
+  nsCString mLocale;
+#endif
+
   static bool sIsMainProcess;
   static bool sFullSynchronousMode;
   static PRLogModuleInfo* sLoggingModule;
   static Atomic<LoggingMode> sLoggingMode;
   static mozilla::Atomic<bool> sLowDiskSpaceMode;
 };
 
 } // namespace indexedDB
--- a/dom/indexedDB/Key.cpp
+++ b/dom/indexedDB/Key.cpp
@@ -3,28 +3,33 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 
 #include "Key.h"
 
 #include <algorithm>
+#include "IndexedDatabaseManager.h"
 #include "js/Date.h"
 #include "js/Value.h"
 #include "jsfriendapi.h"
 #include "mozilla/Endian.h"
 #include "mozilla/FloatingPoint.h"
 #include "mozIStorageStatement.h"
 #include "mozIStorageValueArray.h"
 #include "nsAlgorithm.h"
 #include "nsJSUtils.h"
 #include "ReportInternalError.h"
 #include "xpcpublic.h"
 
+#ifdef ENABLE_INTL_API
+#include "unicode/ucol.h"
+#endif
+
 namespace mozilla {
 namespace dom {
 namespace indexedDB {
 
 /*
  Here's how we encode keys:
 
  Basic strategy is the following
@@ -93,16 +98,111 @@ namespace indexedDB {
  end of the encoded buffer.
  
  "foo"         // 0x30 s s s
  1             // 0x10 bf f0
  ["a", "b"]    // 0x80 s 0 0x30 s
  [1, 2]        // 0x60 bf f0 0 0 0 0 0 0 0x10 c0
  [[]]          // 0x80
 */
+#ifdef ENABLE_INTL_API
+nsresult
+Key::ToLocaleBasedKey(Key& aTarget, const nsCString& aLocale) const
+{
+  if (IsUnset()) {
+    aTarget.Unset();
+    return NS_OK;
+  }
+
+  if (IsFloat() || IsDate()) {
+    aTarget.mBuffer = mBuffer;
+    return NS_OK;
+  }
+
+  aTarget.mBuffer.Truncate();
+  aTarget.mBuffer.SetCapacity(mBuffer.Length());
+
+  auto* it = reinterpret_cast<const unsigned char*>(mBuffer.BeginReading());
+  auto* end = reinterpret_cast<const unsigned char*>(mBuffer.EndReading());
+
+  // First we do a pass and see if there are any strings in this key. We only
+  // want to copy/decode when necessary.
+  bool canShareBuffers = true;
+  while (it < end) {
+    auto type = *it % eMaxType;
+    if (type == eTerminator || type == eArray) {
+      it++;
+    } else if (type == eFloat || type == eDate) {
+      it++;
+      it += std::min(sizeof(uint64_t), size_t(end - it));
+    } else {
+      // We have a string!
+      canShareBuffers = false;
+      break;
+    }
+  }
+
+  if (canShareBuffers) {
+    MOZ_ASSERT(it == end);
+    aTarget.mBuffer = mBuffer;
+    return NS_OK;
+  }
+
+  // A string was found, so we need to copy the data we've read so far
+  auto* start = reinterpret_cast<const unsigned char*>(mBuffer.BeginReading());
+  if (it > start) {
+    char* buffer;
+    if (!aTarget.mBuffer.GetMutableData(&buffer, it-start)) {
+      return NS_ERROR_OUT_OF_MEMORY;
+    }
+
+    while (start < it) {
+      *(buffer++) = *(start++);
+    }
+  }
+
+  // Now continue decoding
+  while (it < end) {
+    char* buffer;
+    uint32_t oldLen = aTarget.mBuffer.Length();
+    auto type = *it % eMaxType;
+
+    if (type == eTerminator || type == eArray) {
+      // Copy array TypeID and terminator from raw key
+      if (!aTarget.mBuffer.GetMutableData(&buffer, oldLen + 1)) {
+        return NS_ERROR_OUT_OF_MEMORY;
+      }
+      *(buffer + oldLen) = *(it++);
+    } else if (type == eFloat || type == eDate) {
+      // Copy number from raw key
+      if (!aTarget.mBuffer.GetMutableData(&buffer,
+                                          oldLen + 1 + sizeof(uint64_t))) {
+        return NS_ERROR_OUT_OF_MEMORY;
+      }
+      buffer += oldLen;
+      *(buffer++) = *(it++);
+
+      const size_t byteCount = std::min(sizeof(uint64_t), size_t(end - it));
+      for (size_t count = 0; count < byteCount; count++) {
+        *(buffer++) = (*it++);
+      }
+    } else {
+      // Decode string and reencode
+      uint8_t typeOffset = *it - eString;
+      MOZ_ASSERT((typeOffset % eArray == 0) && (typeOffset / eArray <= 2));
+
+      nsDependentString str;
+      DecodeString(it, end, str);
+      aTarget.EncodeLocaleString(str, typeOffset, aLocale);
+    }
+  }
+  aTarget.TrimBuffer();
+  return NS_OK;
+}
+#endif
 
 nsresult
 Key::EncodeJSValInternal(JSContext* aCx, JS::Handle<JS::Value> aVal,
                          uint8_t aTypeOffset, uint16_t aRecursionDepth)
 {
   static_assert(eMaxType * kMaxArrayCollapse < 256,
                 "Unable to encode jsvals.");
 
@@ -273,47 +373,56 @@ Key::EncodeJSVal(JSContext* aCx,
                  uint8_t aTypeOffset)
 {
   return EncodeJSValInternal(aCx, aVal, aTypeOffset, 0);
 }
 
 void
 Key::EncodeString(const nsAString& aString, uint8_t aTypeOffset)
 {
+  const char16_t* start = aString.BeginReading();
+  const char16_t* end = aString.EndReading();
+  EncodeString(start, end, aTypeOffset);
+}
+
+template <typename T>
+void
+Key::EncodeString(const T* aStart, const T* aEnd, uint8_t aTypeOffset)
+{
   // First measure how long the encoded string will be.
 
   // The +2 is for initial 3 and trailing 0. We'll compensate for multi-byte
   // chars below.
-  uint32_t size = aString.Length() + 2;
+  uint32_t size = (aEnd - aStart) + 2;
   
-  const char16_t* start = aString.BeginReading();
-  const char16_t* end = aString.EndReading();
-  for (const char16_t* iter = start; iter < end; ++iter) {
+  const T* start = aStart;
+  const T* end = aEnd;
+  for (const T* iter = start; iter < end; ++iter) {
     if (*iter > ONE_BYTE_LIMIT) {
-      size += *iter > TWO_BYTE_LIMIT ? 2 : 1;
+      size += char16_t(*iter) > TWO_BYTE_LIMIT ? 2 : 1;
     }
   }
 
   // Allocate memory for the new size
   uint32_t oldLen = mBuffer.Length();
   char* buffer;
   if (!mBuffer.GetMutableData(&buffer, oldLen + size)) {
     return;
   }
   buffer += oldLen;
 
   // Write type marker
   *(buffer++) = eString + aTypeOffset;
 
   // Encode string
-  for (const char16_t* iter = start; iter < end; ++iter) {
+  for (const T* iter = start; iter < end; ++iter) {
     if (*iter <= ONE_BYTE_LIMIT) {
       *(buffer++) = *iter + ONE_BYTE_ADJUST;
     }
-    else if (*iter <= TWO_BYTE_LIMIT) {
+    else if (char16_t(*iter) <= TWO_BYTE_LIMIT) {
       char16_t c = char16_t(*iter) + TWO_BYTE_ADJUST + 0x8000;
       *(buffer++) = (char)(c >> 8);
       *(buffer++) = (char)(c & 0xFF);
     }
     else {
       uint32_t c = (uint32_t(*iter) << THREE_BYTE_SHIFT) | 0x00C00000;
       *(buffer++) = (char)(c >> 16);
       *(buffer++) = (char)(c >> 8);
@@ -322,16 +431,57 @@ Key::EncodeString(const nsAString& aStri
   }
 
   // Write end marker
   *(buffer++) = eTerminator;
   
   NS_ASSERTION(buffer == mBuffer.EndReading(), "Wrote wrong number of bytes");
 }
 
+#ifdef ENABLE_INTL_API
+nsresult
+Key::EncodeLocaleString(const nsDependentString& aString, uint8_t aTypeOffset,
+                        const nsCString& aLocale)
+{
+  const int length = aString.Length();
+  if (length == 0) {
+    return NS_OK;
+  }
+  const UChar* ustr = reinterpret_cast<const UChar*>(aString.BeginReading());
+
+  UErrorCode uerror = U_ZERO_ERROR;
+  UCollator* collator = ucol_open(aLocale.get(), &uerror);
+  if (NS_WARN_IF(U_FAILURE(uerror))) {
+    return NS_ERROR_FAILURE;
+  }
+  MOZ_ASSERT(collator);
+
+  nsAutoTArray<uint8_t, 128> keyBuffer;
+  int32_t sortKeyLength = ucol_getSortKey(collator, ustr, length,
+                                          keyBuffer.Elements(),
+                                          keyBuffer.Length());
+  if (sortKeyLength > (int32_t)keyBuffer.Length()) {
+    keyBuffer.SetLength(sortKeyLength);
+    sortKeyLength = ucol_getSortKey(collator, ustr, length,
+                                    keyBuffer.Elements(),
+                                    sortKeyLength);
+  }
+
+  ucol_close(collator);
+  if (NS_WARN_IF(sortKeyLength == 0)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  EncodeString(keyBuffer.Elements(),
+               keyBuffer.Elements()+sortKeyLength,
+               aTypeOffset);
+  return NS_OK;
+}
+#endif
+
 // static
 nsresult
 Key::DecodeJSVal(const unsigned char*& aPos,
                  const unsigned char* aEnd,
                  JSContext* aCx,
                  JS::MutableHandle<JS::Value> aVal)
 {
   return DecodeJSValInternal(aPos, aEnd, aCx, 0, aVal, 0);
--- a/dom/indexedDB/Key.h
+++ b/dom/indexedDB/Key.h
@@ -202,16 +202,21 @@ public:
   ToJSVal(JSContext* aCx, JS::MutableHandle<JS::Value> aVal) const;
 
   nsresult
   ToJSVal(JSContext* aCx, JS::Heap<JS::Value>& aVal) const;
 
   nsresult
   AppendItem(JSContext* aCx, bool aFirstOfArray, JS::Handle<JS::Value> aVal);
 
+#ifdef ENABLE_INTL_API
+  nsresult
+  ToLocaleBasedKey(Key& aTarget, const nsCString& aLocale) const;
+#endif
+
   void
   FinishArray()
   {
     TrimBuffer();
   }
 
   const nsCString&
   GetBuffer() const
@@ -273,16 +278,26 @@ private:
 
   // Encoding functions. These append the encoded value to the end of mBuffer
   nsresult
   EncodeJSVal(JSContext* aCx, JS::Handle<JS::Value> aVal, uint8_t aTypeOffset);
 
   void
   EncodeString(const nsAString& aString, uint8_t aTypeOffset);
 
+  template <typename T>
+  void
+  EncodeString(const T* aStart, const T* aEnd, uint8_t aTypeOffset);
+
+#ifdef ENABLE_INTL_API
+  nsresult
+  EncodeLocaleString(const nsDependentString& aString, uint8_t aTypeOffset,
+                     const nsCString& aLocale);
+#endif
+
   void
   EncodeNumber(double aFloat, uint8_t aType);
 
   // Decoding functions. aPos points into mBuffer and is adjusted to point
   // past the consumed value.
   static nsresult
   DecodeJSVal(const unsigned char*& aPos,
               const unsigned char* aEnd,
--- a/dom/indexedDB/moz.build
+++ b/dom/indexedDB/moz.build
@@ -55,24 +55,24 @@ UNIFIED_SOURCES += [
     'IDBIndex.cpp',
     'IDBKeyRange.cpp',
     'IDBMutableFile.cpp',
     'IDBObjectStore.cpp',
     'IDBRequest.cpp',
     'IDBTransaction.cpp',
     'IDBWrapperCache.cpp',
     'IndexedDatabaseManager.cpp',
-    'Key.cpp',
     'KeyPath.cpp',
     'PermissionRequestBase.cpp',
     'ReportInternalError.cpp',
 ]
 
 SOURCES += [
     'ActorsParent.cpp', # This file is huge.
+    'Key.cpp', # We disable a warning on this file only
 ]
 
 IPDL_SOURCES += [
     'PBackgroundIDBCursor.ipdl',
     'PBackgroundIDBDatabase.ipdl',
     'PBackgroundIDBDatabaseFile.ipdl',
     'PBackgroundIDBFactory.ipdl',
     'PBackgroundIDBFactoryRequest.ipdl',
@@ -82,17 +82,25 @@ IPDL_SOURCES += [
     'PBackgroundIDBVersionChangeTransaction.ipdl',
     'PIndexedDBPermissionRequest.ipdl',
 ]
 
 include('/ipc/chromium/chromium-config.mozbuild')
 
 FINAL_LIBRARY = 'xul'
 
+if CONFIG['GNU_CC']:
+    # Suppress gcc warning about a comparison being always false due to the
+    # range of the data type
+    SOURCES['Key.cpp'].flags += ['-Wno-error=type-limits']
+
 LOCAL_INCLUDES += [
     '/db/sqlite3/src',
     '/dom/base',
     '/dom/storage',
     '/dom/workers',
     '/ipc/glue',
     '/xpcom/build',
     '/xpcom/threads',
 ]
+
+if CONFIG['ENABLE_INTL_API']:
+    CXXFLAGS += CONFIG['MOZ_ICU_CFLAGS']