Bug 1271500 - Binary Key Support. r=janv
authorBevis Tseng <btseng@mozilla.com>
Fri, 12 Aug 2016 15:03:59 +0800
changeset 309525 23ba2c37b8ef5fe6eee5ee16ea48a3f60123fea1
parent 309524 a9213b054219f4de00c066692330b630e19d5416
child 309526 c76190898611d3d5b111462daa1487986e5417a6
push id80631
push userryanvm@gmail.com
push dateTue, 16 Aug 2016 13:32:12 +0000
treeherdermozilla-inbound@23ba2c37b8ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjanv
bugs1271500
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1271500 - Binary Key Support. r=janv
dom/indexedDB/Key.cpp
dom/indexedDB/Key.h
dom/indexedDB/ProfilerHelpers.h
testing/web-platform/meta/MANIFEST.json
testing/web-platform/tests/IndexedDB/idb_binary_key_conversion.htm
testing/web-platform/tests/IndexedDB/idbfactory_cmp3.htm
testing/web-platform/tests/IndexedDB/idbfactory_cmp4.htm
--- a/dom/indexedDB/Key.cpp
+++ b/dom/indexedDB/Key.cpp
@@ -29,20 +29,21 @@ namespace mozilla {
 namespace dom {
 namespace indexedDB {
 
 /*
  Here's how we encode keys:
 
  Basic strategy is the following
 
- Numbers: 0x10 n n n n n n n n    ("n"s are encoded 64bit float)
- Dates:   0x20 n n n n n n n n    ("n"s are encoded 64bit float)
- Strings: 0x30 s s s ... 0        ("s"s are encoded unicode bytes)
- Arrays:  0x50 i i i ... 0        ("i"s are encoded array items)
+ Numbers:  0x10 n n n n n n n n    ("n"s are encoded 64bit float)
+ Dates:    0x20 n n n n n n n n    ("n"s are encoded 64bit float)
+ Strings:  0x30 s s s ... 0        ("s"s are encoded unicode bytes)
+ Binaries: 0x40 s s s ... 0        ("s"s are encoded unicode bytes)
+ Arrays:   0x50 i i i ... 0        ("i"s are encoded array items)
 
 
  When encoding floats, 64bit IEEE 754 are almost sortable, except that
  positive sort lower than negative, and negative sort descending. So we use
  the following encoding:
  
  value < 0 ?
    (-to64bitInt(value)) :
@@ -55,16 +56,19 @@ namespace indexedDB {
  Chars 7F        - (3FFF+7F)    are encoded as 10xxxxxx xxxxxxxx with 7F subtracted
  Chars (3FFF+80) - FFFF         are encoded as 11xxxxxx xxxxxxxx xx000000
 
  This ensures that the first byte is never encoded as 0, which means that the
  string terminator (per basic-stategy table) sorts before any character.
  The reason that (3FFF+80) - FFFF is encoded "shifted up" 6 bits is to maximize
  the chance that the last character is 0. See below for why.
 
+ When encoding binaries, the algorithm is the same to how strings are encoded.
+ Since each octet in binary is in the range of [0-255], it'll take 1 to 2 encoded
+ unicode bytes.
 
  When encoding Arrays, we use an additional trick. Rather than adding a byte
  containing the value 0x50 to indicate type, we instead add 0x50 to the next byte.
  This is usually the byte containing the type of the first item in the array.
  So simple examples are
 
  ["foo"]      0x80 s s s 0 0                              // 0x80 is 0x30 + 0x50
  [1, 2]       0x60 n n n n n n n n 1 n n n n n n n n 0    // 0x60 is 0x10 + 0x50
@@ -107,17 +111,17 @@ namespace indexedDB {
 nsresult
 Key::ToLocaleBasedKey(Key& aTarget, const nsCString& aLocale) const
 {
   if (IsUnset()) {
     aTarget.Unset();
     return NS_OK;
   }
 
-  if (IsFloat() || IsDate()) {
+  if (IsFloat() || IsDate() || IsBinary()) {
     aTarget.mBuffer = mBuffer;
     return NS_OK;
   }
 
   aTarget.mBuffer.Truncate();
   aTarget.mBuffer.SetCapacity(mBuffer.Length());
 
   auto* it = reinterpret_cast<const unsigned char*>(mBuffer.BeginReading());
@@ -287,16 +291,26 @@ Key::EncodeJSValInternal(JSContext* aCx,
       double t;
       if (!js::DateGetMsecSinceEpoch(aCx, obj, &t)) {
         IDB_REPORT_INTERNAL_ERR();
         return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
       }
       EncodeNumber(t, eDate + aTypeOffset);
       return NS_OK;
     }
+
+    if (JS_IsArrayBufferObject(obj)) {
+      EncodeBinary(obj, /* aIsViewObject */ false, aTypeOffset);
+      return NS_OK;
+    }
+
+    if (JS_IsArrayBufferViewObject(obj)) {
+      EncodeBinary(obj, /* aIsViewObject */ true, aTypeOffset);
+      return NS_OK;
+    }
   }
 
   return NS_ERROR_DOM_INDEXEDDB_DATA_ERR;
 }
 
 // static
 nsresult
 Key::DecodeJSValInternal(const unsigned char*& aPos, const unsigned char* aEnd,
@@ -364,16 +378,25 @@ Key::DecodeJSValInternal(const unsigned 
       return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
     }
 
     aVal.setObject(*date);
   }
   else if (*aPos - aTypeOffset == eFloat) {
     aVal.setDouble(DecodeNumber(aPos, aEnd));
   }
+  else if (*aPos - aTypeOffset == eBinary) {
+    JSObject* binary = DecodeBinary(aPos, aEnd, aCx);
+    if (!binary) {
+      IDB_REPORT_INTERNAL_ERR();
+      return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
+    }
+
+    aVal.setObject(*binary);
+  }
   else {
     NS_NOTREACHED("Unknown key type!");
   }
 
   return NS_OK;
 }
 
 #define ONE_BYTE_LIMIT 0x7E
@@ -398,16 +421,23 @@ Key::EncodeString(const nsAString& aStri
   const char16_t* end = aString.EndReading();
   EncodeString(start, end, aTypeOffset);
 }
 
 template <typename T>
 void
 Key::EncodeString(const T* aStart, const T* aEnd, uint8_t aTypeOffset)
 {
+  EncodeAsString(aStart, aEnd, eString + aTypeOffset);
+}
+
+template <typename T>
+void
+Key::EncodeAsString(const T* aStart, const T* aEnd, uint8_t aType)
+{
   // First measure how long the encoded string will be.
 
   // The +2 is for initial 3 and trailing 0. We'll compensate for multi-byte
   // chars below.
   uint32_t size = (aEnd - aStart) + 2;
   
   const T* start = aStart;
   const T* end = aEnd;
@@ -421,17 +451,17 @@ Key::EncodeString(const T* aStart, const
   uint32_t oldLen = mBuffer.Length();
   char* buffer;
   if (!mBuffer.GetMutableData(&buffer, oldLen + size)) {
     return;
   }
   buffer += oldLen;
 
   // Write type marker
-  *(buffer++) = eString + aTypeOffset;
+  *(buffer++) = aType;
 
   // Encode string
   for (const T* iter = start; iter < end; ++iter) {
     if (*iter <= ONE_BYTE_LIMIT) {
       *(buffer++) = *iter + ONE_BYTE_ADJUST;
     }
     else if (char16_t(*iter) <= TWO_BYTE_LIMIT) {
       char16_t c = char16_t(*iter) + TWO_BYTE_ADJUST + 0x8000;
@@ -613,16 +643,92 @@ Key::DecodeNumber(const unsigned char*& 
   // MSVC build warning C4146 (negating an unsigned value).
   pun.u = number & PR_UINT64(0x8000000000000000) ?
           (number & ~PR_UINT64(0x8000000000000000)) :
           (0 - number);
 
   return pun.d;
 }
 
+void
+Key::EncodeBinary(JSObject* aObject, bool aIsViewObject, uint8_t aTypeOffset)
+{
+  uint8_t* bufferData;
+  uint32_t bufferLength;
+  bool unused;
+
+  if (aIsViewObject) {
+    js::GetArrayBufferViewLengthAndData(aObject, &bufferLength, &unused, &bufferData);
+  } else {
+    js::GetArrayBufferLengthAndData(aObject, &bufferLength, &unused, &bufferData);
+  }
+
+  EncodeAsString(bufferData, bufferData + bufferLength, eBinary + aTypeOffset);
+}
+
+// static
+JSObject*
+Key::DecodeBinary(const unsigned char*& aPos,
+                  const unsigned char* aEnd,
+                  JSContext* aCx)
+{
+  MOZ_ASSERT(*aPos % eMaxType == eBinary, "Don't call me!");
+
+  const unsigned char* buffer = ++aPos;
+
+  // First measure how big the decoded array buffer will be.
+  size_t size = 0;
+  const unsigned char* iter;
+  for (iter = buffer; iter < aEnd && *iter != eTerminator; ++iter) {
+    if (*iter & 0x80) {
+      iter++;
+    }
+    ++size;
+  }
+
+  if (!size) {
+    return JS_NewArrayBuffer(aCx, 0);
+  }
+
+  uint8_t* out = static_cast<uint8_t*>(JS_malloc(aCx, size));
+  if (NS_WARN_IF(!out)) {
+    return nullptr;
+  }
+
+  uint8_t* pos = out;
+
+  // Set end so that we don't have to check for null termination in the loop
+  // below
+  if (iter < aEnd) {
+    aEnd = iter;
+  }
+
+  for (iter = buffer; iter < aEnd;) {
+    if (!(*iter & 0x80)) {
+      *pos = *(iter++) - ONE_BYTE_ADJUST;
+    }
+    else {
+      uint16_t c = (uint16_t(*(iter++)) << 8);
+      if (iter < aEnd) {
+        c |= *(iter++);
+      }
+      *pos = static_cast<uint8_t>(c - TWO_BYTE_ADJUST - 0x8000);
+    }
+
+    ++pos;
+  }
+
+  aPos = iter + 1;
+
+  MOZ_ASSERT(static_cast<size_t>(pos - out) == size,
+             "Should have written the whole buffer");
+
+  return JS_NewArrayBufferWithContents(aCx, size, out);
+}
+
 nsresult
 Key::BindToStatement(mozIStorageStatement* aStatement,
                      const nsACString& aParamName) const
 {
   nsresult rv;
   if (IsUnset()) {
     rv = aStatement->BindNullByName(aParamName);
   } else {
--- a/dom/indexedDB/Key.h
+++ b/dom/indexedDB/Key.h
@@ -30,16 +30,17 @@ class Key
   nsCString mBuffer;
 
 public:
   enum {
     eTerminator = 0,
     eFloat = 0x10,
     eDate = 0x20,
     eString = 0x30,
+    eBinary = 0x40,
     eArray = 0x50,
     eMaxType = eArray
   };
 
   static const uint8_t kMaxArrayCollapse = uint8_t(3);
   static const uint8_t kMaxRecursionDepth = uint8_t(64);
 
   Key()
@@ -140,16 +141,22 @@ public:
 
   bool
   IsString() const
   {
     return !IsUnset() && *BufferStart() == eString;
   }
 
   bool
+  IsBinary() const
+  {
+    return !IsUnset() && *BufferStart() == eBinary;
+  }
+
+  bool
   IsArray() const
   {
     return !IsUnset() && *BufferStart() >= eArray;
   }
 
   double
   ToFloat() const
   {
@@ -282,41 +289,53 @@ private:
 
   void
   EncodeString(const nsAString& aString, uint8_t aTypeOffset);
 
   template <typename T>
   void
   EncodeString(const T* aStart, const T* aEnd, uint8_t aTypeOffset);
 
+  template <typename T>
+  void
+  EncodeAsString(const T* aStart, const T* aEnd, uint8_t aType);
+
 #ifdef ENABLE_INTL_API
   nsresult
   EncodeLocaleString(const nsDependentString& aString, uint8_t aTypeOffset,
                      const nsCString& aLocale);
 #endif
 
   void
   EncodeNumber(double aFloat, uint8_t aType);
 
+  void
+  EncodeBinary(JSObject* aObject, bool aIsViewObject, uint8_t aTypeOffset);
+
   // Decoding functions. aPos points into mBuffer and is adjusted to point
   // past the consumed value.
   static nsresult
   DecodeJSVal(const unsigned char*& aPos,
               const unsigned char* aEnd,
               JSContext* aCx,
               JS::MutableHandle<JS::Value> aVal);
 
   static void
   DecodeString(const unsigned char*& aPos,
                const unsigned char* aEnd,
                nsString& aString);
 
   static double
   DecodeNumber(const unsigned char*& aPos, const unsigned char* aEnd);
 
+  static JSObject*
+  DecodeBinary(const unsigned char*& aPos,
+               const unsigned char* aEnd,
+               JSContext* aCx);
+
   nsresult
   EncodeJSValInternal(JSContext* aCx,
                       JS::Handle<JS::Value> aVal,
                       uint8_t aTypeOffset,
                       uint16_t aRecursionDepth);
 
   static nsresult
   DecodeJSValInternal(const unsigned char*& aPos,
--- a/dom/indexedDB/ProfilerHelpers.h
+++ b/dom/indexedDB/ProfilerHelpers.h
@@ -196,16 +196,18 @@ public:
     } else if (aKey.IsFloat()) {
       AppendPrintf("%g", aKey.ToFloat());
     } else if (aKey.IsDate()) {
       AppendPrintf("<Date %g>", aKey.ToDateMsec());
     } else if (aKey.IsString()) {
       nsAutoString str;
       aKey.ToString(str);
       AppendPrintf("\"%s\"", NS_ConvertUTF16toUTF8(str).get());
+    } else if (aKey.IsBinary()) {
+      AssignLiteral("[object ArrayBuffer]");
     } else {
       MOZ_ASSERT(aKey.IsArray());
       AssignLiteral("[...]");
     }
   }
 
   explicit
   LoggingString(const IDBCursor::Direction aDirection)
--- a/testing/web-platform/meta/MANIFEST.json
+++ b/testing/web-platform/meta/MANIFEST.json
@@ -11381,16 +11381,20 @@
         "path": "IndexedDB/close-in-upgradeneeded.html",
         "url": "/IndexedDB/close-in-upgradeneeded.html"
       },
       {
         "path": "IndexedDB/cursor-overloads.htm",
         "url": "/IndexedDB/cursor-overloads.htm"
       },
       {
+        "path": "IndexedDB/idb_binary_key_conversion.htm",
+        "url": "/IndexedDB/idb_binary_key_conversion.htm"
+      },
+      {
         "path": "IndexedDB/idb_webworkers.htm",
         "url": "/IndexedDB/idb_webworkers.htm"
       },
       {
         "path": "IndexedDB/idbcursor-advance-continue-async.htm",
         "url": "/IndexedDB/idbcursor-advance-continue-async.htm"
       },
       {
@@ -11781,16 +11785,24 @@
         "path": "IndexedDB/idbfactory_cmp.htm",
         "url": "/IndexedDB/idbfactory_cmp.htm"
       },
       {
         "path": "IndexedDB/idbfactory_cmp2.htm",
         "url": "/IndexedDB/idbfactory_cmp2.htm"
       },
       {
+        "path": "IndexedDB/idbfactory_cmp3.htm",
+        "url": "/IndexedDB/idbfactory_cmp3.htm"
+      },
+      {
+        "path": "IndexedDB/idbfactory_cmp4.htm",
+        "url": "/IndexedDB/idbfactory_cmp4.htm"
+      },
+      {
         "path": "IndexedDB/idbfactory_deleteDatabase.htm",
         "url": "/IndexedDB/idbfactory_deleteDatabase.htm"
       },
       {
         "path": "IndexedDB/idbfactory_deleteDatabase2.htm",
         "url": "/IndexedDB/idbfactory_deleteDatabase2.htm"
       },
       {
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/IndexedDB/idb_binary_key_conversion.htm
@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+<meta charset=utf-8>
+<title>Verify the coversion of various types of BufferSource</title>
+<link rel="author" title="Mozilla" href="https://www.mozilla.org">
+<link rel="help" href="http://w3c.github.io/IndexedDB/#key-construct">
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+
+<script>
+  test(function() {
+    let binary = new ArrayBuffer(0);
+    let key = IDBKeyRange.lowerBound(binary).lower;
+
+    assert_true(key instanceof ArrayBuffer);
+    assert_equals(key.byteLength, 0);
+    assert_equals(key.byteLength, binary.byteLength);
+  }, "Empty ArrayBuffer");
+
+  test(function() {
+    let binary = new ArrayBuffer(4);
+    let dataView = new DataView(binary);
+    dataView.setUint32(0, 1234567890);
+
+    let key = IDBKeyRange.lowerBound(binary).lower;
+
+    assert_true(key instanceof ArrayBuffer);
+    assert_equals(key.byteLength, 4);
+    assert_equals(dataView.getUint32(0), new DataView(key).getUint32(0));
+  }, "ArrayBuffer");
+
+  test(function() {
+    let binary = new ArrayBuffer(4);
+    let dataView = new DataView(binary);
+    dataView.setUint32(0, 1234567890);
+
+    let key = IDBKeyRange.lowerBound(dataView).lower;
+
+    assert_true(key instanceof ArrayBuffer);
+    assert_equals(key.byteLength, 4);
+    assert_equals(dataView.getUint32(0), new DataView(key).getUint32(0));
+  }, "DataView");
+
+  test(function() {
+    let binary = new ArrayBuffer(4);
+    let dataView = new DataView(binary);
+    let int8Array = new Int8Array(binary);
+    int8Array.set([16, -32, 64, -128]);
+
+    let key = IDBKeyRange.lowerBound(int8Array).lower;
+    let keyInInt8Array = new Int8Array(key);
+
+    assert_true(key instanceof ArrayBuffer);
+    assert_equals(key.byteLength, 4);
+    for(let i = 0; i < int8Array.length; i++) {
+      assert_equals(keyInInt8Array[i], int8Array[i]);
+    }
+  }, "TypedArray(Int8Array)");
+
+  test(function() {
+    let binary = new ArrayBuffer(4);
+    let dataView = new DataView(binary);
+    let int8Array = new Int8Array(binary);
+    int8Array.set([16, -32, 64, -128]);
+
+    let key = IDBKeyRange.lowerBound([int8Array]).lower;
+
+    assert_true(key instanceof Array);
+    assert_true(key[0] instanceof ArrayBuffer);
+    assert_equals(key[0].byteLength, 4);
+
+    let keyInInt8Array = new Int8Array(key[0]);
+
+    for(let i = 0; i < int8Array.length; i++) {
+      assert_equals(keyInInt8Array[i], int8Array[i]);
+    }
+  }, "Array of TypedArray(Int8Array)");
+</script>
+
+<div id=log></div>
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/IndexedDB/idbfactory_cmp3.htm
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<meta charset=utf-8>
+<title>IDBFactory.cmp() - compared keys in different types</title>
+<link rel="author" title="Mozilla" href="https://www.mozilla.org">
+<link rel="help" href="http://w3c.github.io/IndexedDB/#key-construct">
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+
+<script>
+  test(function() {
+    assert_equals(indexedDB.cmp([0], new Uint8Array([0])), 1, "Array > Binary");
+  }, "Array v.s. Binary");
+
+  test(function() {
+    assert_equals(indexedDB.cmp(new Uint8Array([0]), "0"), 1, "Binary > String");
+  }, "Binary v.s. String");
+
+  test(function() {
+    assert_equals(indexedDB.cmp("", new Date(0)), 1, "String > Date");
+  }, "String v.s. Date");
+
+  test(function() {
+    assert_equals(indexedDB.cmp(new Date(0), 0), 1, "Date > Number");
+  }, "Date v.s. Number");
+</script>
+
+<div id=log></div>
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/IndexedDB/idbfactory_cmp4.htm
@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<meta charset=utf-8>
+<title>IDBFactory.cmp() - comparison of binary keys</title>
+<link rel="author" title="Mozilla" href="https://www.mozilla.org">
+<link rel="help" href="http://w3c.github.io/IndexedDB/#key-construct">
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+
+<script>
+  test(function() {
+    assert_equals(indexedDB.cmp(new Int8Array([-1]), new Uint8Array([0])), 1,
+    "255(-1) shall be larger than 0");
+  }, "Compare in unsigned octet values (in the range [0, 255])");
+
+  test(function() {
+    assert_equals(indexedDB.cmp(
+        new Uint8Array([255, 254, 253]),
+        new Uint8Array([255, 253, 254])),
+        1,
+        "[255, 254, 253] shall be larger than [255, 253, 254]");
+  }, "Compare values in then same length");
+
+  test(function() {
+    assert_equals(indexedDB.cmp(
+        new Uint8Array([255, 254]),
+        new Uint8Array([255, 253, 254])),
+        1,
+        "[255, 254] shall be larger than [255, 253, 254]");
+  }, "Compare values in different lengths");
+
+  test(function() {
+    assert_equals(indexedDB.cmp(
+        new Uint8Array([255, 253, 254]),
+        new Uint8Array([255, 253])),
+        1,
+        "[255, 253, 254] shall be larger than [255, 253]");
+  }, "Compare when the values in the range of their minimal length are the same");
+</script>
+
+<div id=log></div>