Bug 1284031 (Part 1) - Advance SourceBufferIterator in Lex() per-state. r=njn
authorSeth Fowler <mark.seth.fowler@gmail.com>
Thu, 14 Jul 2016 19:55:35 -0700
changeset 305125 656d5e742049c4ad966389fa2db3b11bf956c7cf
parent 305124 762323d53afaf171645aa0117bad84ed19842fc9
child 305126 0284587d752d35d2ddf6f2d9983047597ff1e38e
push id79504
push usermfowler@mozilla.com
push dateSat, 16 Jul 2016 07:37:15 +0000
treeherdermozilla-inbound@b1e09b3b77dd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnjn
bugs1284031
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1284031 (Part 1) - Advance SourceBufferIterator in Lex() per-state. r=njn
image/StreamingLexer.h
image/test/gtest/TestStreamingLexer.cpp
--- a/image/StreamingLexer.h
+++ b/image/StreamingLexer.h
@@ -26,23 +26,26 @@ namespace image {
 
 /// Buffering behaviors for StreamingLexer transitions.
 enum class BufferingStrategy
 {
   BUFFERED,   // Data will be buffered and processed in one chunk.
   UNBUFFERED  // Data will be processed as it arrives, in multiple chunks.
 };
 
-/// The result of a call to StreamingLexer::Lex().
+/// Possible terminal states for the lexer.
 enum class TerminalState
 {
   SUCCESS,
   FAILURE
 };
 
+/// The result of a call to StreamingLexer::Lex().
+typedef Variant<TerminalState> LexerResult;
+
 /**
  * LexerTransition is a type used to give commands to the lexing framework.
  * Code that uses StreamingLexer can create LexerTransition values using the
  * static methods on Transition, and then return them to the lexing framework
  * for execution.
  */
 template <typename State>
 class LexerTransition
@@ -254,213 +257,172 @@ private:
  * 1198451 lands, since we can then just return a function representing the next
  * state directly.
  */
 template <typename State, size_t InlineBufferSize = 16>
 class StreamingLexer
 {
 public:
   explicit StreamingLexer(LexerTransition<State> aStartState)
-    : mTransition(aStartState)
+    : mTransition(TerminalState::FAILURE)
     , mToReadUnbuffered(0)
-  { }
+  {
+    SetTransition(aStartState);
+  }
 
   template <typename Func>
   Maybe<TerminalState> Lex(SourceBufferIterator& aIterator,
                            IResumable* aOnResume,
                            Func aFunc)
   {
     if (mTransition.NextStateIsTerminal()) {
       // We've already reached a terminal state. We never deliver any more data
       // in this case; just return the terminal state again immediately.
       return Some(mTransition.NextStateAsTerminal());
     }
 
+    Maybe<LexerResult> result;
     do {
-      switch (aIterator.AdvanceOrScheduleResume(SIZE_MAX, aOnResume)) {
+      // Figure out how much we need to read.
+      const size_t toRead = mTransition.Buffering() == BufferingStrategy::UNBUFFERED
+                          ? mToReadUnbuffered
+                          : mTransition.Size() - mBuffer.length();
+
+      // Attempt to advance the iterator by |toRead| bytes.
+      switch (aIterator.AdvanceOrScheduleResume(toRead, aOnResume)) {
         case SourceBufferIterator::WAITING:
           // We can't continue because the rest of the data hasn't arrived from
           // the network yet. We don't have to do anything special; the
           // SourceBufferIterator will ensure that |aOnResume| gets called when
           // more data is available.
           return Nothing();
 
         case SourceBufferIterator::COMPLETE:
           // Normally even if the data is truncated, we want decoding to
           // succeed so we can display whatever we got. However, if the
           // SourceBuffer was completed with a failing status, we want to fail.
           // This happens only in exceptional situations like SourceBuffer
           // itself encountering a failure due to OOM.
-          mTransition = NS_SUCCEEDED(aIterator.CompletionStatus())
-                      ? Transition::TerminateSuccess()
-                      : Transition::TerminateFailure();
+          result = SetTransition(NS_SUCCEEDED(aIterator.CompletionStatus())
+                 ? Transition::TerminateSuccess()
+                 : Transition::TerminateFailure());
           break;
 
         case SourceBufferIterator::READY:
-          // Process the new data that became available. This may result in us
-          // transitioning to a terminal state; we'll check if that happened at
-          // the bottom of the loop.
+          // Process the new data that became available.
           MOZ_ASSERT(aIterator.Data());
-          MOZ_ASSERT(aIterator.Length() > 0);
-          Lex(aIterator.Data(), aIterator.Length(), aFunc);
+
+          result = mTransition.Buffering() == BufferingStrategy::UNBUFFERED
+                 ? UnbufferedRead(aIterator, aFunc)
+                 : BufferedRead(aIterator, aFunc);
           break;
 
         default:
           MOZ_ASSERT_UNREACHABLE("Unknown SourceBufferIterator state");
-          mTransition = Transition::TerminateFailure();
+          result = SetTransition(Transition::TerminateFailure());
       }
-    } while (!mTransition.NextStateIsTerminal());
+    } while (!result);
 
-    // We're done. Return the terminal state.
-    return Some(mTransition.NextStateAsTerminal());
+    // Map |LexerResult| onto the old |Maybe<TerminalState>| API.
+    return result->is<TerminalState>() ? Some(result->as<TerminalState>())
+                                       : Nothing();
   }
 
+private:
   template <typename Func>
-  Maybe<TerminalState> Lex(const char* aInput, size_t aLength, Func aFunc)
+  Maybe<LexerResult> UnbufferedRead(SourceBufferIterator& aIterator, Func aFunc)
   {
-    MOZ_ASSERT(aInput);
-
-    if (mTransition.NextStateIsTerminal()) {
-      // We've already reached a terminal state. We never deliver any more data
-      // in this case; just return the terminal state again immediately.
-      return Some(mTransition.NextStateAsTerminal());
-    }
+    MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::UNBUFFERED);
+    MOZ_ASSERT(mBuffer.empty(),
+               "Buffered read at the same time as unbuffered read?");
 
     if (mToReadUnbuffered > 0) {
-      // We're continuing an unbuffered read.
-
-      MOZ_ASSERT(mBuffer.empty(),
-                 "Shouldn't be continuing an unbuffered read and a buffered "
-                 "read at the same time");
-
-      size_t toRead = std::min(mToReadUnbuffered, aLength);
-
       // Call aFunc with the unbuffered state to indicate that we're in the
       // middle of an unbuffered read. We enforce that any state transition
       // passed back to us is either a terminal state or takes us back to the
       // unbuffered state.
       LexerTransition<State> unbufferedTransition =
-        aFunc(mTransition.UnbufferedState(), aInput, toRead);
+        aFunc(mTransition.UnbufferedState(), aIterator.Data(), aIterator.Length());
       if (unbufferedTransition.NextStateIsTerminal()) {
-        mTransition = unbufferedTransition;
-        return Some(mTransition.NextStateAsTerminal());  // Done!
+        return SetTransition(unbufferedTransition);
       }
+
       MOZ_ASSERT(mTransition.UnbufferedState() ==
                    unbufferedTransition.NextState());
 
-      aInput += toRead;
-      aLength -= toRead;
-      mToReadUnbuffered -= toRead;
+      mToReadUnbuffered -= aIterator.Length();
       if (mToReadUnbuffered != 0) {
-        return Nothing();  // Need more input.
-      }
-
-      // We're done with the unbuffered read, so transition to the next state.
-      mTransition = aFunc(mTransition.NextState(), nullptr, 0);
-      if (mTransition.NextStateIsTerminal()) {
-        return Some(mTransition.NextStateAsTerminal());  // Done!
-      }
-    } else if (0 < mBuffer.length()) {
-      // We're continuing a buffered read.
-
-      MOZ_ASSERT(mToReadUnbuffered == 0,
-                 "Shouldn't be continuing an unbuffered read and a buffered "
-                 "read at the same time");
-      MOZ_ASSERT(mBuffer.length() < mTransition.Size(),
-                 "Buffered more than we needed?");
-
-      size_t toRead = std::min(aLength, mTransition.Size() - mBuffer.length());
-
-      if (!mBuffer.append(aInput, toRead)) {
-        return Some(TerminalState::FAILURE);
-      }
-      aInput += toRead;
-      aLength -= toRead;
-      if (mBuffer.length() != mTransition.Size()) {
-        return Nothing();  // Need more input.
-      }
-
-      // We've buffered everything, so transition to the next state.
-      mTransition =
-        aFunc(mTransition.NextState(), mBuffer.begin(), mBuffer.length());
-      mBuffer.clear();
-      if (mTransition.NextStateIsTerminal()) {
-        return Some(mTransition.NextStateAsTerminal());  // Done!
+        return Nothing();  // Keep processing.
       }
     }
 
-    MOZ_ASSERT(mToReadUnbuffered == 0);
-    MOZ_ASSERT(mBuffer.empty());
-
-    // Process states as long as we continue to have enough input to do so.
-    while (mTransition.Size() <= aLength) {
-      size_t toRead = mTransition.Size();
-
-      if (mTransition.Buffering() == BufferingStrategy::BUFFERED) {
-        mTransition = aFunc(mTransition.NextState(), aInput, toRead);
-      } else {
-        MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::UNBUFFERED);
+    // We're done with the unbuffered read, so transition to the next state.
+    return SetTransition(aFunc(mTransition.NextState(), nullptr, 0));
+  }
 
-        // Call aFunc with the unbuffered state to indicate that we're in the
-        // middle of an unbuffered read. We enforce that any state transition
-        // passed back to us is either a terminal state or takes us back to the
-        // unbuffered state.
-        LexerTransition<State> unbufferedTransition =
-          aFunc(mTransition.UnbufferedState(), aInput, toRead);
-        if (unbufferedTransition.NextStateIsTerminal()) {
-          mTransition = unbufferedTransition;
-          return Some(mTransition.NextStateAsTerminal());  // Done!
-        }
-        MOZ_ASSERT(mTransition.UnbufferedState() ==
-                     unbufferedTransition.NextState());
+  template <typename Func>
+  Maybe<LexerResult> BufferedRead(SourceBufferIterator& aIterator, Func aFunc)
+  {
+    MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::BUFFERED);
+    MOZ_ASSERT(mToReadUnbuffered == 0,
+               "Buffered read at the same time as unbuffered read?");
+    MOZ_ASSERT(mBuffer.length() < mTransition.Size() ||
+               (mBuffer.length() == 0 && mTransition.Size() == 0),
+               "Buffered more than we needed?");
 
-        // We're done with the unbuffered read, so transition to the next state.
-        mTransition = aFunc(mTransition.NextState(), nullptr, 0);
-      }
+    // If we have all the data, we don't actually need to buffer anything.
+    if (mBuffer.empty() && aIterator.Length() == mTransition.Size()) {
+      return SetTransition(aFunc(mTransition.NextState(),
+                                 aIterator.Data(),
+                                 aIterator.Length()));
+    }
 
-      aInput += toRead;
-      aLength -= toRead;
-
-      if (mTransition.NextStateIsTerminal()) {
-        return Some(mTransition.NextStateAsTerminal());  // Done!
-      }
+    // We do need to buffer, so make sure the buffer has enough capacity. We
+    // deliberately wait until we know for sure we need to buffer to call
+    // reserve() since it could require memory allocation.
+    if (!mBuffer.reserve(mTransition.Size())) {
+      return SetTransition(Transition::TerminateFailure());
     }
 
-    if (aLength == 0) {
-      // We finished right at a transition point. Just wait for more data.
-      return Nothing();
+    // Append the new data we just got to the buffer.
+    if (!mBuffer.append(aIterator.Data(), aIterator.Length())) {
+      return SetTransition(Transition::TerminateFailure());
+    }
+
+    if (mBuffer.length() != mTransition.Size()) {
+      return Nothing();  // Keep processing.
     }
 
-    // If the next state is unbuffered, deliver what we can and then wait.
-    if (mTransition.Buffering() == BufferingStrategy::UNBUFFERED) {
-      LexerTransition<State> unbufferedTransition =
-        aFunc(mTransition.UnbufferedState(), aInput, aLength);
-      if (unbufferedTransition.NextStateIsTerminal()) {
-        mTransition = unbufferedTransition;
-        return Some(mTransition.NextStateAsTerminal());  // Done!
-      }
-      MOZ_ASSERT(mTransition.UnbufferedState() ==
-                   unbufferedTransition.NextState());
+    // We've buffered everything, so transition to the next state.
+    return SetTransition(aFunc(mTransition.NextState(),
+                               mBuffer.begin(),
+                               mBuffer.length()));
+  }
 
-      mToReadUnbuffered = mTransition.Size() - aLength;
-      return Nothing();  // Need more input.
+  Maybe<LexerResult> SetTransition(const LexerTransition<State>& aTransition)
+  {
+    mTransition = aTransition;
+
+    // Get rid of anything left over from the previous state.
+    mBuffer.clear();
+    mToReadUnbuffered = 0;
+
+    // If we reached a terminal state, let the caller know.
+    if (mTransition.NextStateIsTerminal()) {
+      return Some(LexerResult(mTransition.NextStateAsTerminal()));
     }
 
-    // If the next state is buffered, buffer what we can and then wait.
-    MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::BUFFERED);
-    if (!mBuffer.reserve(mTransition.Size())) {
-      return Some(TerminalState::FAILURE);  // Done due to allocation failure.
+    // If we're entering an unbuffered state, record how long we'll stay in it.
+    if (mTransition.Buffering() == BufferingStrategy::UNBUFFERED) {
+      mToReadUnbuffered = mTransition.Size();
     }
-    if (!mBuffer.append(aInput, aLength)) {
-      return Some(TerminalState::FAILURE);
-    }
-    return Nothing();  // Need more input.
+
+    return Nothing();  // Keep processing.
   }
 
-private:
   Vector<char, InlineBufferSize> mBuffer;
   LexerTransition<State> mTransition;
   size_t mToReadUnbuffered;
 };
 
 } // namespace image
 } // namespace mozilla
 
--- a/image/test/gtest/TestStreamingLexer.cpp
+++ b/image/test/gtest/TestStreamingLexer.cpp
@@ -14,76 +14,118 @@ enum class TestState
 {
   ONE,
   TWO,
   THREE,
   UNBUFFERED
 };
 
 void
-CheckData(const char* aData, size_t aLength)
+CheckLexedData(const char* aData, size_t aLength, size_t aExpectedLength)
 {
-  EXPECT_TRUE(aLength == 3);
-  EXPECT_EQ(1, aData[0]);
-  EXPECT_EQ(2, aData[1]);
-  EXPECT_EQ(3, aData[2]);
+  EXPECT_TRUE(aLength == aExpectedLength);
+
+  for (size_t i = 0; i < aLength; ++i) {
+    EXPECT_EQ(aData[i], char((i % 3) + 1));
+  }
 }
 
 LexerTransition<TestState>
 DoLex(TestState aState, const char* aData, size_t aLength)
 {
   switch (aState) {
     case TestState::ONE:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::To(TestState::TWO, 3);
     case TestState::TWO:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::To(TestState::THREE, 3);
     case TestState::THREE:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::TerminateSuccess();
     default:
-      MOZ_CRASH("Unknown TestState");
+      MOZ_CRASH("Unexpected or unhandled TestState");
   }
 }
 
 LexerTransition<TestState>
 DoLexWithUnbuffered(TestState aState, const char* aData, size_t aLength,
                     Vector<char>& aUnbufferedVector)
 {
   switch (aState) {
     case TestState::ONE:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 3);
     case TestState::UNBUFFERED:
       EXPECT_TRUE(aLength <= 3);
       EXPECT_TRUE(aUnbufferedVector.append(aData, aLength));
       return Transition::ContinueUnbuffered(TestState::UNBUFFERED);
     case TestState::TWO:
-      CheckData(aUnbufferedVector.begin(), aUnbufferedVector.length());
+      CheckLexedData(aUnbufferedVector.begin(), aUnbufferedVector.length(), 3);
       return Transition::To(TestState::THREE, 3);
     case TestState::THREE:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::TerminateSuccess();
     default:
-      MOZ_CRASH("Unknown TestState");
+      MOZ_CRASH("Unexpected or unhandled TestState");
   }
 }
 
 LexerTransition<TestState>
 DoLexWithUnbufferedTerminate(TestState aState, const char* aData, size_t aLength)
 {
   switch (aState) {
     case TestState::ONE:
-      CheckData(aData, aLength);
+      CheckLexedData(aData, aLength, 3);
       return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 3);
     case TestState::UNBUFFERED:
       return Transition::TerminateSuccess();
     default:
-      MOZ_CRASH("Unknown TestState");
+      MOZ_CRASH("Unexpected or unhandled TestState");
+  }
+}
+
+LexerTransition<TestState>
+DoLexWithZeroLengthStates(TestState aState, const char* aData, size_t aLength)
+{
+  switch (aState) {
+    case TestState::ONE:
+      EXPECT_TRUE(aLength == 0);
+      return Transition::To(TestState::TWO, 0);
+    case TestState::TWO:
+      EXPECT_TRUE(aLength == 0);
+      return Transition::To(TestState::THREE, 9);
+    case TestState::THREE:
+      CheckLexedData(aData, aLength, 9);
+      return Transition::TerminateSuccess();
+    default:
+      MOZ_CRASH("Unexpected or unhandled TestState");
+  }
+}
+
+LexerTransition<TestState>
+DoLexWithZeroLengthStatesUnbuffered(TestState aState,
+                                    const char* aData,
+                                    size_t aLength)
+{
+  switch (aState) {
+    case TestState::ONE:
+      EXPECT_TRUE(aLength == 0);
+      return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 0);
+    case TestState::TWO:
+      EXPECT_TRUE(aLength == 0);
+      return Transition::To(TestState::THREE, 9);
+    case TestState::THREE:
+      CheckLexedData(aData, aLength, 9);
+      return Transition::TerminateSuccess();
+    case TestState::UNBUFFERED:
+      ADD_FAILURE() << "Should not enter zero-length unbuffered state";
+      return Transition::TerminateFailure();
+    default:
+      MOZ_CRASH("Unexpected or unhandled TestState");
   }
 }
 
 class ImageStreamingLexer : public ::testing::Test
 {
 public:
   ImageStreamingLexer()
     : mLexer(Transition::To(TestState::ONE, 3))
@@ -100,55 +142,37 @@ protected:
   RefPtr<SourceBuffer> mSourceBuffer;
   SourceBufferIterator mIterator;
   RefPtr<ExpectNoResume> mExpectNoResume;
   RefPtr<CountResumes> mCountResumes;
 };
 
 TEST_F(ImageStreamingLexer, ZeroLengthData)
 {
-  // Test delivering a zero-length piece of data.
-  Maybe<TerminalState> result = mLexer.Lex(mData, 0, DoLex);
-  EXPECT_TRUE(result.isNothing());
-}
+  // Test a zero-length input.
+  mSourceBuffer->Complete(NS_OK);
 
-TEST_F(ImageStreamingLexer, SingleChunk)
-{
-  // Test delivering all the data at once.
-  Maybe<TerminalState> result = mLexer.Lex(mData, sizeof(mData), DoLex);
+  Maybe<TerminalState> result = mLexer.Lex(mIterator, mExpectNoResume, DoLex);
+
   EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::SUCCESS), result);
+  EXPECT_EQ(Some(TerminalState::FAILURE), result);
 }
 
 TEST_F(ImageStreamingLexer, SingleChunkFromSourceBuffer)
 {
   // Test delivering all the data at once.
   mSourceBuffer->Append(mData, sizeof(mData));
   mSourceBuffer->Complete(NS_OK);
 
   Maybe<TerminalState> result = mLexer.Lex(mIterator, mExpectNoResume, DoLex);
 
   EXPECT_TRUE(result.isSome());
   EXPECT_EQ(Some(TerminalState::SUCCESS), result);
 }
 
-TEST_F(ImageStreamingLexer, SingleChunkWithUnbuffered)
-{
-  Vector<char> unbufferedVector;
-
-  // Test delivering all the data at once.
-  Maybe<TerminalState> result =
-    mLexer.Lex(mData, sizeof(mData),
-               [&](TestState aState, const char* aData, size_t aLength) {
-      return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
-  });
-  EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-}
-
 TEST_F(ImageStreamingLexer, SingleChunkWithUnbufferedFromSourceBuffer)
 {
   Vector<char> unbufferedVector;
 
   // Test delivering all the data at once.
   mSourceBuffer->Append(mData, sizeof(mData));
   mSourceBuffer->Complete(NS_OK);
 
@@ -157,31 +181,16 @@ TEST_F(ImageStreamingLexer, SingleChunkW
                [&](TestState aState, const char* aData, size_t aLength) {
       return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
   });
 
   EXPECT_TRUE(result.isSome());
   EXPECT_EQ(Some(TerminalState::SUCCESS), result);
 }
 
-TEST_F(ImageStreamingLexer, ChunkPerState)
-{
-  // Test delivering in perfectly-sized chunks, one per state.
-  for (unsigned i = 0; i < 3; ++i) {
-    Maybe<TerminalState> result = mLexer.Lex(mData + 3 * i, 3, DoLex);
-
-    if (i == 2) {
-      EXPECT_TRUE(result.isSome());
-      EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-    } else {
-      EXPECT_TRUE(result.isNothing());
-    }
-  }
-}
-
 TEST_F(ImageStreamingLexer, ChunkPerStateFromSourceBuffer)
 {
   // Test delivering in perfectly-sized chunks, one per state.
   for (unsigned i = 0; i < 3; ++i) {
     mSourceBuffer->Append(mData + 3 * i, 3);
     Maybe<TerminalState> result = mLexer.Lex(mIterator, mCountResumes, DoLex);
 
     if (i == 2) {
@@ -191,37 +200,16 @@ TEST_F(ImageStreamingLexer, ChunkPerStat
       EXPECT_TRUE(result.isNothing());
     }
   }
 
   EXPECT_EQ(2u, mCountResumes->Count());
   mSourceBuffer->Complete(NS_OK);
 }
 
-TEST_F(ImageStreamingLexer, ChunkPerStateWithUnbuffered)
-{
-  Vector<char> unbufferedVector;
-
-  // Test delivering in perfectly-sized chunks, one per state.
-  for (unsigned i = 0; i < 3; ++i) {
-    Maybe<TerminalState> result =
-      mLexer.Lex(mData + 3 * i, 3,
-                 [&](TestState aState, const char* aData, size_t aLength) {
-        return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
-    });
-
-    if (i == 2) {
-      EXPECT_TRUE(result.isSome());
-      EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-    } else {
-      EXPECT_TRUE(result.isNothing());
-    }
-  }
-}
-
 TEST_F(ImageStreamingLexer, ChunkPerStateWithUnbufferedFromSourceBuffer)
 {
   Vector<char> unbufferedVector;
 
   // Test delivering in perfectly-sized chunks, one per state.
   for (unsigned i = 0; i < 3; ++i) {
     mSourceBuffer->Append(mData + 3 * i, 3);
     Maybe<TerminalState> result =
@@ -237,31 +225,16 @@ TEST_F(ImageStreamingLexer, ChunkPerStat
       EXPECT_TRUE(result.isNothing());
     }
   }
 
   EXPECT_EQ(2u, mCountResumes->Count());
   mSourceBuffer->Complete(NS_OK);
 }
 
-TEST_F(ImageStreamingLexer, OneByteChunks)
-{
-  // Test delivering in one byte chunks.
-  for (unsigned i = 0; i < 9; ++i) {
-    Maybe<TerminalState> result = mLexer.Lex(mData + i, 1, DoLex);
-
-    if (i == 8) {
-      EXPECT_TRUE(result.isSome());
-      EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-    } else {
-      EXPECT_TRUE(result.isNothing());
-    }
-  }
-}
-
 TEST_F(ImageStreamingLexer, OneByteChunksFromSourceBuffer)
 {
   // Test delivering in one byte chunks.
   for (unsigned i = 0; i < 9; ++i) {
     mSourceBuffer->Append(mData + i, 1);
     Maybe<TerminalState> result = mLexer.Lex(mIterator, mCountResumes, DoLex);
 
     if (i == 8) {
@@ -271,37 +244,16 @@ TEST_F(ImageStreamingLexer, OneByteChunk
       EXPECT_TRUE(result.isNothing());
     }
   }
 
   EXPECT_EQ(8u, mCountResumes->Count());
   mSourceBuffer->Complete(NS_OK);
 }
 
-TEST_F(ImageStreamingLexer, OneByteChunksWithUnbuffered)
-{
-  Vector<char> unbufferedVector;
-
-  // Test delivering in one byte chunks.
-  for (unsigned i = 0; i < 9; ++i) {
-    Maybe<TerminalState> result =
-      mLexer.Lex(mData + i, 1,
-                 [&](TestState aState, const char* aData, size_t aLength) {
-        return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
-    });
-
-    if (i == 8) {
-      EXPECT_TRUE(result.isSome());
-      EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-    } else {
-      EXPECT_TRUE(result.isNothing());
-    }
-  }
-}
-
 TEST_F(ImageStreamingLexer, OneByteChunksWithUnbufferedFromSourceBuffer)
 {
   Vector<char> unbufferedVector;
 
   // Test delivering in one byte chunks.
   for (unsigned i = 0; i < 9; ++i) {
     mSourceBuffer->Append(mData + i, 1);
     Maybe<TerminalState> result =
@@ -317,38 +269,16 @@ TEST_F(ImageStreamingLexer, OneByteChunk
       EXPECT_TRUE(result.isNothing());
     }
   }
 
   EXPECT_EQ(8u, mCountResumes->Count());
   mSourceBuffer->Complete(NS_OK);
 }
 
-TEST_F(ImageStreamingLexer, TerminateSuccess)
-{
-  // Test that Terminate is "sticky".
-  Maybe<TerminalState> result =
-    mLexer.Lex(mData, sizeof(mData),
-               [&](TestState aState, const char* aData, size_t aLength) {
-      EXPECT_TRUE(aState == TestState::ONE);
-      return Transition::TerminateSuccess();
-  });
-  EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-
-  result =
-    mLexer.Lex(mData, sizeof(mData),
-               [&](TestState aState, const char* aData, size_t aLength) {
-      EXPECT_TRUE(false);  // Shouldn't get here.
-      return Transition::TerminateFailure();
-  });
-  EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-}
-
 TEST_F(ImageStreamingLexer, TerminateSuccessFromSourceBuffer)
 {
   mSourceBuffer->Append(mData, sizeof(mData));
   mSourceBuffer->Complete(NS_OK);
 
   // Test that Terminate is "sticky".
   SourceBufferIterator iterator = mSourceBuffer->Iterator();
   Maybe<TerminalState> result =
@@ -366,38 +296,16 @@ TEST_F(ImageStreamingLexer, TerminateSuc
                [&](TestState aState, const char* aData, size_t aLength) {
       EXPECT_TRUE(false);  // Shouldn't get here.
       return Transition::TerminateFailure();
   });
   EXPECT_TRUE(result.isSome());
   EXPECT_EQ(Some(TerminalState::SUCCESS), result);
 }
 
-TEST_F(ImageStreamingLexer, TerminateFailure)
-{
-  // Test that Terminate is "sticky".
-  Maybe<TerminalState> result =
-    mLexer.Lex(mData, sizeof(mData),
-               [&](TestState aState, const char* aData, size_t aLength) {
-      EXPECT_TRUE(aState == TestState::ONE);
-      return Transition::TerminateFailure();
-  });
-  EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::FAILURE), result);
-
-  result =
-    mLexer.Lex(mData, sizeof(mData),
-               [&](TestState aState, const char* aData, size_t aLength) {
-      EXPECT_TRUE(false);  // Shouldn't get here.
-      return Transition::TerminateFailure();
-  });
-  EXPECT_TRUE(result.isSome());
-  EXPECT_EQ(Some(TerminalState::FAILURE), result);
-}
-
 TEST_F(ImageStreamingLexer, TerminateFailureFromSourceBuffer)
 {
   mSourceBuffer->Append(mData, sizeof(mData));
   mSourceBuffer->Complete(NS_OK);
 
   // Test that Terminate is "sticky".
   SourceBufferIterator iterator = mSourceBuffer->Iterator();
   Maybe<TerminalState> result =
@@ -415,32 +323,16 @@ TEST_F(ImageStreamingLexer, TerminateFai
                [&](TestState aState, const char* aData, size_t aLength) {
       EXPECT_TRUE(false);  // Shouldn't get here.
       return Transition::TerminateFailure();
   });
   EXPECT_TRUE(result.isSome());
   EXPECT_EQ(Some(TerminalState::FAILURE), result);
 }
 
-TEST_F(ImageStreamingLexer, TerminateUnbuffered)
-{
-  // Test that Terminate works during an unbuffered read.
-  for (unsigned i = 0; i < 9; ++i) {
-    Maybe<TerminalState> result =
-      mLexer.Lex(mData + i, 1, DoLexWithUnbufferedTerminate);
-
-    if (i > 2) {
-      EXPECT_TRUE(result.isSome());
-      EXPECT_EQ(Some(TerminalState::SUCCESS), result);
-    } else {
-      EXPECT_TRUE(result.isNothing());
-    }
-  }
-}
-
 TEST_F(ImageStreamingLexer, TerminateUnbufferedFromSourceBuffer)
 {
   // Test that Terminate works during an unbuffered read.
   for (unsigned i = 0; i < 9; ++i) {
     mSourceBuffer->Append(mData + i, 1);
     Maybe<TerminalState> result =
       mLexer.Lex(mIterator, mCountResumes, DoLexWithUnbufferedTerminate);