Bug 735083 - batch inserts into Fennec history provider. r=rnewman
authorNick Alexander <nalexander@mozilla.com>
Thu, 05 Apr 2012 19:25:34 -0700
changeset 94456 72ae9117ba882773b1ce4588810c84a701fd3a30
parent 94455 d966f9a5e4f27862bfea69d624ec6bfe43edc34b
child 94457 c527278f647a213cae9dac90b35ad1ddd2b82589
push id886
push userlsblakk@mozilla.com
push dateMon, 04 Jun 2012 19:57:52 +0000
treeherdermozilla-beta@bbd8d5efd6d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrnewman
bugs735083
milestone14.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 735083 - batch inserts into Fennec history provider. r=rnewman
mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataAccessor.java
mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataExtender.java
mobile/android/base/sync/repositories/android/AndroidBrowserHistoryRepositorySession.java
mobile/android/base/sync/repositories/android/AndroidBrowserRepositoryDataAccessor.java
--- a/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataAccessor.java
+++ b/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataAccessor.java
@@ -1,23 +1,29 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.sync.Logger;
+import org.mozilla.gecko.sync.repositories.NullCursorException;
 import org.mozilla.gecko.sync.repositories.domain.HistoryRecord;
 import org.mozilla.gecko.sync.repositories.domain.Record;
 
 import android.content.ContentValues;
 import android.content.Context;
+import android.database.Cursor;
 import android.net.Uri;
 
 public class AndroidBrowserHistoryDataAccessor extends
     AndroidBrowserRepositoryDataAccessor {
 
   private AndroidBrowserHistoryDataExtender dataExtender;
 
   public AndroidBrowserHistoryDataAccessor(Context context) {
@@ -88,9 +94,92 @@ public class AndroidBrowserHistoryDataAc
     Logger.debug(LOG_TAG, "Purging record with " + guid);
     dataExtender.delete(guid);
     return super.purgeGuid(guid);
   }
 
   public void closeExtender() {
     dataExtender.close();
   }
+
+  public static String[] GUID_AND_ID = new String[] { BrowserContract.History.GUID, BrowserContract.History._ID };
+
+  /**
+   * Insert records.
+   * <p>
+   * This inserts all the records (using <code>ContentProvider.bulkInsert</code>),
+   * then inserts all the visit information (using the data extender's
+   * <code>bulkInsert</code>, which internally uses a single database
+   * transaction), and then optionally updates the <code>androidID</code> of
+   * each record.
+   *
+   * @param records
+   *          The records to insert.
+   * @param fetchFreshAndroidIDs
+   *          <code>true</code> to update the <code>androidID</code> of each
+   *          record; <code>false</code> to invalidate them all.
+   * @throws NullCursorException
+   */
+  public void bulkInsert(ArrayList<HistoryRecord> records, boolean fetchFreshAndroidIDs) throws NullCursorException {
+    if (records.isEmpty()) {
+      Logger.debug(LOG_TAG, "No records to insert, returning.");
+    }
+
+    int size = records.size();
+    ContentValues[] cvs = new ContentValues[size];
+    String[] guids = new String[size];
+    Map<String, Record> guidToRecord = new HashMap<String, Record>();
+    int index = 0;
+    for (Record record : records) {
+      if (record.guid == null) {
+        throw new IllegalArgumentException("Record with null GUID passed in to bulkInsert.");
+      }
+      cvs[index] = getContentValues(record);
+      guids[index] = record.guid;
+      guidToRecord.put(record.guid, record);
+      index += 1;
+    }
+
+    // First update the history records.
+    int inserted = context.getContentResolver().bulkInsert(getUri(), cvs);
+    if (inserted == size) {
+      Logger.debug(LOG_TAG, "Inserted " + inserted + " records, as expected.");
+    } else {
+      Logger.debug(LOG_TAG, "Inserted " +
+                   inserted + " records but expected " +
+                   size     + " records; continuing to update visits.");
+    }
+    // Then update the history visits.
+    dataExtender.bulkInsert(records);
+
+    // And finally patch up the androidIDs.
+    if (!fetchFreshAndroidIDs) {
+      return;
+    }
+
+    // We do this here to save a few loops.
+    String guidIn = RepoUtils.computeSQLInClause(guids.length, BrowserContract.History.GUID);
+    Cursor cursor = queryHelper.safeQuery("", GUID_AND_ID, guidIn, guids, null);
+    int guidIndex = cursor.getColumnIndexOrThrow(BrowserContract.History.GUID);
+    int androidIDIndex = cursor.getColumnIndexOrThrow(BrowserContract.History._ID);
+
+    try {
+      cursor.moveToFirst();
+      while (!cursor.isAfterLast()) {
+        String guid = cursor.getString(guidIndex);
+        int androidID = cursor.getInt(androidIDIndex);
+        cursor.moveToNext();
+
+        Record record = guidToRecord.get(guid);
+        if (record == null) {
+          // Should never happen!
+          Logger.warn(LOG_TAG, "Failed to update androidID for record with guid " + guid + ".");
+          continue;
+        }
+        record.androidID = androidID;
+      }
+    } finally {
+      if (cursor != null) {
+        cursor.close();
+      }
+    }
+  }
 }
--- a/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataExtender.java
+++ b/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataExtender.java
@@ -1,55 +1,25 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Original Code is Android Sync Client.
- *
- * The Initial Developer of the Original Code is
- * the Mozilla Foundation.
- * Portions created by the Initial Developer are Copyright (C) 2011
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *   Jason Voll <jvoll@mozilla.com>
- *   Richard Newman <rnewman@mozilla.com>
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+
 import org.json.simple.JSONArray;
 import org.mozilla.gecko.sync.Logger;
 import org.mozilla.gecko.sync.repositories.NullCursorException;
+import org.mozilla.gecko.sync.repositories.domain.HistoryRecord;
 
 import android.content.ContentValues;
 import android.content.Context;
 import android.database.Cursor;
+import android.database.SQLException;
 import android.database.sqlite.SQLiteDatabase;
 
 public class AndroidBrowserHistoryDataExtender extends CachedSQLiteOpenHelper {
 
   public static final String LOG_TAG = "SyncHistoryVisits";
 
   // Database Specifications.
   protected static final String DB_NAME = "history_extension_database";
@@ -114,16 +84,40 @@ public class AndroidBrowserHistoryDataEx
     if (rowsUpdated >= 1) {
       Logger.debug(LOG_TAG, "Replaced history extension record for row with GUID " + guid);
     } else {
       long rowId = db.insert(TBL_HISTORY_EXT, null, cv);
       Logger.debug(LOG_TAG, "Inserted history extension record into row: " + rowId);
     }
   }
 
+  public void bulkInsert(ArrayList<HistoryRecord> records) {
+    SQLiteDatabase db = this.getCachedWritableDatabase();
+    try {
+      db.beginTransaction();
+
+      for (HistoryRecord record : records) {
+        ContentValues cv = new ContentValues();
+        cv.put(COL_GUID, record.guid);
+        if (record.visits == null) {
+          cv.put(COL_VISITS, "[]");
+        } else {
+          cv.put(COL_VISITS, record.visits.toJSONString());
+        }
+        db.insert(TBL_HISTORY_EXT, null, cv);
+      }
+
+      db.setTransactionSuccessful();
+    } catch (SQLException e) {
+      Logger.error(LOG_TAG, "Caught exception in bulkInsert new history visits.", e);
+    } finally {
+      db.endTransaction();
+    }
+  }
+
   /**
    * Fetch a row.
    *
    * @param guid The GUID of the row to fetch.
    * @return A Cursor.
    * @throws NullCursorException
    */
   public Cursor fetch(String guid) throws NullCursorException {
--- a/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryRepositorySession.java
+++ b/mobile/android/base/sync/repositories/android/AndroidBrowserHistoryRepositorySession.java
@@ -1,36 +1,47 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.mozilla.gecko.db.BrowserContract;
+import org.mozilla.gecko.sync.Logger;
 import org.mozilla.gecko.sync.repositories.InactiveSessionException;
 import org.mozilla.gecko.sync.repositories.InvalidSessionTransitionException;
+import org.mozilla.gecko.sync.repositories.NoGuidForIdException;
 import org.mozilla.gecko.sync.repositories.NullCursorException;
+import org.mozilla.gecko.sync.repositories.ParentNotFoundException;
 import org.mozilla.gecko.sync.repositories.Repository;
 import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionBeginDelegate;
 import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFinishDelegate;
 import org.mozilla.gecko.sync.repositories.domain.HistoryRecord;
 import org.mozilla.gecko.sync.repositories.domain.Record;
 
 import android.content.Context;
 import android.database.Cursor;
 import android.util.Log;
 
 public class AndroidBrowserHistoryRepositorySession extends AndroidBrowserRepositorySession {
-  
+  public static final String LOG_TAG = "ABHistoryRepoSess";
+
   public static final String KEY_DATE = "date";
   public static final String KEY_TYPE = "type";
   public static final long DEFAULT_VISIT_TYPE = 1;
 
+  /**
+   * The number of records to queue for insertion before writing to databases.
+   */
+  public static int INSERT_RECORD_THRESHOLD = 50;
+
   public AndroidBrowserHistoryRepositorySession(Repository repository, Context context) {
     super(repository);
     dbHelper = new AndroidBrowserHistoryDataAccessor(context);
   }
 
   @Override
   public void begin(RepositorySessionBeginDelegate delegate) throws InvalidSessionTransitionException {
     // HACK: Fennec creates history records without a GUID. Mercilessly drop
@@ -108,21 +119,102 @@ public class AndroidBrowserHistoryReposi
     hist.visits = visitsArray;
     return hist;
   }
 
   @Override
   protected Record prepareRecord(Record record) {
     return record;
   }
-  
+
   @Override
   public void abort() {
     ((AndroidBrowserHistoryDataAccessor) dbHelper).closeExtender();
     super.abort();
   }
 
   @Override
   public void finish(final RepositorySessionFinishDelegate delegate) throws InactiveSessionException {
     ((AndroidBrowserHistoryDataAccessor) dbHelper).closeExtender();
     super.finish(delegate);
   }
-}
+
+  protected Object recordsBufferMonitor = new Object();
+  protected ArrayList<HistoryRecord> recordsBuffer = new ArrayList<HistoryRecord>();
+
+  /**
+   * Queue record for insertion, possibly flushing the queue.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread! But this is only
+   * called from <code>store</code>, which is called on the queue thread.
+   *
+   * @param record
+   *          A <code>Record</code> with a GUID that is not present locally.
+   * @return The <code>Record</code> to be inserted. <b>Warning:</b> the
+   *         <code>androidID</code> is not valid! It will be set after the
+   *         records are flushed to the database.
+   */
+  @Override
+  protected Record insert(Record record) throws NoGuidForIdException, NullCursorException, ParentNotFoundException {
+    HistoryRecord toStore = (HistoryRecord) prepareRecord(record);
+    toStore.androidID = -111; // Hopefully this special value will make it easy to catch future errors.
+    updateBookkeeping(toStore); // Does not use androidID -- just GUID -> String map.
+    enqueueNewRecord(toStore);
+    return toStore;
+  }
+
+  /**
+   * Batch incoming records until some reasonable threshold is hit or storeDone
+   * is received.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread!
+   *
+   * @param record A <code>Record</code> with a GUID that is not present locally.
+   * @throws NullCursorException
+   */
+  protected void enqueueNewRecord(HistoryRecord record) throws NullCursorException {
+    synchronized (recordsBufferMonitor) {
+      if (recordsBuffer.size() >= INSERT_RECORD_THRESHOLD) {
+        flushNewRecords();
+      }
+      Logger.debug(LOG_TAG, "Enqueuing new record with GUID " + record.guid);
+      recordsBuffer.add(record);
+    }
+  }
+
+  /**
+   * Flush queue of incoming records to database.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread!
+   * <p>
+   * Must be locked by recordsBufferMonitor!
+   * @throws NullCursorException
+   */
+  protected void flushNewRecords() throws NullCursorException {
+    if (recordsBuffer.size() < 1) {
+      Logger.debug(LOG_TAG, "No records to flush, returning.");
+      return;
+    }
+
+    final ArrayList<HistoryRecord> outgoing = recordsBuffer;
+    recordsBuffer = new ArrayList<HistoryRecord>();
+    Logger.debug(LOG_TAG, "Flushing " + outgoing.size() + " records to database.");
+    // TODO: move bulkInsert to AndroidBrowserDataAccessor?
+    ((AndroidBrowserHistoryDataAccessor) dbHelper).bulkInsert(outgoing, false); // Don't need to update any androidIDs.
+  }
+
+  @Override
+  public void storeDone() {
+    storeWorkQueue.execute(new Runnable() {
+      @Override
+      public void run() {
+        synchronized (recordsBufferMonitor) {
+          try {
+            flushNewRecords();
+          } catch (NullCursorException e) {
+            Logger.warn(LOG_TAG, "Error flushing records to database.", e);
+          }
+        }
+        storeDone(System.currentTimeMillis());
+      }
+    });
+  }
+}
\ No newline at end of file
--- a/mobile/android/base/sync/repositories/android/AndroidBrowserRepositoryDataAccessor.java
+++ b/mobile/android/base/sync/repositories/android/AndroidBrowserRepositoryDataAccessor.java
@@ -14,17 +14,17 @@ import android.content.Context;
 import android.database.Cursor;
 import android.net.Uri;
 
 public abstract class AndroidBrowserRepositoryDataAccessor {
 
   private static final String[] GUID_COLUMNS = new String[] { BrowserContract.SyncColumns.GUID };
   protected Context context;
   protected static String LOG_TAG = "BrowserDataAccessor";
-  private final RepoUtils.QueryHelper queryHelper;
+  protected final RepoUtils.QueryHelper queryHelper;
 
   public AndroidBrowserRepositoryDataAccessor(Context context) {
     this.context = context;
     this.queryHelper = new RepoUtils.QueryHelper(context, getUri(), LOG_TAG);
   }
 
   protected abstract String[] getAllColumns();