Bug 1204596 - Part 2: Update the schema of the DOM Cache database to remove the response_redirected and response_redirected_url columns; r=bkelly
☠☠ backed out by 83e1183fff66 ☠ ☠
authorEhsan Akhgari <ehsan@mozilla.com>
Tue, 15 Sep 2015 17:50:09 -0400
changeset 296279 daa0e457650e98f7fd9060eab3df0430f26820e1
parent 296278 33a968e122bc207129970f57ea11a45127f0bd78
child 296280 35b2edcb498b63f52124aeb792da395adb188314
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbkelly
bugs1204596
milestone43.0a2
Bug 1204596 - Part 2: Update the schema of the DOM Cache database to remove the response_redirected and response_redirected_url columns; r=bkelly
dom/cache/DBSchema.cpp
dom/cache/test/xpcshell/test_migration.js
--- a/dom/cache/DBSchema.cpp
+++ b/dom/cache/DBSchema.cpp
@@ -32,17 +32,17 @@ namespace dom {
 namespace cache {
 namespace db {
 
 const int32_t kFirstShippedSchemaVersion = 15;
 
 namespace {
 
 // Update this whenever the DB schema is changed.
-const int32_t kLatestSchemaVersion = 16;
+const int32_t kLatestSchemaVersion = 17;
 
 // ---------
 // The following constants define the SQL schema.  These are defined in the
 // same order the SQL should be executed in CreateOrMigrateSchema().  They are
 // broken out as constants for convenient use in validation and migration.
 // ---------
 
 // The caches table is the single source of truth about what Cache
@@ -97,20 +97,16 @@ const char* const kTableEntries =
     "response_type INTEGER NOT NULL, "
     "response_url TEXT NOT NULL, "
     "response_status INTEGER NOT NULL, "
     "response_status_text TEXT NOT NULL, "
     "response_headers_guard INTEGER NOT NULL, "
     "response_body_id TEXT NULL, "
     "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
     "response_principal_info TEXT NOT NULL, "
-    "response_redirected INTEGER NOT NULL, "
-    // Note that response_redirected_url is either going to be empty, or
-    // it's going to be a URL different than response_url.
-    "response_redirected_url TEXT NOT NULL, "
     "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "
 
     // New columns must be added at the end of table to migrate and
     // validate properly.
     "request_redirect INTEGER NOT NULL"
   ")";
 
 // Create an index to support the QueryCache() matching algorithm.  This
@@ -337,16 +333,61 @@ static nsresult CreateAndBindKeyStatemen
                                           const nsAString& aKey,
                                           mozIStorageStatement** aStateOut);
 static nsresult HashCString(nsICryptoHash* aCrypto, const nsACString& aIn,
                             nsACString& aOut);
 nsresult Validate(mozIStorageConnection* aConn);
 nsresult Migrate(mozIStorageConnection* aConn);
 } // namespace
 
+class MOZ_RAII AutoDisableForeignKeyChecking
+{
+public:
+  explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)
+    : mConn(aConn)
+    , mForeignKeyCheckingDisabled(false)
+  {
+    nsCOMPtr<mozIStorageStatement> state;
+    nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA foreign_keys;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    bool hasMoreData = false;
+    rv = state->ExecuteStep(&hasMoreData);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    int32_t mode;
+    rv = state->GetInt32(0, &mode);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    if (mode) {
+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA foreign_keys = OFF;"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+      mForeignKeyCheckingDisabled = true;
+    }
+  }
+
+  ~AutoDisableForeignKeyChecking()
+  {
+    if (mForeignKeyCheckingDisabled) {
+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA foreign_keys = ON;"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+    }
+  }
+
+private:
+  nsCOMPtr<mozIStorageConnection> mConn;
+  bool mForeignKeyCheckingDisabled;
+};
+
 nsresult
 CreateOrMigrateSchema(mozIStorageConnection* aConn)
 {
   MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConn);
 
   int32_t schemaVersion;
   nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
@@ -356,31 +397,33 @@ CreateOrMigrateSchema(mozIStorageConnect
     // We already have the correct schema version.  Validate it matches
     // our expected schema and then proceed.
     rv = Validate(aConn);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     return rv;
   }
 
+  // Turn off checking foreign keys before starting a transaction, and restore
+  // it once we're done.
+  AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn);
   mozStorageTransaction trans(aConn, false,
                               mozIStorageConnection::TRANSACTION_IMMEDIATE);
   bool needVacuum = false;
 
   if (schemaVersion) {
     // A schema exists, but its not the current version.  Attempt to
     // migrate it to our new schema.
     rv = Migrate(aConn);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     // Migrations happen infrequently and reflect a chance in DB structure.
     // This is a good time to rebuild the database.  It also helps catch
     // if a new migration is incorrect by fast failing on the corruption.
     needVacuum = true;
-
   } else {
     // There is no schema installed.  Create the database from scratch.
     rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableCaches));
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableSecurityInfo));
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
@@ -2360,20 +2403,22 @@ struct Migration
   { }
   int32_t mFromVersion;
   MigrationFunc mFunc;
 };
 
 // Declare migration functions here.  Each function should upgrade
 // the version by a single increment.  Don't skip versions.
 nsresult MigrateFrom15To16(mozIStorageConnection* aConn);
+nsresult MigrateFrom16To17(mozIStorageConnection* aConn);
 
 // Configure migration functions to run for the given starting version.
 Migration sMigrationList[] = {
   Migration(15, MigrateFrom15To16),
+  Migration(16, MigrateFrom16To17),
 };
 
 uint32_t sMigrationListLength = sizeof(sMigrationList) / sizeof(Migration);
 
 nsresult
 Migrate(mozIStorageConnection* aConn)
 {
   MOZ_ASSERT(!NS_IsMainThread());
@@ -2403,36 +2448,20 @@ Migrate(mozIStorageConnection* aConn)
     MOZ_ASSERT(currentVersion > lastVersion);
   }
 
   MOZ_ASSERT(currentVersion == kLatestSchemaVersion);
 
   return rv;
 }
 
-nsresult MigrateFrom15To16(mozIStorageConnection* aConn)
+nsresult
+RewriteEntriesSchema(mozIStorageConnection* aConn)
 {
-  MOZ_ASSERT(!NS_IsMainThread());
-  MOZ_ASSERT(aConn);
-
-  // Add the request_redirect column with a default value of "follow".  Note,
-  // we only use a default value here because its required by ALTER TABLE and
-  // we need to apply the default "follow" to existing records in the table.
-  // We don't actually want to keep the default in the schema for future
-  // INSERTs.
   nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
-    "ALTER TABLE entries "
-    "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0"
-  ));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  // Now overwrite the master SQL for the entries table to remove the column
-  // default value.  This is also necessary for our Validate() method to
-  // pass on this database.
-  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
     "PRAGMA writable_schema = ON"
   ));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   nsCOMPtr<mozIStorageStatement> state;
   rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
     "UPDATE sqlite_master SET sql=:sql WHERE name='entries'"
   ), getter_AddRefs(state));
@@ -2440,25 +2469,191 @@ nsresult MigrateFrom15To16(mozIStorageCo
 
   rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("sql"),
                                    nsDependentCString(kTableEntries));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = state->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-  rv = aConn->SetSchemaVersion(16);
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
   rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
     "PRAGMA writable_schema = OFF"
   ));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   return rv;
 }
 
+nsresult MigrateFrom15To16(mozIStorageConnection* aConn)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  mozStorageTransaction trans(aConn, true,
+                              mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+  // Add the request_redirect column with a default value of "follow".  Note,
+  // we only use a default value here because its required by ALTER TABLE and
+  // we need to apply the default "follow" to existing records in the table.
+  // We don't actually want to keep the default in the schema for future
+  // INSERTs.
+  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "ALTER TABLE entries "
+    "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Now overwrite the master SQL for the entries table to remove the column
+  // default value.  This is also necessary for our Validate() method to
+  // pass on this database.
+  rv = RewriteEntriesSchema(aConn);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->SetSchemaVersion(16);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return rv;
+}
+
+nsresult
+MigrateFrom16To17(mozIStorageConnection* aConn)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  // This migration path removes the response_redirected and
+  // response_redirected_url columns from the entries table.  sqlite doesn't
+  // support removing a column from a table using ALTER TABLE, so we need to
+  // create a new table without those columns, fill it up with the existing
+  // data, and then drop the original table and rename the new one to the old
+  // one.
+
+  // Create a new_entries table with the new fields as of version 17.
+  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "CREATE TABLE new_entries ("
+      "id INTEGER NOT NULL PRIMARY KEY, "
+      "request_method TEXT NOT NULL, "
+      "request_url_no_query TEXT NOT NULL, "
+      "request_url_no_query_hash BLOB NOT NULL, "
+      "request_url_query TEXT NOT NULL, "
+      "request_url_query_hash BLOB NOT NULL, "
+      "request_referrer TEXT NOT NULL, "
+      "request_headers_guard INTEGER NOT NULL, "
+      "request_mode INTEGER NOT NULL, "
+      "request_credentials INTEGER NOT NULL, "
+      "request_contentpolicytype INTEGER NOT NULL, "
+      "request_cache INTEGER NOT NULL, "
+      "request_body_id TEXT NULL, "
+      "response_type INTEGER NOT NULL, "
+      "response_url TEXT NOT NULL, "
+      "response_status INTEGER NOT NULL, "
+      "response_status_text TEXT NOT NULL, "
+      "response_headers_guard INTEGER NOT NULL, "
+      "response_body_id TEXT NULL, "
+      "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
+      "response_principal_info TEXT NOT NULL, "
+      "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "
+      "request_redirect INTEGER NOT NULL"
+    ")"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Copy all of the data to the newly created table.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "INSERT INTO new_entries ("
+      "id, "
+      "request_method, "
+      "request_url_no_query, "
+      "request_url_no_query_hash, "
+      "request_url_query, "
+      "request_url_query_hash, "
+      "request_referrer, "
+      "request_headers_guard, "
+      "request_mode, "
+      "request_credentials, "
+      "request_contentpolicytype, "
+      "request_cache, "
+      "request_redirect, "
+      "request_body_id, "
+      "response_type, "
+      "response_url, "
+      "response_status, "
+      "response_status_text, "
+      "response_headers_guard, "
+      "response_body_id, "
+      "response_security_info_id, "
+      "response_principal_info, "
+      "cache_id "
+    ") SELECT "
+      "id, "
+      "request_method, "
+      "request_url_no_query, "
+      "request_url_no_query_hash, "
+      "request_url_query, "
+      "request_url_query_hash, "
+      "request_referrer, "
+      "request_headers_guard, "
+      "request_mode, "
+      "request_credentials, "
+      "request_contentpolicytype, "
+      "request_cache, "
+      "request_redirect, "
+      "request_body_id, "
+      "response_type, "
+      "response_url, "
+      "response_status, "
+      "response_status_text, "
+      "response_headers_guard, "
+      "response_body_id, "
+      "response_security_info_id, "
+      "response_principal_info, "
+      "cache_id "
+    "FROM entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Remove the old table.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "DROP TABLE entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Rename new_entries to entries.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "ALTER TABLE new_entries RENAME to entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Now, recreate our indices.
+  rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Revalidate the foreign key constraints, and ensure that there are no
+  // violations.
+  nsCOMPtr<mozIStorageStatement> state;
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "PRAGMA foreign_key_check;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  if (NS_WARN_IF(hasMoreData)) { return NS_ERROR_FAILURE; }
+
+  // Finally, rewrite the schema for the entries database, otherwise the
+  // returned SQL string from sqlite will wrap the name of the table in quotes,
+  // breaking the checks in Validate().
+  rv = RewriteEntriesSchema(aConn);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->SetSchemaVersion(17);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return rv;
+}
+
 } // anonymous namespace
 
 } // namespace db
 } // namespace cache
 } // namespace dom
 } // namespace mozilla
--- a/dom/cache/test/xpcshell/test_migration.js
+++ b/dom/cache/test/xpcshell/test_migration.js
@@ -22,17 +22,19 @@ function run_test() {
       ok(request.redirect === 'follow', 'request.redirect should default to "follow"');
     });
     return Promise.all(requestList.map(function(request) {
       return cache.match(request);
     }));
   }).then(function(responseList) {
     ok(responseList.length > 0, 'should have at least one response in cache');
     responseList.forEach(function(response) {
-      ok(response, 'each request in list should be non-null');
+      ok(response, 'each response in list should be non-null');
+      do_check_eq(response.headers.get('Content-Type'), 'text/plain;charset=UTF-8',
+                  'the response should have the correct header');
     });
   }).then(function() {
     do_test_finished();
   }).catch(function(e) {
     ok(false, 'caught exception ' + e);
     do_test_finished();
   });
 }