Bug 1204596 - Part 2: Update the schema of the DOM Cache database to remove the response_redirected and response_redirected_url columns; r=bkelly
☠☠ backed out by 004e30faaea8 ☠ ☠
authorEhsan Akhgari <ehsan@mozilla.com>
Tue, 15 Sep 2015 17:50:09 -0400
changeset 295542 cfc4c4ecbbf520970834fe7cd6e2c017bcd86852
parent 295541 d10c6f32ce461a3aef76868eaa78de009145657d
child 295543 2662a1ad4cad84a6209c8116ad714d6a0c1bb302
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbkelly
bugs1204596
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1204596 - Part 2: Update the schema of the DOM Cache database to remove the response_redirected and response_redirected_url columns; r=bkelly
dom/cache/DBSchema.cpp
dom/cache/test/xpcshell/test_migration.js
--- a/dom/cache/DBSchema.cpp
+++ b/dom/cache/DBSchema.cpp
@@ -32,17 +32,17 @@ namespace dom {
 namespace cache {
 namespace db {
 
 const int32_t kFirstShippedSchemaVersion = 15;
 
 namespace {
 
 // Update this whenever the DB schema is changed.
-const int32_t kLatestSchemaVersion = 16;
+const int32_t kLatestSchemaVersion = 17;
 
 // ---------
 // The following constants define the SQL schema.  These are defined in the
 // same order the SQL should be executed in CreateOrMigrateSchema().  They are
 // broken out as constants for convenient use in validation and migration.
 // ---------
 
 // The caches table is the single source of truth about what Cache
@@ -97,20 +97,16 @@ const char* const kTableEntries =
     "response_type INTEGER NOT NULL, "
     "response_url TEXT NOT NULL, "
     "response_status INTEGER NOT NULL, "
     "response_status_text TEXT NOT NULL, "
     "response_headers_guard INTEGER NOT NULL, "
     "response_body_id TEXT NULL, "
     "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
     "response_principal_info TEXT NOT NULL, "
-    "response_redirected INTEGER NOT NULL, "
-    // Note that response_redirected_url is either going to be empty, or
-    // it's going to be a URL different than response_url.
-    "response_redirected_url TEXT NOT NULL, "
     "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "
 
     // New columns must be added at the end of table to migrate and
     // validate properly.
     "request_redirect INTEGER NOT NULL"
   ")";
 
 // Create an index to support the QueryCache() matching algorithm.  This
@@ -337,16 +333,61 @@ static nsresult CreateAndBindKeyStatemen
                                           const nsAString& aKey,
                                           mozIStorageStatement** aStateOut);
 static nsresult HashCString(nsICryptoHash* aCrypto, const nsACString& aIn,
                             nsACString& aOut);
 nsresult Validate(mozIStorageConnection* aConn);
 nsresult Migrate(mozIStorageConnection* aConn);
 } // namespace
 
+class MOZ_RAII AutoDisableForeignKeyChecking
+{
+public:
+  explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)
+    : mConn(aConn)
+    , mForeignKeyCheckingDisabled(false)
+  {
+    nsCOMPtr<mozIStorageStatement> state;
+    nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA foreign_keys;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    bool hasMoreData = false;
+    rv = state->ExecuteStep(&hasMoreData);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    int32_t mode;
+    rv = state->GetInt32(0, &mode);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+
+    if (mode) {
+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA foreign_keys = OFF;"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+      mForeignKeyCheckingDisabled = true;
+    }
+  }
+
+  ~AutoDisableForeignKeyChecking()
+  {
+    if (mForeignKeyCheckingDisabled) {
+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA foreign_keys = ON;"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
+    }
+  }
+
+private:
+  nsCOMPtr<mozIStorageConnection> mConn;
+  bool mForeignKeyCheckingDisabled;
+};
+
 nsresult
 CreateOrMigrateSchema(mozIStorageConnection* aConn)
 {
   MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConn);
 
   int32_t schemaVersion;
   nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
@@ -356,31 +397,33 @@ CreateOrMigrateSchema(mozIStorageConnect
     // We already have the correct schema version.  Validate it matches
     // our expected schema and then proceed.
     rv = Validate(aConn);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     return rv;
   }
 
+  // Turn off checking foreign keys before starting a transaction, and restore
+  // it once we're done.
+  AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn);
   mozStorageTransaction trans(aConn, false,
                               mozIStorageConnection::TRANSACTION_IMMEDIATE);
   bool needVacuum = false;
 
   if (schemaVersion) {
     // A schema exists, but its not the current version.  Attempt to
     // migrate it to our new schema.
     rv = Migrate(aConn);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     // Migrations happen infrequently and reflect a chance in DB structure.
     // This is a good time to rebuild the database.  It also helps catch
     // if a new migration is incorrect by fast failing on the corruption.
     needVacuum = true;
-
   } else {
     // There is no schema installed.  Create the database from scratch.
     rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableCaches));
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
     rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableSecurityInfo));
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
@@ -2360,20 +2403,22 @@ struct Migration
   { }
   int32_t mFromVersion;
   MigrationFunc mFunc;
 };
 
 // Declare migration functions here.  Each function should upgrade
 // the version by a single increment.  Don't skip versions.
 nsresult MigrateFrom15To16(mozIStorageConnection* aConn);
+nsresult MigrateFrom16To17(mozIStorageConnection* aConn);
 
 // Configure migration functions to run for the given starting version.
 Migration sMigrationList[] = {
   Migration(15, MigrateFrom15To16),
+  Migration(16, MigrateFrom16To17),
 };
 
 uint32_t sMigrationListLength = sizeof(sMigrationList) / sizeof(Migration);
 
 nsresult
 Migrate(mozIStorageConnection* aConn)
 {
   MOZ_ASSERT(!NS_IsMainThread());
@@ -2403,36 +2448,20 @@ Migrate(mozIStorageConnection* aConn)
     MOZ_ASSERT(currentVersion > lastVersion);
   }
 
   MOZ_ASSERT(currentVersion == kLatestSchemaVersion);
 
   return rv;
 }
 
-nsresult MigrateFrom15To16(mozIStorageConnection* aConn)
+nsresult
+RewriteEntriesSchema(mozIStorageConnection* aConn)
 {
-  MOZ_ASSERT(!NS_IsMainThread());
-  MOZ_ASSERT(aConn);
-
-  // Add the request_redirect column with a default value of "follow".  Note,
-  // we only use a default value here because its required by ALTER TABLE and
-  // we need to apply the default "follow" to existing records in the table.
-  // We don't actually want to keep the default in the schema for future
-  // INSERTs.
   nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
-    "ALTER TABLE entries "
-    "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0"
-  ));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  // Now overwrite the master SQL for the entries table to remove the column
-  // default value.  This is also necessary for our Validate() method to
-  // pass on this database.
-  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
     "PRAGMA writable_schema = ON"
   ));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   nsCOMPtr<mozIStorageStatement> state;
   rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
     "UPDATE sqlite_master SET sql=:sql WHERE name='entries'"
   ), getter_AddRefs(state));
@@ -2440,25 +2469,191 @@ nsresult MigrateFrom15To16(mozIStorageCo
 
   rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("sql"),
                                    nsDependentCString(kTableEntries));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = state->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-  rv = aConn->SetSchemaVersion(16);
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
   rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
     "PRAGMA writable_schema = OFF"
   ));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   return rv;
 }
 
+nsresult MigrateFrom15To16(mozIStorageConnection* aConn)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  mozStorageTransaction trans(aConn, true,
+                              mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+  // Add the request_redirect column with a default value of "follow".  Note,
+  // we only use a default value here because its required by ALTER TABLE and
+  // we need to apply the default "follow" to existing records in the table.
+  // We don't actually want to keep the default in the schema for future
+  // INSERTs.
+  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "ALTER TABLE entries "
+    "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Now overwrite the master SQL for the entries table to remove the column
+  // default value.  This is also necessary for our Validate() method to
+  // pass on this database.
+  rv = RewriteEntriesSchema(aConn);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->SetSchemaVersion(16);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return rv;
+}
+
+nsresult
+MigrateFrom16To17(mozIStorageConnection* aConn)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  // This migration path removes the response_redirected and
+  // response_redirected_url columns from the entries table.  sqlite doesn't
+  // support removing a column from a table using ALTER TABLE, so we need to
+  // create a new table without those columns, fill it up with the existing
+  // data, and then drop the original table and rename the new one to the old
+  // one.
+
+  // Create a new_entries table with the new fields as of version 17.
+  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "CREATE TABLE new_entries ("
+      "id INTEGER NOT NULL PRIMARY KEY, "
+      "request_method TEXT NOT NULL, "
+      "request_url_no_query TEXT NOT NULL, "
+      "request_url_no_query_hash BLOB NOT NULL, "
+      "request_url_query TEXT NOT NULL, "
+      "request_url_query_hash BLOB NOT NULL, "
+      "request_referrer TEXT NOT NULL, "
+      "request_headers_guard INTEGER NOT NULL, "
+      "request_mode INTEGER NOT NULL, "
+      "request_credentials INTEGER NOT NULL, "
+      "request_contentpolicytype INTEGER NOT NULL, "
+      "request_cache INTEGER NOT NULL, "
+      "request_body_id TEXT NULL, "
+      "response_type INTEGER NOT NULL, "
+      "response_url TEXT NOT NULL, "
+      "response_status INTEGER NOT NULL, "
+      "response_status_text TEXT NOT NULL, "
+      "response_headers_guard INTEGER NOT NULL, "
+      "response_body_id TEXT NULL, "
+      "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
+      "response_principal_info TEXT NOT NULL, "
+      "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "
+      "request_redirect INTEGER NOT NULL"
+    ")"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Copy all of the data to the newly created table.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "INSERT INTO new_entries ("
+      "id, "
+      "request_method, "
+      "request_url_no_query, "
+      "request_url_no_query_hash, "
+      "request_url_query, "
+      "request_url_query_hash, "
+      "request_referrer, "
+      "request_headers_guard, "
+      "request_mode, "
+      "request_credentials, "
+      "request_contentpolicytype, "
+      "request_cache, "
+      "request_redirect, "
+      "request_body_id, "
+      "response_type, "
+      "response_url, "
+      "response_status, "
+      "response_status_text, "
+      "response_headers_guard, "
+      "response_body_id, "
+      "response_security_info_id, "
+      "response_principal_info, "
+      "cache_id "
+    ") SELECT "
+      "id, "
+      "request_method, "
+      "request_url_no_query, "
+      "request_url_no_query_hash, "
+      "request_url_query, "
+      "request_url_query_hash, "
+      "request_referrer, "
+      "request_headers_guard, "
+      "request_mode, "
+      "request_credentials, "
+      "request_contentpolicytype, "
+      "request_cache, "
+      "request_redirect, "
+      "request_body_id, "
+      "response_type, "
+      "response_url, "
+      "response_status, "
+      "response_status_text, "
+      "response_headers_guard, "
+      "response_body_id, "
+      "response_security_info_id, "
+      "response_principal_info, "
+      "cache_id "
+    "FROM entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Remove the old table.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "DROP TABLE entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Rename new_entries to entries.
+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "ALTER TABLE new_entries RENAME to entries;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Now, recreate our indices.
+  rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Revalidate the foreign key constraints, and ensure that there are no
+  // violations.
+  nsCOMPtr<mozIStorageStatement> state;
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "PRAGMA foreign_key_check;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  if (NS_WARN_IF(hasMoreData)) { return NS_ERROR_FAILURE; }
+
+  // Finally, rewrite the schema for the entries database, otherwise the
+  // returned SQL string from sqlite will wrap the name of the table in quotes,
+  // breaking the checks in Validate().
+  rv = RewriteEntriesSchema(aConn);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->SetSchemaVersion(17);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return rv;
+}
+
 } // anonymous namespace
 
 } // namespace db
 } // namespace cache
 } // namespace dom
 } // namespace mozilla
--- a/dom/cache/test/xpcshell/test_migration.js
+++ b/dom/cache/test/xpcshell/test_migration.js
@@ -22,17 +22,19 @@ function run_test() {
       ok(request.redirect === 'follow', 'request.redirect should default to "follow"');
     });
     return Promise.all(requestList.map(function(request) {
       return cache.match(request);
     }));
   }).then(function(responseList) {
     ok(responseList.length > 0, 'should have at least one response in cache');
     responseList.forEach(function(response) {
-      ok(response, 'each request in list should be non-null');
+      ok(response, 'each response in list should be non-null');
+      do_check_eq(response.headers.get('Content-Type'), 'text/plain;charset=UTF-8',
+                  'the response should have the correct header');
     });
   }).then(function() {
     do_test_finished();
   }).catch(function(e) {
     ok(false, 'caught exception ' + e);
     do_test_finished();
   });
 }