Bug 1270882 - Enable support for SQLite custom FTS3 tokenizers at run time. r=mak
authorArfrever Frehtes Taifersar Arahesis <Arfrever@Apache.Org>
Thu, 29 Nov 2018 23:02:10 +0000
changeset 510003 c69cf0bb19cd8d0814011546f5e3e2e6185eb0cd
parent 510002 d3d18e1a63e346bc4e8cb6aba3a59bf9a28181ee
child 510004 19051af76b77ea0dc8e433b5e95d4714f35808d0
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmak
bugs1270882
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1270882 - Enable support for SQLite custom FTS3 tokenizers at run time. r=mak Do not require that SQLite has been built with support for custom FTS3 tokenizers enabled by default. This allows to use system SQLite in distributions which provide SQLite configured in this way (which is SQLite upstream's default configuration due to security concerns). Requires exposing the sqlite3_db_config symbol in bundled SQLite. Disable no longer needed setting of SQLITE_ENABLE_FTS3_TOKENIZER macro in bundled SQLite build.
db/sqlite3/src/moz.build
db/sqlite3/src/sqlite.symbols
storage/moz.build
storage/mozStorageConnection.cpp
--- a/db/sqlite3/src/moz.build
+++ b/db/sqlite3/src/moz.build
@@ -53,20 +53,16 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'wind
 # -DSQLITE_ENABLE_LOCKING_STYLE=1 to help with AFP folders
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa':
     DEFINES['SQLITE_ENABLE_LOCKING_STYLE'] = 1
 
 # sqlite defaults this to on on __APPLE_ but it breaks on newer iOS SDKs
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'uikit':
     DEFINES['SQLITE_ENABLE_LOCKING_STYLE'] = 0
 
-# Thunderbird needs the 2-argument version of fts3_tokenizer()
-if CONFIG['MOZ_THUNDERBIRD'] or CONFIG['MOZ_SUITE']:
-    DEFINES['SQLITE_ENABLE_FTS3_TOKENIZER'] = 1
-
 # Turn on SQLite's assertions in debug builds.
 if CONFIG['MOZ_DEBUG']:
     DEFINES['SQLITE_DEBUG'] = 1
     DEFINES['SQLITE_ENABLE_API_ARMOR'] = True
 
 if CONFIG['OS_TARGET'] == 'Android':
     # default to user readable only to fit Android security model
     DEFINES['SQLITE_DEFAULT_FILE_PERMISSIONS'] = '0600'
--- a/db/sqlite3/src/sqlite.symbols
+++ b/db/sqlite3/src/sqlite.symbols
@@ -40,16 +40,17 @@ sqlite3_complete
 sqlite3_complete16
 sqlite3_config
 sqlite3_create_collation
 sqlite3_create_collation16
 sqlite3_create_function
 sqlite3_create_function16
 sqlite3_create_module
 sqlite3_data_count
+sqlite3_db_config
 sqlite3_db_filename
 sqlite3_db_handle
 sqlite3_db_mutex
 sqlite3_db_status
 sqlite3_declare_vtab
 sqlite3_enable_load_extension
 sqlite3_enable_shared_cache
 sqlite3_errcode
--- a/storage/moz.build
+++ b/storage/moz.build
@@ -101,16 +101,20 @@ FINAL_LIBRARY = 'xul'
 #
 # Note: On Windows our sqlite build assumes we use jemalloc.  If you disable
 # MOZ_STORAGE_MEMORY on Windows, you will also need to change the "ifdef
 # MOZ_MEMORY" options in db/sqlite3/src/Makefile.in.
 if CONFIG['MOZ_MEMORY'] and not CONFIG['MOZ_SYSTEM_SQLITE']:
     if CONFIG['OS_TARGET'] != 'Android':
         DEFINES['MOZ_STORAGE_MEMORY'] = True
 
+# Thunderbird needs the 2-argument version of fts3_tokenizer()
+if CONFIG['MOZ_THUNDERBIRD'] or CONFIG['MOZ_SUITE']:
+    DEFINES['MOZ_SQLITE_FTS3_TOKENIZER'] = 1
+
 # This is the default value.  If we ever change it when compiling sqlite, we
 # will need to change it here as well.
 DEFINES['SQLITE_MAX_LIKE_PATTERN_LENGTH'] = 50000
 
 # See Sqlite moz.build for reasoning about TEMP_STORE.
 # For system sqlite we cannot use the compile time option, so we use a pragma.
 if CONFIG['MOZ_SYSTEM_SQLITE'] and (CONFIG['OS_TARGET'] == 'Android'
                                     or CONFIG['HAVE_64BIT_BUILD']):
--- a/storage/mozStorageConnection.cpp
+++ b/storage/mozStorageConnection.cpp
@@ -597,16 +597,21 @@ nsresult Connection::initialize() {
 
   // in memory database requested, sqlite uses a magic file name
   int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, GetVFSName());
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
+#ifdef MOZ_SQLITE_FTS3_TOKENIZER
+  srv = ::sqlite3_db_config(mDBConn, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
+  MOZ_ASSERT(srv == SQLITE_OK, "SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER should be enabled");
+#endif
+
   // Do not set mDatabaseFile or mFileURL here since this is a "memory"
   // database.
 
   nsresult rv = initializeInternal();
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
 }
@@ -631,16 +636,21 @@ nsresult Connection::initialize(nsIFile 
 
   int srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn,
                               mFlags, vfs);
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
+#ifdef MOZ_SQLITE_FTS3_TOKENIZER
+  srv = ::sqlite3_db_config(mDBConn, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
+  MOZ_ASSERT(srv == SQLITE_OK, "SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER should be enabled");
+#endif
+
   // Do not set mFileURL here since this is database does not have an associated
   // URL.
   mDatabaseFile = aDatabaseFile;
 
   rv = initializeInternal();
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
@@ -660,16 +670,21 @@ nsresult Connection::initialize(nsIFileU
   NS_ENSURE_SUCCESS(rv, rv);
 
   int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, GetVFSName());
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
+#ifdef MOZ_SQLITE_FTS3_TOKENIZER
+  srv = ::sqlite3_db_config(mDBConn, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
+  MOZ_ASSERT(srv == SQLITE_OK, "SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER should be enabled");
+#endif
+
   // Set both mDatabaseFile and mFileURL here.
   mFileURL = aFileURL;
   mDatabaseFile = databaseFile;
 
   rv = initializeInternal();
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;