Bug 478297 - Upgrade to SQLite 3.6.10
authorShawn Wilsher <me@shawnwilsher.com>
Sun, 15 Feb 2009 11:34:32 -0800
changeset 23894 acc7928d1972344722f3bbbce5557da1c4992f39
parent 23893 3b86a81208cac213760d0994b9294590dbab557d
child 23895 ac1562d96d161a5916c16099b74bd1c3f89bc584
child 23898 042250237c7d909b1b1d596828c6715242c337c2
push id976
push usersdwilsh@shawnwilsher.com
push dateTue, 24 Mar 2009 17:35:26 +0000
bugs478297
milestone1.9.1b4pre
Bug 478297 - Upgrade to SQLite 3.6.10 This changeset contains the changes to SQLite code.
db/sqlite3/src/sqlite3.c
db/sqlite3/src/sqlite3.h
--- a/db/sqlite3/src/sqlite3.c
+++ b/db/sqlite3/src/sqlite3.c
@@ -1,28 +1,28 @@
 /******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version 3.6.7.  By combining all the individual C code files into this 
+** version 3.6.10.  By combining all the individual C code files into this 
 ** single large file, the entire code can be compiled as a one translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
 ** of 5% are more are commonly seen when SQLite is compiled as a single
 ** translation unit.
 **
 ** This file is all you need to compile SQLite.  To use SQLite in other
 ** programs, you need this file and the "sqlite3.h" header file that defines
 ** the programming interface to the SQLite library.  (If you do not have 
 ** the "sqlite3.h" header file at hand, you will find a copy in the first
-** 6735 lines past this header comment.)  Additional code files may be
+** 6736 lines past this header comment.)  Additional code files may be
 ** needed if you want a wrapper to interface SQLite with your choice of
 ** programming language.  The code for the "sqlite3" command-line shell
 ** is also in a separate file.  This file contains only code for the core
 ** SQLite library.
 **
-** This amalgamation was generated on 2008-12-16 18:00:15 UTC.
+** This amalgamation was generated on 2009-01-15 16:00:39 UTC.
 */
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
 #ifndef SQLITE_PRIVATE
 # define SQLITE_PRIVATE static
 #endif
 #ifndef SQLITE_API
 # define SQLITE_API
@@ -36,17 +36,17 @@
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.809 2008/12/10 21:19:57 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.824 2009/01/14 23:03:41 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
 
 /*
 ** Include the configuration header output by 'configure' if we're using the
 ** autoconf-based build
 */
@@ -65,17 +65,17 @@
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** 
 ** This file defines various limits of what SQLite can process.
 **
-** @(#) $Id: sqliteLimit.h,v 1.8 2008/03/26 15:56:22 drh Exp $
+** @(#) $Id: sqliteLimit.h,v 1.10 2009/01/10 16:15:09 danielk1977 Exp $
 */
 
 /*
 ** The maximum length of a TEXT or BLOB in bytes.   This also
 ** limits the size of a row in a table or index.
 **
 ** The hard limit is the ability of a 32-bit signed integer
 ** to count the size: 2^31-1 or 2147483647.
@@ -149,17 +149,17 @@
 #ifndef SQLITE_MAX_VDBE_OP
 # define SQLITE_MAX_VDBE_OP 25000
 #endif
 
 /*
 ** The maximum number of arguments to an SQL function.
 */
 #ifndef SQLITE_MAX_FUNCTION_ARG
-# define SQLITE_MAX_FUNCTION_ARG 100
+# define SQLITE_MAX_FUNCTION_ARG 127
 #endif
 
 /*
 ** The maximum number of in-memory pages to use for the main database
 ** table and for temporary tables.  The SQLITE_DEFAULT_CACHE_SIZE
 */
 #ifndef SQLITE_DEFAULT_CACHE_SIZE
 # define SQLITE_DEFAULT_CACHE_SIZE  2000
@@ -183,16 +183,23 @@
 */
 #ifndef SQLITE_MAX_VARIABLE_NUMBER
 # define SQLITE_MAX_VARIABLE_NUMBER 999
 #endif
 
 /* Maximum page size.  The upper bound on this value is 32768.  This a limit
 ** imposed by the necessity of storing the value in a 2-byte unsigned integer
 ** and the fact that the page size must be a power of 2.
+**
+** If this limit is changed, then the compiled library is technically
+** incompatible with an SQLite library compiled with a different limit. If
+** a process operating on a database with a page-size of 65536 bytes 
+** crashes, then an instance of SQLite compiled with the default page-size 
+** limit will not be able to rollback the aborted transaction. This could
+** lead to database corruption.
 */
 #ifndef SQLITE_MAX_PAGE_SIZE
 # define SQLITE_MAX_PAGE_SIZE 32768
 #endif
 
 
 /*
 ** The default size of a database page.
@@ -263,65 +270,16 @@
 #ifdef HAVE_STDINT_H
 #include <stdint.h>
 #endif
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
 
 /*
-** A macro used to aid in coverage testing.  When doing coverage
-** testing, the condition inside the argument must be evaluated 
-** both true and false in order to get full branch coverage.
-** This macro can be inserted to ensure adequate test coverage
-** in places where simple condition/decision coverage is inadequate.
-*/
-#ifdef SQLITE_COVERAGE_TEST
-SQLITE_PRIVATE   void sqlite3Coverage(int);
-# define testcase(X)  if( X ){ sqlite3Coverage(__LINE__); }
-#else
-# define testcase(X)
-#endif
-
-/*
-** The ALWAYS and NEVER macros surround boolean expressions which 
-** are intended to always be true or false, respectively.  Such
-** expressions could be omitted from the code completely.  But they
-** are included in a few cases in order to enhance the resilience
-** of SQLite to unexpected behavior - to make the code "self-healing"
-** or "ductile" rather than being "brittle" and crashing at the first
-** hint of unplanned behavior.
-**
-** When doing coverage testing ALWAYS and NEVER are hard-coded to
-** be true and false so that the unreachable code then specify will
-** not be counted as untested code.
-*/
-#ifdef SQLITE_COVERAGE_TEST
-# define ALWAYS(X)      (1)
-# define NEVER(X)       (0)
-#else
-# define ALWAYS(X)      (X)
-# define NEVER(X)       (X)
-#endif
-
-/*
-** The macro unlikely() is a hint that surrounds a boolean
-** expression that is usually false.  Macro likely() surrounds
-** a boolean expression that is usually true.  GCC is able to
-** use these hints to generate better code, sometimes.
-*/
-#if defined(__GNUC__) && 0
-# define likely(X)    __builtin_expect((X),1)
-# define unlikely(X)  __builtin_expect((X),0)
-#else
-# define likely(X)    !!(X)
-# define unlikely(X)  !!(X)
-#endif
-
-/*
  * This macro is used to "hide" some ugliness in casting an int
  * value to a ptr value under the MSVC 64-bit compiler.   Casting
  * non 64-bit values to ptr types results in a "hard" error with 
  * the MSVC 64-bit compiler which this attempts to avoid.  
  *
  * A simple compiler pragma or casting sequence could not be found
  * to correct this in all situations, so this macro was introduced.
  *
@@ -448,16 +406,103 @@ SQLITE_PRIVATE   void sqlite3Coverage(in
 ** lines are added to automatically set NDEBUG unless the -DSQLITE_DEBUG=1
 ** option is set.  Thus NDEBUG becomes an opt-in rather than an opt-out
 ** feature.
 */
 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
 # define NDEBUG 1
 #endif
 
+/*
+** The testcase() macro is used to aid in coverage testing.  When 
+** doing coverage testing, the condition inside the argument to
+** testcase() must be evaluated both true and false in order to
+** get full branch coverage.  The testcase() macro is inserted
+** to help ensure adequate test coverage in places where simple
+** condition/decision coverage is inadequate.  For example, testcase()
+** can be used to make sure boundary values are tested.  For
+** bitmask tests, testcase() can be used to make sure each bit
+** is significant and used at least once.  On switch statements
+** where multiple cases go to the same block of code, testcase()
+** can insure that all cases are evaluated.
+**
+*/
+#ifdef SQLITE_COVERAGE_TEST
+SQLITE_PRIVATE   void sqlite3Coverage(int);
+# define testcase(X)  if( X ){ sqlite3Coverage(__LINE__); }
+#else
+# define testcase(X)
+#endif
+
+/*
+** The TESTONLY macro is used to enclose variable declarations or
+** other bits of code that are needed to support the arguments
+** within testcase() and assert() macros.
+*/
+#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST)
+# define TESTONLY(X)  X
+#else
+# define TESTONLY(X)
+#endif
+
+/*
+** The ALWAYS and NEVER macros surround boolean expressions which 
+** are intended to always be true or false, respectively.  Such
+** expressions could be omitted from the code completely.  But they
+** are included in a few cases in order to enhance the resilience
+** of SQLite to unexpected behavior - to make the code "self-healing"
+** or "ductile" rather than being "brittle" and crashing at the first
+** hint of unplanned behavior.
+**
+** In other words, ALWAYS and NEVER are added for defensive code.
+**
+** When doing coverage testing ALWAYS and NEVER are hard-coded to
+** be true and false so that the unreachable code then specify will
+** not be counted as untested code.
+*/
+#if defined(SQLITE_COVERAGE_TEST)
+# define ALWAYS(X)      (1)
+# define NEVER(X)       (0)
+#elif !defined(NDEBUG)
+SQLITE_PRIVATE   int sqlite3Assert(void);
+# define ALWAYS(X)      ((X)?1:sqlite3Assert())
+# define NEVER(X)       ((X)?sqlite3Assert():0)
+#else
+# define ALWAYS(X)      (X)
+# define NEVER(X)       (X)
+#endif
+
+/*
+** The macro unlikely() is a hint that surrounds a boolean
+** expression that is usually false.  Macro likely() surrounds
+** a boolean expression that is usually true.  GCC is able to
+** use these hints to generate better code, sometimes.
+*/
+#if defined(__GNUC__) && 0
+# define likely(X)    __builtin_expect((X),1)
+# define unlikely(X)  __builtin_expect((X),0)
+#else
+# define likely(X)    !!(X)
+# define unlikely(X)  !!(X)
+#endif
+
+/*
+** Sometimes we need a small amount of code such as a variable initialization
+** to setup for a later assert() statement.  We do not want this code to
+** appear when assert() is disabled.  The following macro is therefore
+** used to contain that setup code.  The "VVA" acronym stands for
+** "Verification, Validation, and Accreditation".  In other words, the
+** code within VVA_ONLY() will only run during verification processes.
+*/
+#ifndef NDEBUG
+# define VVA_ONLY(X)  X
+#else
+# define VVA_ONLY(X)
+#endif
+
 /************** Include sqlite3.h in the middle of sqliteInt.h ***************/
 /************** Begin file sqlite3.h *****************************************/
 /*
 ** 2001 September 15
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
@@ -482,17 +527,17 @@ SQLITE_PRIVATE   void sqlite3Coverage(in
 ** from comments in this file.  This file is the authoritative source
 ** on how SQLite interfaces are suppose to operate.
 **
 ** The name of this file under configuration management is "sqlite.h.in".
 ** The makefile makes some minor changes to this file (such as inserting
 ** the version number) and changes its name to "sqlite3.h" as
 ** part of the build process.
 **
-** @(#) $Id: sqlite.h.in,v 1.420 2008/12/16 13:46:30 drh Exp $
+** @(#) $Id: sqlite.h.in,v 1.421 2008/12/30 06:24:58 danielk1977 Exp $
 */
 #ifndef _SQLITE3_H_
 #define _SQLITE3_H_
 #include <stdarg.h>     /* Needed for the definition of va_list */
 
 /*
 ** Make sure we can call this stuff from C++.
 */
@@ -559,18 +604,18 @@ extern "C" {
 ** {H10011} The SQLITE_VERSION #define in the sqlite3.h header file shall
 **          evaluate to a string literal that is the SQLite version
 **          with which the header file is associated.
 **
 ** {H10014} The SQLITE_VERSION_NUMBER #define shall resolve to an integer
 **          with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z
 **          are the major version, minor version, and release number.
 */
-#define SQLITE_VERSION         "3.6.7"
-#define SQLITE_VERSION_NUMBER  3006007
+#define SQLITE_VERSION         "3.6.10"
+#define SQLITE_VERSION_NUMBER  3006010
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
 ** KEYWORDS: sqlite3_version
 **
 ** These features provide the same information as the [SQLITE_VERSION]
 ** and [SQLITE_VERSION_NUMBER] #defines in the header, but are associated
 ** with the library instead of the header file.  Cautious programmers might
@@ -2849,26 +2894,27 @@ SQLITE_API int sqlite3_set_authorizer(
 #define SQLITE_DROP_TEMP_TRIGGER    14   /* Trigger Name    Table Name      */
 #define SQLITE_DROP_TEMP_VIEW       15   /* View Name       NULL            */
 #define SQLITE_DROP_TRIGGER         16   /* Trigger Name    Table Name      */
 #define SQLITE_DROP_VIEW            17   /* View Name       NULL            */
 #define SQLITE_INSERT               18   /* Table Name      NULL            */
 #define SQLITE_PRAGMA               19   /* Pragma Name     1st arg or NULL */
 #define SQLITE_READ                 20   /* Table Name      Column Name     */
 #define SQLITE_SELECT               21   /* NULL            NULL            */
-#define SQLITE_TRANSACTION          22   /* NULL            NULL            */
+#define SQLITE_TRANSACTION          22   /* Operation       NULL            */
 #define SQLITE_UPDATE               23   /* Table Name      Column Name     */
 #define SQLITE_ATTACH               24   /* Filename        NULL            */
 #define SQLITE_DETACH               25   /* Database Name   NULL            */
 #define SQLITE_ALTER_TABLE          26   /* Database Name   Table Name      */
 #define SQLITE_REINDEX              27   /* Index Name      NULL            */
 #define SQLITE_ANALYZE              28   /* Table Name      NULL            */
 #define SQLITE_CREATE_VTABLE        29   /* Table Name      Module Name     */
 #define SQLITE_DROP_VTABLE          30   /* Table Name      Module Name     */
 #define SQLITE_FUNCTION             31   /* NULL            Function Name   */
+#define SQLITE_SAVEPOINT            32   /* Operation       Savepoint Name  */
 #define SQLITE_COPY                  0   /* No longer used */
 
 /*
 ** CAPI3REF: Tracing And Profiling Functions {H12280} <S60400>
 ** EXPERIMENTAL
 **
 ** These routines register callback functions that can be used for
 ** tracing and profiling the execution of SQL statements.
@@ -7294,156 +7340,158 @@ SQLITE_PRIVATE void sqlite3HashClear(Has
 #define TK_BEGIN                           5
 #define TK_TRANSACTION                     6
 #define TK_DEFERRED                        7
 #define TK_IMMEDIATE                       8
 #define TK_EXCLUSIVE                       9
 #define TK_COMMIT                         10
 #define TK_END                            11
 #define TK_ROLLBACK                       12
-#define TK_CREATE                         13
-#define TK_TABLE                          14
-#define TK_IF                             15
-#define TK_NOT                            16
-#define TK_EXISTS                         17
-#define TK_TEMP                           18
-#define TK_LP                             19
-#define TK_RP                             20
-#define TK_AS                             21
-#define TK_COMMA                          22
-#define TK_ID                             23
-#define TK_ABORT                          24
-#define TK_AFTER                          25
-#define TK_ANALYZE                        26
-#define TK_ASC                            27
-#define TK_ATTACH                         28
-#define TK_BEFORE                         29
-#define TK_CASCADE                        30
-#define TK_CAST                           31
-#define TK_CONFLICT                       32
-#define TK_DATABASE                       33
-#define TK_DESC                           34
-#define TK_DETACH                         35
-#define TK_EACH                           36
-#define TK_FAIL                           37
-#define TK_FOR                            38
-#define TK_IGNORE                         39
-#define TK_INITIALLY                      40
-#define TK_INSTEAD                        41
-#define TK_LIKE_KW                        42
-#define TK_MATCH                          43
-#define TK_KEY                            44
-#define TK_OF                             45
-#define TK_OFFSET                         46
-#define TK_PRAGMA                         47
-#define TK_RAISE                          48
-#define TK_REPLACE                        49
-#define TK_RESTRICT                       50
-#define TK_ROW                            51
-#define TK_TRIGGER                        52
-#define TK_VACUUM                         53
-#define TK_VIEW                           54
-#define TK_VIRTUAL                        55
-#define TK_REINDEX                        56
-#define TK_RENAME                         57
-#define TK_CTIME_KW                       58
-#define TK_ANY                            59
-#define TK_OR                             60
-#define TK_AND                            61
-#define TK_IS                             62
-#define TK_BETWEEN                        63
-#define TK_IN                             64
-#define TK_ISNULL                         65
-#define TK_NOTNULL                        66
-#define TK_NE                             67
-#define TK_EQ                             68
-#define TK_GT                             69
-#define TK_LE                             70
-#define TK_LT                             71
-#define TK_GE                             72
-#define TK_ESCAPE                         73
-#define TK_BITAND                         74
-#define TK_BITOR                          75
-#define TK_LSHIFT                         76
-#define TK_RSHIFT                         77
-#define TK_PLUS                           78
-#define TK_MINUS                          79
-#define TK_STAR                           80
-#define TK_SLASH                          81
-#define TK_REM                            82
-#define TK_CONCAT                         83
-#define TK_COLLATE                        84
-#define TK_UMINUS                         85
-#define TK_UPLUS                          86
-#define TK_BITNOT                         87
-#define TK_STRING                         88
-#define TK_JOIN_KW                        89
-#define TK_CONSTRAINT                     90
-#define TK_DEFAULT                        91
-#define TK_NULL                           92
-#define TK_PRIMARY                        93
-#define TK_UNIQUE                         94
-#define TK_CHECK                          95
-#define TK_REFERENCES                     96
-#define TK_AUTOINCR                       97
-#define TK_ON                             98
-#define TK_DELETE                         99
-#define TK_UPDATE                         100
-#define TK_INSERT                         101
-#define TK_SET                            102
-#define TK_DEFERRABLE                     103
-#define TK_FOREIGN                        104
-#define TK_DROP                           105
-#define TK_UNION                          106
-#define TK_ALL                            107
-#define TK_EXCEPT                         108
-#define TK_INTERSECT                      109
-#define TK_SELECT                         110
-#define TK_DISTINCT                       111
-#define TK_DOT                            112
-#define TK_FROM                           113
-#define TK_JOIN                           114
-#define TK_INDEXED                        115
-#define TK_BY                             116
-#define TK_USING                          117
-#define TK_ORDER                          118
-#define TK_GROUP                          119
-#define TK_HAVING                         120
-#define TK_LIMIT                          121
-#define TK_WHERE                          122
-#define TK_INTO                           123
-#define TK_VALUES                         124
-#define TK_INTEGER                        125
-#define TK_FLOAT                          126
-#define TK_BLOB                           127
-#define TK_REGISTER                       128
-#define TK_VARIABLE                       129
-#define TK_CASE                           130
-#define TK_WHEN                           131
-#define TK_THEN                           132
-#define TK_ELSE                           133
-#define TK_INDEX                          134
-#define TK_ALTER                          135
-#define TK_TO                             136
-#define TK_ADD                            137
-#define TK_COLUMNKW                       138
-#define TK_TO_TEXT                        139
-#define TK_TO_BLOB                        140
-#define TK_TO_NUMERIC                     141
-#define TK_TO_INT                         142
-#define TK_TO_REAL                        143
-#define TK_END_OF_FILE                    144
-#define TK_ILLEGAL                        145
-#define TK_SPACE                          146
-#define TK_UNCLOSED_STRING                147
-#define TK_FUNCTION                       148
-#define TK_COLUMN                         149
-#define TK_AGG_FUNCTION                   150
-#define TK_AGG_COLUMN                     151
-#define TK_CONST_FUNC                     152
+#define TK_SAVEPOINT                      13
+#define TK_RELEASE                        14
+#define TK_TO                             15
+#define TK_CREATE                         16
+#define TK_TABLE                          17
+#define TK_IF                             18
+#define TK_NOT                            19
+#define TK_EXISTS                         20
+#define TK_TEMP                           21
+#define TK_LP                             22
+#define TK_RP                             23
+#define TK_AS                             24
+#define TK_COMMA                          25
+#define TK_ID                             26
+#define TK_ABORT                          27
+#define TK_AFTER                          28
+#define TK_ANALYZE                        29
+#define TK_ASC                            30
+#define TK_ATTACH                         31
+#define TK_BEFORE                         32
+#define TK_CASCADE                        33
+#define TK_CAST                           34
+#define TK_CONFLICT                       35
+#define TK_DATABASE                       36
+#define TK_DESC                           37
+#define TK_DETACH                         38
+#define TK_EACH                           39
+#define TK_FAIL                           40
+#define TK_FOR                            41
+#define TK_IGNORE                         42
+#define TK_INITIALLY                      43
+#define TK_INSTEAD                        44
+#define TK_LIKE_KW                        45
+#define TK_MATCH                          46
+#define TK_KEY                            47
+#define TK_OF                             48
+#define TK_OFFSET                         49
+#define TK_PRAGMA                         50
+#define TK_RAISE                          51
+#define TK_REPLACE                        52
+#define TK_RESTRICT                       53
+#define TK_ROW                            54
+#define TK_TRIGGER                        55
+#define TK_VACUUM                         56
+#define TK_VIEW                           57
+#define TK_VIRTUAL                        58
+#define TK_REINDEX                        59
+#define TK_RENAME                         60
+#define TK_CTIME_KW                       61
+#define TK_ANY                            62
+#define TK_OR                             63
+#define TK_AND                            64
+#define TK_IS                             65
+#define TK_BETWEEN                        66
+#define TK_IN                             67
+#define TK_ISNULL                         68
+#define TK_NOTNULL                        69
+#define TK_NE                             70
+#define TK_EQ                             71
+#define TK_GT                             72
+#define TK_LE                             73
+#define TK_LT                             74
+#define TK_GE                             75
+#define TK_ESCAPE                         76
+#define TK_BITAND                         77
+#define TK_BITOR                          78
+#define TK_LSHIFT                         79
+#define TK_RSHIFT                         80
+#define TK_PLUS                           81
+#define TK_MINUS                          82
+#define TK_STAR                           83
+#define TK_SLASH                          84
+#define TK_REM                            85
+#define TK_CONCAT                         86
+#define TK_COLLATE                        87
+#define TK_UMINUS                         88
+#define TK_UPLUS                          89
+#define TK_BITNOT                         90
+#define TK_STRING                         91
+#define TK_JOIN_KW                        92
+#define TK_CONSTRAINT                     93
+#define TK_DEFAULT                        94
+#define TK_NULL                           95
+#define TK_PRIMARY                        96
+#define TK_UNIQUE                         97
+#define TK_CHECK                          98
+#define TK_REFERENCES                     99
+#define TK_AUTOINCR                       100
+#define TK_ON                             101
+#define TK_DELETE                         102
+#define TK_UPDATE                         103
+#define TK_INSERT                         104
+#define TK_SET                            105
+#define TK_DEFERRABLE                     106
+#define TK_FOREIGN                        107
+#define TK_DROP                           108
+#define TK_UNION                          109
+#define TK_ALL                            110
+#define TK_EXCEPT                         111
+#define TK_INTERSECT                      112
+#define TK_SELECT                         113
+#define TK_DISTINCT                       114
+#define TK_DOT                            115
+#define TK_FROM                           116
+#define TK_JOIN                           117
+#define TK_INDEXED                        118
+#define TK_BY                             119
+#define TK_USING                          120
+#define TK_ORDER                          121
+#define TK_GROUP                          122
+#define TK_HAVING                         123
+#define TK_LIMIT                          124
+#define TK_WHERE                          125
+#define TK_INTO                           126
+#define TK_VALUES                         127
+#define TK_INTEGER                        128
+#define TK_FLOAT                          129
+#define TK_BLOB                           130
+#define TK_REGISTER                       131
+#define TK_VARIABLE                       132
+#define TK_CASE                           133
+#define TK_WHEN                           134
+#define TK_THEN                           135
+#define TK_ELSE                           136
+#define TK_INDEX                          137
+#define TK_ALTER                          138
+#define TK_ADD                            139
+#define TK_COLUMNKW                       140
+#define TK_TO_TEXT                        141
+#define TK_TO_BLOB                        142
+#define TK_TO_NUMERIC                     143
+#define TK_TO_INT                         144
+#define TK_TO_REAL                        145
+#define TK_END_OF_FILE                    146
+#define TK_ILLEGAL                        147
+#define TK_SPACE                          148
+#define TK_UNCLOSED_STRING                149
+#define TK_FUNCTION                       150
+#define TK_COLUMN                         151
+#define TK_AGG_FUNCTION                   152
+#define TK_AGG_COLUMN                     153
+#define TK_CONST_FUNC                     154
 
 /************** End of parse.h ***********************************************/
 /************** Continuing where we left off in sqliteInt.h ******************/
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
 #include <stddef.h>
@@ -7720,27 +7768,29 @@ typedef struct IdList IdList;
 typedef struct Index Index;
 typedef struct KeyClass KeyClass;
 typedef struct KeyInfo KeyInfo;
 typedef struct Lookaside Lookaside;
 typedef struct LookasideSlot LookasideSlot;
 typedef struct Module Module;
 typedef struct NameContext NameContext;
 typedef struct Parse Parse;
+typedef struct Savepoint Savepoint;
 typedef struct Select Select;
 typedef struct SrcList SrcList;
 typedef struct StrAccum StrAccum;
 typedef struct Table Table;
 typedef struct TableLock TableLock;
 typedef struct Token Token;
 typedef struct TriggerStack TriggerStack;
 typedef struct TriggerStep TriggerStep;
 typedef struct Trigger Trigger;
 typedef struct UnpackedRecord UnpackedRecord;
 typedef struct Walker Walker;
+typedef struct WherePlan WherePlan;
 typedef struct WhereInfo WhereInfo;
 typedef struct WhereLevel WhereLevel;
 
 /*
 ** Defer sourcing vdbe.h and btree.h until after the "u8" and 
 ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
 ** pointer types (i.e. FuncDef) defined above.
 */
@@ -7756,17 +7806,17 @@ typedef struct WhereLevel WhereLevel;
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This header file defines the interface that the sqlite B-Tree file
 ** subsystem.  See comments in the source code for a detailed description
 ** of what each interface routine does.
 **
-** @(#) $Id: btree.h,v 1.105 2008/10/27 13:59:34 danielk1977 Exp $
+** @(#) $Id: btree.h,v 1.106 2008/12/17 17:30:26 danielk1977 Exp $
 */
 #ifndef _BTREE_H_
 #define _BTREE_H_
 
 /* TODO: This definition is just included so other modules compile. It
 ** needs to be revisited.
 */
 #define SQLITE_N_BTREE_META 10
@@ -7844,16 +7894,17 @@ SQLITE_PRIVATE int sqlite3BtreeCommitStm
 SQLITE_PRIVATE int sqlite3BtreeRollbackStmt(Btree*);
 SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, int*, int flags);
 SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree*);
 SQLITE_PRIVATE int sqlite3BtreeIsInStmt(Btree*);
 SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree*);
 SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *));
 SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *);
 SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *, int, u8);
+SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int);
 
 SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *);
 SQLITE_PRIVATE const char *sqlite3BtreeGetDirname(Btree *);
 SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *);
 SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *);
 
 SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *);
 
@@ -8124,188 +8175,190 @@ typedef struct VdbeOpList VdbeOpList;
 /************** Begin file opcodes.h *****************************************/
 /* Automatically generated.  Do not edit */
 /* See the mkopcodeh.awk script for details */
 #define OP_VNext                                1
 #define OP_Affinity                             2
 #define OP_Column                               3
 #define OP_SetCookie                            4
 #define OP_Seek                                 5
-#define OP_Real                               126   /* same as TK_FLOAT    */
+#define OP_Real                               129   /* same as TK_FLOAT    */
 #define OP_Sequence                             6
-#define OP_Ge                                  72   /* same as TK_GE       */
-#define OP_RowKey                               7
-#define OP_SCopy                                8
-#define OP_Eq                                  68   /* same as TK_EQ       */
-#define OP_OpenWrite                            9
-#define OP_NotNull                             66   /* same as TK_NOTNULL  */
-#define OP_If                                  10
-#define OP_ToInt                              142   /* same as TK_TO_INT   */
-#define OP_String8                             88   /* same as TK_STRING   */
-#define OP_VRowid                              11
-#define OP_CollSeq                             12
-#define OP_OpenRead                            13
-#define OP_Expire                              14
-#define OP_AutoCommit                          15
-#define OP_Gt                                  69   /* same as TK_GT       */
+#define OP_Savepoint                            7
+#define OP_Ge                                  75   /* same as TK_GE       */
+#define OP_RowKey                               8
+#define OP_SCopy                                9
+#define OP_Eq                                  71   /* same as TK_EQ       */
+#define OP_OpenWrite                           10
+#define OP_NotNull                             69   /* same as TK_NOTNULL  */
+#define OP_If                                  11
+#define OP_ToInt                              144   /* same as TK_TO_INT   */
+#define OP_String8                             91   /* same as TK_STRING   */
+#define OP_VRowid                              12
+#define OP_CollSeq                             13
+#define OP_OpenRead                            14
+#define OP_Expire                              15
+#define OP_AutoCommit                          16
+#define OP_Gt                                  72   /* same as TK_GT       */
 #define OP_Pagecount                           17
 #define OP_IntegrityCk                         18
-#define OP_Sort                                19
-#define OP_Copy                                20
-#define OP_Trace                               21
-#define OP_Function                            22
-#define OP_IfNeg                               23
-#define OP_And                                 61   /* same as TK_AND      */
-#define OP_Subtract                            79   /* same as TK_MINUS    */
-#define OP_Noop                                24
-#define OP_Return                              25
-#define OP_Remainder                           82   /* same as TK_REM      */
-#define OP_NewRowid                            26
-#define OP_Multiply                            80   /* same as TK_STAR     */
-#define OP_Variable                            27
-#define OP_String                              28
-#define OP_RealAffinity                        29
-#define OP_VRename                             30
-#define OP_ParseSchema                         31
-#define OP_VOpen                               32
-#define OP_Close                               33
-#define OP_CreateIndex                         34
-#define OP_IsUnique                            35
-#define OP_NotFound                            36
-#define OP_Int64                               37
-#define OP_MustBeInt                           38
-#define OP_Halt                                39
-#define OP_Rowid                               40
-#define OP_IdxLT                               41
-#define OP_AddImm                              42
-#define OP_Statement                           43
-#define OP_RowData                             44
-#define OP_MemMax                              45
-#define OP_Or                                  60   /* same as TK_OR       */
-#define OP_NotExists                           46
-#define OP_Gosub                               47
-#define OP_Divide                              81   /* same as TK_SLASH    */
-#define OP_Integer                             48
-#define OP_ToNumeric                          141   /* same as TK_TO_NUMERIC*/
-#define OP_Prev                                49
-#define OP_RowSetRead                          50
-#define OP_Concat                              83   /* same as TK_CONCAT   */
-#define OP_RowSetAdd                           51
-#define OP_BitAnd                              74   /* same as TK_BITAND   */
-#define OP_VColumn                             52
-#define OP_CreateTable                         53
-#define OP_Last                                54
-#define OP_SeekLe                              55
-#define OP_IsNull                              65   /* same as TK_ISNULL   */
-#define OP_IncrVacuum                          56
-#define OP_IdxRowid                            57
-#define OP_ShiftRight                          77   /* same as TK_RSHIFT   */
-#define OP_ResetCount                          58
-#define OP_ContextPush                         59
-#define OP_Yield                               62
-#define OP_DropTrigger                         63
-#define OP_DropIndex                           64
-#define OP_IdxGE                               73
-#define OP_IdxDelete                           84
-#define OP_Vacuum                              85
-#define OP_IfNot                               86
-#define OP_DropTable                           89
-#define OP_SeekLt                              90
-#define OP_MakeRecord                          91
-#define OP_ToBlob                             140   /* same as TK_TO_BLOB  */
-#define OP_ResultRow                           92
-#define OP_Delete                              93
-#define OP_AggFinal                            94
-#define OP_Compare                             95
-#define OP_ShiftLeft                           76   /* same as TK_LSHIFT   */
-#define OP_Goto                                96
-#define OP_TableLock                           97
-#define OP_Clear                               98
-#define OP_Le                                  70   /* same as TK_LE       */
-#define OP_VerifyCookie                        99
-#define OP_AggStep                            100
-#define OP_ToText                             139   /* same as TK_TO_TEXT  */
-#define OP_Not                                 16   /* same as TK_NOT      */
-#define OP_ToReal                             143   /* same as TK_TO_REAL  */
-#define OP_SetNumColumns                      101
-#define OP_Transaction                        102
-#define OP_VFilter                            103
-#define OP_Ne                                  67   /* same as TK_NE       */
-#define OP_VDestroy                           104
-#define OP_ContextPop                         105
-#define OP_BitOr                               75   /* same as TK_BITOR    */
-#define OP_Next                               106
-#define OP_IdxInsert                          107
-#define OP_Lt                                  71   /* same as TK_LT       */
-#define OP_SeekGe                             108
-#define OP_Insert                             109
-#define OP_Destroy                            110
-#define OP_ReadCookie                         111
-#define OP_LoadAnalysis                       112
-#define OP_Explain                            113
-#define OP_OpenPseudo                         114
-#define OP_OpenEphemeral                      115
-#define OP_Null                               116
-#define OP_Move                               117
-#define OP_Blob                               118
-#define OP_Add                                 78   /* same as TK_PLUS     */
-#define OP_Rewind                             119
-#define OP_SeekGt                             120
-#define OP_VBegin                             121
-#define OP_VUpdate                            122
-#define OP_IfZero                             123
-#define OP_BitNot                              87   /* same as TK_BITNOT   */
-#define OP_VCreate                            124
-#define OP_Found                              125
+#define OP_Sort                                20
+#define OP_Copy                                21
+#define OP_Trace                               22
+#define OP_Function                            23
+#define OP_IfNeg                               24
+#define OP_And                                 64   /* same as TK_AND      */
+#define OP_Subtract                            82   /* same as TK_MINUS    */
+#define OP_Noop                                25
+#define OP_Return                              26
+#define OP_Remainder                           85   /* same as TK_REM      */
+#define OP_NewRowid                            27
+#define OP_Multiply                            83   /* same as TK_STAR     */
+#define OP_Variable                            28
+#define OP_String                              29
+#define OP_RealAffinity                        30
+#define OP_VRename                             31
+#define OP_ParseSchema                         32
+#define OP_VOpen                               33
+#define OP_Close                               34
+#define OP_CreateIndex                         35
+#define OP_IsUnique                            36
+#define OP_NotFound                            37
+#define OP_Int64                               38
+#define OP_MustBeInt                           39
+#define OP_Halt                                40
+#define OP_Rowid                               41
+#define OP_IdxLT                               42
+#define OP_AddImm                              43
+#define OP_Statement                           44
+#define OP_RowData                             45
+#define OP_MemMax                              46
+#define OP_Or                                  63   /* same as TK_OR       */
+#define OP_NotExists                           47
+#define OP_Gosub                               48
+#define OP_Divide                              84   /* same as TK_SLASH    */
+#define OP_Integer                             49
+#define OP_ToNumeric                          143   /* same as TK_TO_NUMERIC*/
+#define OP_Prev                                50
+#define OP_RowSetRead                          51
+#define OP_Concat                              86   /* same as TK_CONCAT   */
+#define OP_RowSetAdd                           52
+#define OP_BitAnd                              77   /* same as TK_BITAND   */
+#define OP_VColumn                             53
+#define OP_CreateTable                         54
+#define OP_Last                                55
+#define OP_SeekLe                              56
+#define OP_IsNull                              68   /* same as TK_ISNULL   */
+#define OP_IncrVacuum                          57
+#define OP_IdxRowid                            58
+#define OP_ShiftRight                          80   /* same as TK_RSHIFT   */
+#define OP_ResetCount                          59
+#define OP_ContextPush                         60
+#define OP_Yield                               61
+#define OP_DropTrigger                         62
+#define OP_DropIndex                           65
+#define OP_IdxGE                               66
+#define OP_IdxDelete                           67
+#define OP_Vacuum                              76
+#define OP_IfNot                               87
+#define OP_DropTable                           88
+#define OP_SeekLt                              89
+#define OP_MakeRecord                          92
+#define OP_ToBlob                             142   /* same as TK_TO_BLOB  */
+#define OP_ResultRow                           93
+#define OP_Delete                              94
+#define OP_AggFinal                            95
+#define OP_Compare                             96
+#define OP_ShiftLeft                           79   /* same as TK_LSHIFT   */
+#define OP_Goto                                97
+#define OP_TableLock                           98
+#define OP_Clear                               99
+#define OP_Le                                  73   /* same as TK_LE       */
+#define OP_VerifyCookie                       100
+#define OP_AggStep                            101
+#define OP_ToText                             141   /* same as TK_TO_TEXT  */
+#define OP_Not                                 19   /* same as TK_NOT      */
+#define OP_ToReal                             145   /* same as TK_TO_REAL  */
+#define OP_SetNumColumns                      102
+#define OP_Transaction                        103
+#define OP_VFilter                            104
+#define OP_Ne                                  70   /* same as TK_NE       */
+#define OP_VDestroy                           105
+#define OP_ContextPop                         106
+#define OP_BitOr                               78   /* same as TK_BITOR    */
+#define OP_Next                               107
+#define OP_IdxInsert                          108
+#define OP_Lt                                  74   /* same as TK_LT       */
+#define OP_SeekGe                             109
+#define OP_Insert                             110
+#define OP_Destroy                            111
+#define OP_ReadCookie                         112
+#define OP_LoadAnalysis                       113
+#define OP_Explain                            114
+#define OP_OpenPseudo                         115
+#define OP_OpenEphemeral                      116
+#define OP_Null                               117
+#define OP_Move                               118
+#define OP_Blob                               119
+#define OP_Add                                 81   /* same as TK_PLUS     */
+#define OP_Rewind                             120
+#define OP_SeekGt                             121
+#define OP_VBegin                             122
+#define OP_VUpdate                            123
+#define OP_IfZero                             124
+#define OP_BitNot                              90   /* same as TK_BITNOT   */
+#define OP_VCreate                            125
+#define OP_Found                              126
 #define OP_IfPos                              127
 #define OP_NullRow                            128
-#define OP_Jump                               129
-#define OP_Permutation                        130
+#define OP_Jump                               130
+#define OP_Permutation                        131
 
 /* The following opcode values are never used */
-#define OP_NotUsed_131                        131
 #define OP_NotUsed_132                        132
 #define OP_NotUsed_133                        133
 #define OP_NotUsed_134                        134
 #define OP_NotUsed_135                        135
 #define OP_NotUsed_136                        136
 #define OP_NotUsed_137                        137
 #define OP_NotUsed_138                        138
+#define OP_NotUsed_139                        139
+#define OP_NotUsed_140                        140
 
 
 /* Properties such as "out2" or "jump" that are specified in
 ** comments following the "case" for each opcode in the vdbe.c
 ** are encoded into bitvectors as follows:
 */
 #define OPFLG_JUMP            0x0001  /* jump:  P2 holds jmp target */
 #define OPFLG_OUT2_PRERELEASE 0x0002  /* out2-prerelease: */
 #define OPFLG_IN1             0x0004  /* in1:   P1 is an input */
 #define OPFLG_IN2             0x0008  /* in2:   P2 is an input */
 #define OPFLG_IN3             0x0010  /* in3:   P3 is an input */
 #define OPFLG_OUT3            0x0020  /* out3:  P3 is an output */
 #define OPFLG_INITIALIZER {\
 /*   0 */ 0x00, 0x01, 0x00, 0x00, 0x10, 0x08, 0x02, 0x00,\
-/*   8 */ 0x00, 0x00, 0x05, 0x02, 0x00, 0x00, 0x00, 0x00,\
-/*  16 */ 0x04, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05,\
-/*  24 */ 0x00, 0x04, 0x02, 0x02, 0x02, 0x04, 0x00, 0x00,\
-/*  32 */ 0x00, 0x00, 0x02, 0x11, 0x11, 0x02, 0x05, 0x00,\
-/*  40 */ 0x02, 0x11, 0x04, 0x00, 0x00, 0x0c, 0x11, 0x01,\
-/*  48 */ 0x02, 0x01, 0x21, 0x08, 0x00, 0x02, 0x01, 0x11,\
-/*  56 */ 0x01, 0x02, 0x00, 0x00, 0x2c, 0x2c, 0x00, 0x00,\
-/*  64 */ 0x00, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
-/*  72 */ 0x15, 0x11, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,\
-/*  80 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x00, 0x00, 0x05, 0x04,\
-/*  88 */ 0x02, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/*  96 */ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,\
-/* 104 */ 0x00, 0x00, 0x01, 0x08, 0x11, 0x00, 0x02, 0x02,\
-/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x01,\
-/* 120 */ 0x11, 0x00, 0x00, 0x05, 0x00, 0x11, 0x02, 0x05,\
-/* 128 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/* 136 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x04,\
-}
+/*   8 */ 0x00, 0x04, 0x00, 0x05, 0x02, 0x00, 0x00, 0x00,\
+/*  16 */ 0x00, 0x02, 0x00, 0x04, 0x01, 0x04, 0x00, 0x00,\
+/*  24 */ 0x05, 0x00, 0x04, 0x02, 0x02, 0x02, 0x04, 0x00,\
+/*  32 */ 0x00, 0x00, 0x00, 0x02, 0x11, 0x11, 0x02, 0x05,\
+/*  40 */ 0x00, 0x02, 0x11, 0x04, 0x00, 0x00, 0x0c, 0x11,\
+/*  48 */ 0x01, 0x02, 0x01, 0x21, 0x08, 0x00, 0x02, 0x01,\
+/*  56 */ 0x11, 0x01, 0x02, 0x00, 0x00, 0x04, 0x00, 0x2c,\
+/*  64 */ 0x2c, 0x00, 0x11, 0x00, 0x05, 0x05, 0x15, 0x15,\
+/*  72 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x2c, 0x2c, 0x2c,\
+/*  80 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x05,\
+/*  88 */ 0x00, 0x11, 0x04, 0x02, 0x00, 0x00, 0x00, 0x00,\
+/*  96 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
+/* 104 */ 0x01, 0x00, 0x00, 0x01, 0x08, 0x11, 0x00, 0x02,\
+/* 112 */ 0x02, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02,\
+/* 120 */ 0x01, 0x11, 0x00, 0x00, 0x05, 0x00, 0x11, 0x05,\
+/* 128 */ 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,\
+/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
+/* 144 */ 0x04, 0x04,}
 
 /************** End of opcodes.h *********************************************/
 /************** Continuing where we left off in vdbe.h ***********************/
 
 /*
 ** Prototypes for the VDBE interface.  See comments on the implementation
 ** for a description of what each of these routines does.
 */
@@ -8378,17 +8431,17 @@ SQLITE_PRIVATE   void sqlite3VdbeNoopCom
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This header file defines the interface that the sqlite page cache
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.88 2008/12/10 16:45:51 drh Exp $
+** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $
 */
 
 #ifndef _PAGER_H_
 #define _PAGER_H_
 
 /*
 ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise
 ** it must be turned on for each database using "PRAGMA auto_vacuum = 1".
@@ -8453,25 +8506,21 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pag
 SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag);
 #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0)
 SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno);
 SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerRef(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerUnref(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerWrite(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerPagecount(Pager*, int*);
-SQLITE_PRIVATE int sqlite3PagerTruncate(Pager*,Pgno);
 SQLITE_PRIVATE int sqlite3PagerBegin(DbPage*, int exFlag);
-SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, Pgno, int);
+SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int);
 SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*);
 SQLITE_PRIVATE int sqlite3PagerRollback(Pager*);
 SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*);
-SQLITE_PRIVATE int sqlite3PagerStmtBegin(Pager*);
-SQLITE_PRIVATE int sqlite3PagerStmtCommit(Pager*);
-SQLITE_PRIVATE int sqlite3PagerStmtRollback(Pager*);
 SQLITE_PRIVATE void sqlite3PagerDontRollback(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerDontWrite(DbPage*);
 SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*);
 SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int);
 SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*);
 SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*);
 SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*);
 SQLITE_PRIVATE const char *sqlite3PagerDirname(Pager*);
@@ -8481,16 +8530,24 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(
 SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); 
 SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); 
 SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int);
 SQLITE_PRIVATE int sqlite3PagerJournalMode(Pager *, int);
 SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64);
 SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*);
 SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager);
 
+SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
+SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
+
+#ifndef SQLITE_OMIT_AUTOVACUUM
+SQLITE_PRIVATE   void sqlite3PagerTruncateImage(Pager*,Pgno);
+SQLITE_PRIVATE   Pgno sqlite3PagerImageSize(Pager *);
+#endif
+
 #ifdef SQLITE_HAS_CODEC
 SQLITE_PRIVATE   void sqlite3PagerSetCodec(Pager*,void*(*)(void*,void*,Pgno,int),void*);
 #endif
 
 #if !defined(NDEBUG) || defined(SQLITE_TEST)
 SQLITE_PRIVATE   Pgno sqlite3PagerPagenumber(DbPage*);
 SQLITE_PRIVATE   int sqlite3PagerIswriteable(DbPage*);
 #endif
@@ -8692,17 +8749,17 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefa
 **
 ** This header file (together with is companion C source-code file
 ** "os.c") attempt to abstract the underlying operating system so that
 ** the SQLite library will work on both POSIX and windows systems.
 **
 ** This header file is #include-ed by sqliteInt.h and thus ends up
 ** being included by every source file.
 **
-** $Id: os.h,v 1.106 2008/12/08 18:19:18 drh Exp $
+** $Id: os.h,v 1.107 2009/01/14 23:03:41 drh Exp $
 */
 #ifndef _SQLITE_OS_H_
 #define _SQLITE_OS_H_
 
 /*
 ** Figure out if we are dealing with Unix, Windows, or some other
 ** operating system.  After the following block of preprocess macros,
 ** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, SQLITE_OS_OS2, and SQLITE_OS_OTHER 
@@ -8920,16 +8977,17 @@ SQLITE_PRIVATE int sqlite3OsRead(sqlite3
 SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset);
 SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size);
 SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int);
 SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize);
 SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int);
 SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int);
 SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut);
 SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*);
+#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0
 SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id);
 SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
 
 /* 
 ** Functions for accessing sqlite3_vfs methods 
 */
 SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
 SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int);
@@ -9246,16 +9304,19 @@ struct sqlite3 {
   FuncDefHash aFunc;            /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
   BusyHandler busyHandler;      /* Busy callback */
   int busyTimeout;              /* Busy handler timeout, in msec */
   Db aDbStatic[2];              /* Static space for the 2 default backends */
 #ifdef SQLITE_SSE
   sqlite3_stmt *pFetch;         /* Used by SSE to fetch stored statements */
 #endif
+  Savepoint *pSavepoint;        /* List of active savepoints */
+  int nSavepoint;               /* Number of non-transaction savepoints */
+  u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
 };
 
 /*
 ** A macro to discover the encoding of a database.
 */
 #define ENC(db) ((db)->aDb[0].pSchema->enc)
 
 /*
@@ -9284,16 +9345,17 @@ struct sqlite3 {
 #define SQLITE_ReadUncommitted 0x00004000 /* For shared-cache mode */
 #define SQLITE_LegacyFileFmt  0x00008000  /* Create new databases in format 1 */
 #define SQLITE_FullFSync      0x00010000  /* Use full fsync on the backend */
 #define SQLITE_LoadExtension  0x00020000  /* Enable load_extension */
 
 #define SQLITE_RecoveryMode   0x00040000  /* Ignore schema errors */
 #define SQLITE_SharedCache    0x00080000  /* Cache sharing is enabled */
 #define SQLITE_Vtab           0x00100000  /* There exists a virtual table */
+#define SQLITE_CommitBusy     0x00200000  /* In the process of committing */
 
 /*
 ** Possible values for the sqlite.magic field.
 ** The numbers are obtained at random and have no special meaning, other
 ** than being distinct from one another.
 */
 #define SQLITE_MAGIC_OPEN     0xa029a697  /* Database is open */
 #define SQLITE_MAGIC_CLOSED   0x9f3c2d33  /* Database is closed */
@@ -9322,16 +9384,17 @@ struct FuncDef {
 
 /*
 ** Possible values for FuncDef.flags
 */
 #define SQLITE_FUNC_LIKE     0x01 /* Candidate for the LIKE optimization */
 #define SQLITE_FUNC_CASE     0x02 /* Case-sensitive LIKE-type function */
 #define SQLITE_FUNC_EPHEM    0x04 /* Ephemeral.  Delete with VDBE */
 #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
+#define SQLITE_FUNC_PRIVATE  0x10 /* Allowed for internal use only */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
 ** used to create the initializers for the FuncDef structures.
 **
 **   FUNCTION(zName, nArg, iArg, bNC, xFunc)
 **     Used to create a scalar function definition of a function zName 
 **     implemented by C function xFunc that accepts nArg arguments. The
@@ -9357,16 +9420,35 @@ struct FuncDef {
   {nArg, SQLITE_UTF8, bNC*8, SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0}
 #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
   {nArg, SQLITE_UTF8, bNC*8, pArg, 0, xFunc, 0, 0, #zName, 0}
 #define LIKEFUNC(zName, nArg, arg, flags) \
   {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0}
 #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
   {nArg, SQLITE_UTF8, nc*8, SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0}
 
+/*
+** All current savepoints are stored in a linked list starting at
+** sqlite3.pSavepoint. The first element in the list is the most recently
+** opened savepoint. Savepoints are added to the list by the vdbe
+** OP_Savepoint instruction.
+*/
+struct Savepoint {
+  char *zName;                        /* Savepoint name (nul-terminated) */
+  Savepoint *pNext;                   /* Parent savepoint (if any) */
+};
+
+/*
+** The following are used as the second parameter to sqlite3Savepoint(),
+** and as the P1 argument to the OP_Savepoint instruction.
+*/
+#define SAVEPOINT_BEGIN      0
+#define SAVEPOINT_RELEASE    1
+#define SAVEPOINT_ROLLBACK   2
+
 
 /*
 ** Each SQLite module (virtual table definition) is defined by an
 ** instance of the following structure, stored in the sqlite3.aModule
 ** hash table.
 */
 struct Module {
   const sqlite3_module *pModule;       /* Callback pointers */
@@ -9956,110 +10038,139 @@ struct SrcList {
   struct SrcList_item {
     char *zDatabase;  /* Name of database holding this table */
     char *zName;      /* Name of the table */
     char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
     Table *pTab;      /* An SQL table corresponding to zName */
     Select *pSelect;  /* A SELECT statement used in place of a table name */
     u8 isPopulated;   /* Temporary table associated with SELECT is populated */
     u8 jointype;      /* Type of join between this able and the previous */
+    u8 notIndexed;    /* True if there is a NOT INDEXED clause */
     int iCursor;      /* The VDBE cursor number used to access this table */
     Expr *pOn;        /* The ON clause of a join */
     IdList *pUsing;   /* The USING clause of a join */
-    Bitmask colUsed;  /* Bit N (1<<N) set if column N or pTab is used */
-    u8 notIndexed;    /* True if there is a NOT INDEXED clause */
+    Bitmask colUsed;  /* Bit N (1<<N) set if column N of pTab is used */
     char *zIndex;     /* Identifier from "INDEXED BY <zIndex>" clause */
     Index *pIndex;    /* Index structure corresponding to zIndex, if any */
   } a[1];             /* One entry for each identifier on the list */
 };
 
 /*
 ** Permitted values of the SrcList.a.jointype field
 */
 #define JT_INNER     0x0001    /* Any kind of inner or cross join */
 #define JT_CROSS     0x0002    /* Explicit use of the CROSS keyword */
 #define JT_NATURAL   0x0004    /* True for a "natural" join */
 #define JT_LEFT      0x0008    /* Left outer join */
 #define JT_RIGHT     0x0010    /* Right outer join */
 #define JT_OUTER     0x0020    /* The "OUTER" keyword is present */
 #define JT_ERROR     0x0040    /* unknown or unsupported join type */
 
+
+/*
+** A WherePlan object holds information that describes a lookup
+** strategy.
+**
+** This object is intended to be opaque outside of the where.c module.
+** It is included here only so that that compiler will know how big it
+** is.  None of the fields in this object should be used outside of
+** the where.c module.
+**
+** Within the union, pIdx is only used when wsFlags&WHERE_INDEXED is true.
+** pTerm is only used when wsFlags&WHERE_MULTI_OR is true.  And pVtabIdx
+** is only used when wsFlags&WHERE_VIRTUALTABLE is true.  It is never the
+** case that more than one of these conditions is true.
+*/
+struct WherePlan {
+  u32 wsFlags;                   /* WHERE_* flags that describe the strategy */
+  u32 nEq;                       /* Number of == constraints */
+  union {
+    Index *pIdx;                   /* Index when WHERE_INDEXED is true */
+    struct WhereTerm *pTerm;       /* WHERE clause term for OR-search */
+    sqlite3_index_info *pVtabIdx;  /* Virtual table index to use */
+  } u;
+};
+
 /*
 ** For each nested loop in a WHERE clause implementation, the WhereInfo
 ** structure contains a single instance of this structure.  This structure
 ** is intended to be private the the where.c module and should not be
 ** access or modified by other modules.
 **
-** The pIdxInfo and pBestIdx fields are used to help pick the best
-** index on a virtual table.  The pIdxInfo pointer contains indexing
+** The pIdxInfo field is used to help pick the best index on a
+** virtual table.  The pIdxInfo pointer contains indexing
 ** information for the i-th table in the FROM clause before reordering.
 ** All the pIdxInfo pointers are freed by whereInfoFree() in where.c.
-** The pBestIdx pointer is a copy of pIdxInfo for the i-th table after
-** FROM clause ordering.  This is a little confusing so I will repeat
-** it in different words.  WhereInfo.a[i].pIdxInfo is index information 
-** for WhereInfo.pTabList.a[i].  WhereInfo.a[i].pBestInfo is the
-** index information for the i-th loop of the join.  pBestInfo is always
-** either NULL or a copy of some pIdxInfo.  So for cleanup it is 
-** sufficient to free all of the pIdxInfo pointers.
-** 
+** All other information in the i-th WhereLevel object for the i-th table
+** after FROM clause ordering.
 */
 struct WhereLevel {
-  int iFrom;            /* Which entry in the FROM clause */
-  int wsFlags;          /* "Where-Scan" flags show the choosen scan strategy */
-  int iMem;             /* First memory cell used by this level */
+  WherePlan plan;       /* query plan for this element of the FROM clause */
   int iLeftJoin;        /* Memory cell used to implement LEFT OUTER JOIN */
-  Index *pIdx;          /* Index used.  NULL if no index */
   int iTabCur;          /* The VDBE cursor used to access the table */
   int iIdxCur;          /* The VDBE cursor used to access pIdx */
   int addrBrk;          /* Jump here to break out of the loop */
   int addrNxt;          /* Jump here to start the next IN combination */
   int addrCont;         /* Jump here to continue with the next loop cycle */
   int addrFirst;        /* First instruction of interior of the loop */
-  int op, p1, p2;       /* Opcode used to terminate the loop */
-  u8 p5;                /* P5 operand of the opcode that terminates the loop */
-  int nEq;              /* Number of == or IN constraints on this loop */
-  int nIn;              /* Number of IN operators constraining this loop */
-  struct InLoop {
-    int iCur;              /* The VDBE cursor used by this IN operator */
-    int addrInTop;         /* Top of the IN loop */
-  } *aInLoop;           /* Information about each nested IN operator */
-  sqlite3_index_info *pBestIdx;  /* Index information for this level */
+  u8 iFrom;             /* Which entry in the FROM clause */
+  u8 op, p5;            /* Opcode and P5 of the opcode that ends the loop */
+  int p1, p2;           /* Operands of the opcode used to ends the loop */
+  union {               /* Information that depends on plan.wsFlags */
+    struct {
+      int nIn;              /* Number of entries in aInLoop[] */
+      struct InLoop {
+        int iCur;              /* The VDBE cursor used by this IN operator */
+        int addrInTop;         /* Top of the IN loop */
+      } *aInLoop;           /* Information about each nested IN operator */
+    } in;                 /* Used when plan.wsFlags&WHERE_IN_ABLE */
+    struct {
+      WherePlan *aPlan;     /* Plans for each term of the WHERE clause */
+    } or;                 /* Used when plan.wsFlags&WHERE_MULTI_OR */
+  } u;
 
   /* The following field is really not part of the current level.  But
-  ** we need a place to cache index information for each table in the
-  ** FROM clause and the WhereLevel structure is a convenient place.
+  ** we need a place to cache virtual table index information for each
+  ** virtual table in the FROM clause and the WhereLevel structure is
+  ** a convenient place since there is one WhereLevel for each FROM clause
+  ** element.
   */
   sqlite3_index_info *pIdxInfo;  /* Index info for n-th source table */
 };
 
 /*
 ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin().
 */
-#define WHERE_ORDERBY_NORMAL     0   /* No-op */
-#define WHERE_ORDERBY_MIN        1   /* ORDER BY processing for min() func */
-#define WHERE_ORDERBY_MAX        2   /* ORDER BY processing for max() func */
-#define WHERE_ONEPASS_DESIRED    4   /* Want to do one-pass UPDATE/DELETE */
+#define WHERE_ORDERBY_NORMAL   0x0000 /* No-op */
+#define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
+#define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
+#define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
+#define WHERE_FILL_ROWSET      0x0008  /* Save results in a RowSet object */
+#define WHERE_OMIT_OPEN        0x0010  /* Table cursor are already open */
+#define WHERE_OMIT_CLOSE       0x0020  /* Omit close of table & index cursors */
 
 /*
 ** The WHERE clause processing routine has two halves.  The
 ** first part does the start of the WHERE loop and the second
 ** half does the tail of the WHERE loop.  An instance of
 ** this structure is returned by the first half and passed
 ** into the second half to give some continuity.
 */
 struct WhereInfo {
   Parse *pParse;       /* Parsing and code generating context */
+  u16 wctrlFlags;      /* Flags originally passed to sqlite3WhereBegin() */
   u8 okOnePass;        /* Ok to use one-pass algorithm for UPDATE or DELETE */
-  SrcList *pTabList;   /* List of tables in the join */
-  int iTop;            /* The very beginning of the WHERE loop */
-  int iContinue;       /* Jump here to continue with next record */
-  int iBreak;          /* Jump here to break out of the loop */
-  int nLevel;          /* Number of nested loop */
-  sqlite3_index_info **apInfo;  /* Array of pointers to index info structures */
-  WhereLevel a[1];     /* Information about each nest loop in the WHERE */
+  int regRowSet;                 /* Store rowids in this rowset if >=0 */
+  SrcList *pTabList;             /* List of tables in the join */
+  int iTop;                      /* The very beginning of the WHERE loop */
+  int iContinue;                 /* Jump here to continue with next record */
+  int iBreak;                    /* Jump here to break out of the loop */
+  int nLevel;                    /* Number of nested loop */
+  struct WhereClause *pWC;       /* Decomposition of the WHERE clause */
+  WhereLevel a[1];               /* Information about each nest loop in WHERE */
 };
 
 /*
 ** A NameContext defines a context in which to resolve table and column
 ** names.  The context consists of a list of tables (the pSrcList) field and
 ** a list of named expression (pEList).  The named expression list may
 ** be NULL.  The pSrc corresponds to the FROM clause of a SELECT or
 ** to the table being operated on by INSERT, UPDATE, or DELETE.  The
@@ -10692,24 +10803,24 @@ SQLITE_PRIVATE void sqlite3SelectDelete(
 SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*);
 SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int);
 SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
 SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *);
 #endif
 SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
 SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
-SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u8);
+SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u8, int);
 SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*);
 SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, int);
 SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int);
 SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, int, int, int);
 SQLITE_PRIVATE void sqlite3ExprClearColumnCache(Parse*, int);
 SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int);
-SQLITE_PRIVATE int sqlite3ExprWritableRegister(Parse*,int,int);
+SQLITE_PRIVATE void sqlite3ExprWritableRegister(Parse*,int);
 SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse*,int,int);
 SQLITE_PRIVATE int sqlite3ExprCode(Parse*, Expr*, int);
 SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*);
 SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int);
 SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse*, Expr*, int);
 SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse*, Expr*);
 SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int);
 SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
@@ -10730,16 +10841,18 @@ SQLITE_PRIVATE Expr *sqlite3CreateIdExpr
 SQLITE_PRIVATE void sqlite3PrngSaveState(void);
 SQLITE_PRIVATE void sqlite3PrngRestoreState(void);
 SQLITE_PRIVATE void sqlite3PrngResetState(void);
 SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*);
 SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int);
 SQLITE_PRIVATE void sqlite3BeginTransaction(Parse*, int);
 SQLITE_PRIVATE void sqlite3CommitTransaction(Parse*);
 SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse*);
+SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*);
+SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *);
 SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*);
 SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
 SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
 SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*);
 SQLITE_PRIVATE int sqlite3IsRowid(const char*);
 SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*, Table*, int, int, int);
 SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int*);
 SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int);
@@ -11028,17 +11141,17 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Pa
 SQLITE_PRIVATE   int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
 SQLITE_PRIVATE   int sqlite3JournalSize(sqlite3_vfs *);
 SQLITE_PRIVATE   int sqlite3JournalCreate(sqlite3_file *);
 #else
   #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile)
 #endif
 
 SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *);
-SQLITE_PRIVATE int sqlite3MemJournalSize();
+SQLITE_PRIVATE int sqlite3MemJournalSize(void);
 SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *);
 
 #if SQLITE_MAX_EXPR_DEPTH>0
 SQLITE_PRIVATE   void sqlite3ExprSetHeight(Parse *pParse, Expr *p);
 SQLITE_PRIVATE   int sqlite3SelectExprHeight(Select *);
 SQLITE_PRIVATE   int sqlite3ExprCheckHeight(Parse*, int);
 #else
   #define sqlite3ExprSetHeight(x,y)
@@ -11318,17 +11431,17 @@ SQLITE_API int sqlite3_db_status(
 *************************************************************************
 ** This file contains the C functions that implement date and time
 ** functions for SQLite.  
 **
 ** There is only one exported symbol in this file - the function
 ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
 ** All other code has file scope.
 **
-** $Id: date.c,v 1.98 2008/12/10 22:30:25 shane Exp $
+** $Id: date.c,v 1.99 2008/12/20 13:18:50 drh Exp $
 **
 ** SQLite processes all times and dates as Julian Day numbers.  The
 ** dates and times are stored as the number of days since noon
 ** in Greenwich on November 24, 4714 B.C. according to the Gregorian
 ** calendar system. 
 **
 ** 1970-01-01 00:00:00 is JD 2440587.5
 ** 2000-01-01 00:00:00 is JD 2451544.5
@@ -11747,17 +11860,17 @@ static sqlite3_int64 localtimeOffset(Dat
     x.s = 0.0;
   } else {
     int s = (int)(x.s + 0.5);
     x.s = s;
   }
   x.tz = 0;
   x.validJD = 0;
   computeJD(&x);
-  t = x.iJD/1000 - 210866760000LL;
+  t = x.iJD/1000 - 21086676*(i64)10000;
 #ifdef HAVE_LOCALTIME_R
   {
     struct tm sLocal;
     localtime_r(&t, &sLocal);
     y.Y = sLocal.tm_year + 1900;
     y.M = sLocal.tm_mon + 1;
     y.D = sLocal.tm_mday;
     y.h = sLocal.tm_hour;
@@ -11849,17 +11962,17 @@ static int parseModifier(const char *zMo
     case 'u': {
       /*
       **    unixepoch
       **
       ** Treat the current value of p->iJD as the number of
       ** seconds since 1970.  Convert to a real julian day number.
       */
       if( strcmp(z, "unixepoch")==0 && p->validJD ){
-        p->iJD = p->iJD/86400 + 210866760000000LL;
+        p->iJD = p->iJD/86400 + 21086676*(i64)10000000;
         clearYMD_HMS_TZ(p);
         rc = 0;
       }
 #ifndef SQLITE_OMIT_LOCALTIME
       else if( strcmp(z, "utc")==0 ){
         sqlite3_int64 c1;
         computeJD(p);
         c1 = localtimeOffset(p);
@@ -17725,17 +17838,17 @@ SQLITE_PRIVATE void sqlite3PrngResetStat
 **
 *************************************************************************
 ** This is the header file for information that is private to the
 ** VDBE.  This information used to all be at the top of the single
 ** source code file "vdbe.c".  When that file became too big (over
 ** 6000 lines long) it was split up into several smaller files and
 ** this header information was factored out.
 **
-** $Id: vdbeInt.h,v 1.160 2008/12/09 02:51:24 drh Exp $
+** $Id: vdbeInt.h,v 1.161 2009/01/05 18:02:27 drh Exp $
 */
 #ifndef _VDBEINT_H_
 #define _VDBEINT_H_
 
 /*
 ** intToKey() and keyToInt() used to transform the rowid.  But with
 ** the latest versions of the design they are no-ops.
 */
@@ -17883,17 +17996,18 @@ struct Mem {
   #undef MEM_Zero
   #define MEM_Zero 0x0000
 #endif
 
 
 /*
 ** Clear any existing type flags from a Mem and replace them with f
 */
-#define MemSetTypeFlag(p, f) ((p)->flags = ((p)->flags&~(MEM_TypeMask))|f)
+#define MemSetTypeFlag(p, f) \
+   ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f)
 
 
 /* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains
 ** additional information about auxiliary information bound to arguments
 ** of the function.  This is used to implement the sqlite3_get_auxdata()
 ** and sqlite3_set_auxdata() APIs.  The "auxdata" is some auxiliary data
 ** that can be associated with a constant argument to a function.  This
 ** allows functions such as "regexp" to compile their constant regular
@@ -18626,19 +18740,50 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(v
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** Utility functions used throughout sqlite.
 **
 ** This file contains functions for allocating memory, comparing
 ** strings, and stuff like that.
 **
-** $Id: util.c,v 1.245 2008/12/10 22:15:00 drh Exp $
-*/
-
+** $Id: util.c,v 1.246 2009/01/10 16:15:22 drh Exp $
+*/
+
+
+/*
+** Routine needed to support the testcase() macro.
+*/
+#ifdef SQLITE_COVERAGE_TEST
+SQLITE_PRIVATE void sqlite3Coverage(int x){
+  static int dummy = 0;
+  dummy += x;
+}
+#endif
+
+/*
+** Routine needed to support the ALWAYS() and NEVER() macros.
+**
+** The argument to ALWAYS() should always be true and the argument
+** to NEVER() should always be false.  If either is not the case
+** then this routine is called in order to throw an error.
+**
+** This routine only exists if assert() is operational.  It always
+** throws an assert on its first invocation.  The variable has a long
+** name to help the assert() message be more readable.  The variable
+** is used to prevent a too-clever optimizer from optimizing out the
+** entire call.
+*/
+#ifndef NDEBUG
+SQLITE_PRIVATE int sqlite3Assert(void){
+  static volatile int ALWAYS_was_false_or_NEVER_was_true = 0;
+  assert( ALWAYS_was_false_or_NEVER_was_true );      /* Always fails */
+  return ALWAYS_was_false_or_NEVER_was_true++;       /* Not Reached */
+}
+#endif
 
 /*
 ** Return true if the floating point value is Not a Number (NaN).
 */
 SQLITE_PRIVATE int sqlite3IsNaN(double x){
   /* This NaN test sometimes fails if compiled on GCC with -ffast-math.
   ** On the other hand, the use of -ffast-math comes with the following
   ** warning:
@@ -19586,17 +19731,17 @@ SQLITE_PRIVATE int sqlite3SafetyCheckSic
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This is the implementation of generic hash-tables
 ** used in SQLite.
 **
-** $Id: hash.c,v 1.32 2008/12/10 19:26:24 drh Exp $
+** $Id: hash.c,v 1.33 2009/01/09 01:12:28 drh Exp $
 */
 
 /* Turn bulk memory into a hash table object by initializing the
 ** fields of the Hash structure.
 **
 ** "pNew" is a pointer to the hash table that is to be initialized.
 ** "copyKey" is true if the hash table should make its own private
 ** copy of keys and false if it should just use the supplied pointer.
@@ -19620,17 +19765,17 @@ SQLITE_PRIVATE void sqlite3HashClear(Has
   assert( pH!=0 );
   elem = pH->first;
   pH->first = 0;
   sqlite3_free(pH->ht);
   pH->ht = 0;
   pH->htsize = 0;
   while( elem ){
     HashElem *next_elem = elem->next;
-    if( pH->copyKey && elem->pKey ){
+    if( pH->copyKey ){
       sqlite3_free(elem->pKey);
     }
     sqlite3_free(elem);
     elem = next_elem;
   }
   pH->count = 0;
 }
 
@@ -19887,153 +20032,155 @@ SQLITE_PRIVATE void *sqlite3HashInsert(H
 SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
  static const char *const azName[] = { "?",
      /*   1 */ "VNext",
      /*   2 */ "Affinity",
      /*   3 */ "Column",
      /*   4 */ "SetCookie",
      /*   5 */ "Seek",
      /*   6 */ "Sequence",
-     /*   7 */ "RowKey",
-     /*   8 */ "SCopy",
-     /*   9 */ "OpenWrite",
-     /*  10 */ "If",
-     /*  11 */ "VRowid",
-     /*  12 */ "CollSeq",
-     /*  13 */ "OpenRead",
-     /*  14 */ "Expire",
-     /*  15 */ "AutoCommit",
-     /*  16 */ "Not",
+     /*   7 */ "Savepoint",
+     /*   8 */ "RowKey",
+     /*   9 */ "SCopy",
+     /*  10 */ "OpenWrite",
+     /*  11 */ "If",
+     /*  12 */ "VRowid",
+     /*  13 */ "CollSeq",
+     /*  14 */ "OpenRead",
+     /*  15 */ "Expire",
+     /*  16 */ "AutoCommit",
      /*  17 */ "Pagecount",
      /*  18 */ "IntegrityCk",
-     /*  19 */ "Sort",
-     /*  20 */ "Copy",
-     /*  21 */ "Trace",
-     /*  22 */ "Function",
-     /*  23 */ "IfNeg",
-     /*  24 */ "Noop",
-     /*  25 */ "Return",
-     /*  26 */ "NewRowid",
-     /*  27 */ "Variable",
-     /*  28 */ "String",
-     /*  29 */ "RealAffinity",
-     /*  30 */ "VRename",
-     /*  31 */ "ParseSchema",
-     /*  32 */ "VOpen",
-     /*  33 */ "Close",
-     /*  34 */ "CreateIndex",
-     /*  35 */ "IsUnique",
-     /*  36 */ "NotFound",
-     /*  37 */ "Int64",
-     /*  38 */ "MustBeInt",
-     /*  39 */ "Halt",
-     /*  40 */ "Rowid",
-     /*  41 */ "IdxLT",
-     /*  42 */ "AddImm",
-     /*  43 */ "Statement",
-     /*  44 */ "RowData",
-     /*  45 */ "MemMax",
-     /*  46 */ "NotExists",
-     /*  47 */ "Gosub",
-     /*  48 */ "Integer",
-     /*  49 */ "Prev",
-     /*  50 */ "RowSetRead",
-     /*  51 */ "RowSetAdd",
-     /*  52 */ "VColumn",
-     /*  53 */ "CreateTable",
-     /*  54 */ "Last",
-     /*  55 */ "SeekLe",
-     /*  56 */ "IncrVacuum",
-     /*  57 */ "IdxRowid",
-     /*  58 */ "ResetCount",
-     /*  59 */ "ContextPush",
-     /*  60 */ "Or",
-     /*  61 */ "And",
-     /*  62 */ "Yield",
-     /*  63 */ "DropTrigger",
-     /*  64 */ "DropIndex",
-     /*  65 */ "IsNull",
-     /*  66 */ "NotNull",
-     /*  67 */ "Ne",
-     /*  68 */ "Eq",
-     /*  69 */ "Gt",
-     /*  70 */ "Le",
-     /*  71 */ "Lt",
-     /*  72 */ "Ge",
-     /*  73 */ "IdxGE",
-     /*  74 */ "BitAnd",
-     /*  75 */ "BitOr",
-     /*  76 */ "ShiftLeft",
-     /*  77 */ "ShiftRight",
-     /*  78 */ "Add",
-     /*  79 */ "Subtract",
-     /*  80 */ "Multiply",
-     /*  81 */ "Divide",
-     /*  82 */ "Remainder",
-     /*  83 */ "Concat",
-     /*  84 */ "IdxDelete",
-     /*  85 */ "Vacuum",
-     /*  86 */ "IfNot",
-     /*  87 */ "BitNot",
-     /*  88 */ "String8",
-     /*  89 */ "DropTable",
-     /*  90 */ "SeekLt",
-     /*  91 */ "MakeRecord",
-     /*  92 */ "ResultRow",
-     /*  93 */ "Delete",
-     /*  94 */ "AggFinal",
-     /*  95 */ "Compare",
-     /*  96 */ "Goto",
-     /*  97 */ "TableLock",
-     /*  98 */ "Clear",
-     /*  99 */ "VerifyCookie",
-     /* 100 */ "AggStep",
-     /* 101 */ "SetNumColumns",
-     /* 102 */ "Transaction",
-     /* 103 */ "VFilter",
-     /* 104 */ "VDestroy",
-     /* 105 */ "ContextPop",
-     /* 106 */ "Next",
-     /* 107 */ "IdxInsert",
-     /* 108 */ "SeekGe",
-     /* 109 */ "Insert",
-     /* 110 */ "Destroy",
-     /* 111 */ "ReadCookie",
-     /* 112 */ "LoadAnalysis",
-     /* 113 */ "Explain",
-     /* 114 */ "OpenPseudo",
-     /* 115 */ "OpenEphemeral",
-     /* 116 */ "Null",
-     /* 117 */ "Move",
-     /* 118 */ "Blob",
-     /* 119 */ "Rewind",
-     /* 120 */ "SeekGt",
-     /* 121 */ "VBegin",
-     /* 122 */ "VUpdate",
-     /* 123 */ "IfZero",
-     /* 124 */ "VCreate",
-     /* 125 */ "Found",
-     /* 126 */ "Real",
+     /*  19 */ "Not",
+     /*  20 */ "Sort",
+     /*  21 */ "Copy",
+     /*  22 */ "Trace",
+     /*  23 */ "Function",
+     /*  24 */ "IfNeg",
+     /*  25 */ "Noop",
+     /*  26 */ "Return",
+     /*  27 */ "NewRowid",
+     /*  28 */ "Variable",
+     /*  29 */ "String",
+     /*  30 */ "RealAffinity",
+     /*  31 */ "VRename",
+     /*  32 */ "ParseSchema",
+     /*  33 */ "VOpen",
+     /*  34 */ "Close",
+     /*  35 */ "CreateIndex",
+     /*  36 */ "IsUnique",
+     /*  37 */ "NotFound",
+     /*  38 */ "Int64",
+     /*  39 */ "MustBeInt",
+     /*  40 */ "Halt",
+     /*  41 */ "Rowid",
+     /*  42 */ "IdxLT",
+     /*  43 */ "AddImm",
+     /*  44 */ "Statement",
+     /*  45 */ "RowData",
+     /*  46 */ "MemMax",
+     /*  47 */ "NotExists",
+     /*  48 */ "Gosub",
+     /*  49 */ "Integer",
+     /*  50 */ "Prev",
+     /*  51 */ "RowSetRead",
+     /*  52 */ "RowSetAdd",
+     /*  53 */ "VColumn",
+     /*  54 */ "CreateTable",
+     /*  55 */ "Last",
+     /*  56 */ "SeekLe",
+     /*  57 */ "IncrVacuum",
+     /*  58 */ "IdxRowid",
+     /*  59 */ "ResetCount",
+     /*  60 */ "ContextPush",
+     /*  61 */ "Yield",
+     /*  62 */ "DropTrigger",
+     /*  63 */ "Or",
+     /*  64 */ "And",
+     /*  65 */ "DropIndex",
+     /*  66 */ "IdxGE",
+     /*  67 */ "IdxDelete",
+     /*  68 */ "IsNull",
+     /*  69 */ "NotNull",
+     /*  70 */ "Ne",
+     /*  71 */ "Eq",
+     /*  72 */ "Gt",
+     /*  73 */ "Le",
+     /*  74 */ "Lt",
+     /*  75 */ "Ge",
+     /*  76 */ "Vacuum",
+     /*  77 */ "BitAnd",
+     /*  78 */ "BitOr",
+     /*  79 */ "ShiftLeft",
+     /*  80 */ "ShiftRight",
+     /*  81 */ "Add",
+     /*  82 */ "Subtract",
+     /*  83 */ "Multiply",
+     /*  84 */ "Divide",
+     /*  85 */ "Remainder",
+     /*  86 */ "Concat",
+     /*  87 */ "IfNot",
+     /*  88 */ "DropTable",
+     /*  89 */ "SeekLt",
+     /*  90 */ "BitNot",
+     /*  91 */ "String8",
+     /*  92 */ "MakeRecord",
+     /*  93 */ "ResultRow",
+     /*  94 */ "Delete",
+     /*  95 */ "AggFinal",
+     /*  96 */ "Compare",
+     /*  97 */ "Goto",
+     /*  98 */ "TableLock",
+     /*  99 */ "Clear",
+     /* 100 */ "VerifyCookie",
+     /* 101 */ "AggStep",
+     /* 102 */ "SetNumColumns",
+     /* 103 */ "Transaction",
+     /* 104 */ "VFilter",
+     /* 105 */ "VDestroy",
+     /* 106 */ "ContextPop",
+     /* 107 */ "Next",
+     /* 108 */ "IdxInsert",
+     /* 109 */ "SeekGe",
+     /* 110 */ "Insert",
+     /* 111 */ "Destroy",
+     /* 112 */ "ReadCookie",
+     /* 113 */ "LoadAnalysis",
+     /* 114 */ "Explain",
+     /* 115 */ "OpenPseudo",
+     /* 116 */ "OpenEphemeral",
+     /* 117 */ "Null",
+     /* 118 */ "Move",
+     /* 119 */ "Blob",
+     /* 120 */ "Rewind",
+     /* 121 */ "SeekGt",
+     /* 122 */ "VBegin",
+     /* 123 */ "VUpdate",
+     /* 124 */ "IfZero",
+     /* 125 */ "VCreate",
+     /* 126 */ "Found",
      /* 127 */ "IfPos",
      /* 128 */ "NullRow",
-     /* 129 */ "Jump",
-     /* 130 */ "Permutation",
-     /* 131 */ "NotUsed_131",
+     /* 129 */ "Real",
+     /* 130 */ "Jump",
+     /* 131 */ "Permutation",
      /* 132 */ "NotUsed_132",
      /* 133 */ "NotUsed_133",
      /* 134 */ "NotUsed_134",
      /* 135 */ "NotUsed_135",
      /* 136 */ "NotUsed_136",
      /* 137 */ "NotUsed_137",
      /* 138 */ "NotUsed_138",
-     /* 139 */ "ToText",
-     /* 140 */ "ToBlob",
-     /* 141 */ "ToNumeric",
-     /* 142 */ "ToInt",
-     /* 143 */ "ToReal",
+     /* 139 */ "NotUsed_139",
+     /* 140 */ "NotUsed_140",
+     /* 141 */ "ToText",
+     /* 142 */ "ToBlob",
+     /* 143 */ "ToNumeric",
+     /* 144 */ "ToInt",
+     /* 145 */ "ToReal",
   };
   return azName[i];
 }
 #endif
 
 /************** End of opcodes.c *********************************************/
 /************** Begin file os_os2.c ******************************************/
 /*
@@ -21437,17 +21584,17 @@ SQLITE_API int sqlite3_os_end(void){
 **   *  sqlite3_file methods not associated with locking.
 **   *  Definitions of sqlite3_io_methods objects for all locking
 **      methods plus "finder" functions for each locking method.
 **   *  sqlite3_vfs method implementations.
 **   *  Locking primitives for the proxy uber-locking-method. (MacOSX only)
 **   *  Definitions of sqlite3_vfs objects for all locking methods
 **      plus implementations of sqlite3_os_init() and sqlite3_os_end().
 **
-** $Id: os_unix.c,v 1.232 2008/12/11 02:56:07 drh Exp $
+** $Id: os_unix.c,v 1.237 2009/01/15 04:30:03 drh Exp $
 */
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
 **
 **   1. POSIX locking (the default),
@@ -21459,17 +21606,17 @@ SQLITE_API int sqlite3_os_end(void){
 **   7. proxy locking. (OSX only)
 **
 ** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE
 ** is defined to 1.  The SQLITE_ENABLE_LOCKING_STYLE also enables automatic
 ** selection of the appropriate locking style based on the filesystem
 ** where the database is located.  
 */
 #if !defined(SQLITE_ENABLE_LOCKING_STYLE)
-#  if defined(__DARWIN__)
+#  if defined(__APPLE__)
 #    define SQLITE_ENABLE_LOCKING_STYLE 1
 #  else
 #    define SQLITE_ENABLE_LOCKING_STYLE 0
 #  endif
 #endif
 
 /*
 ** Define the OS_VXWORKS pre-processor macro to 1 if building on 
@@ -21582,16 +21729,28 @@ struct unixFile {
   int openFlags;                   /* The flags specified at open */
 #if SQLITE_THREADSAFE && defined(__linux__)
   pthread_t tid;                   /* The thread that "owns" this unixFile */
 #endif
 #if OS_VXWORKS
   int isDelete;                    /* Delete on close if true */
   struct vxworksFileId *pId;       /* Unique file ID */
 #endif
+#ifndef NDEBUG
+  /* The next group of variables are used to track whether or not the
+  ** transaction counter in bytes 24-27 of database files are updated
+  ** whenever any part of the database changes.  An assertion fault will
+  ** occur if a file is updated without also updating the transaction
+  ** counter.  This test is made to avoid new problems similar to the
+  ** one described by ticket #3584. 
+  */
+  unsigned char transCntrChng;   /* True if the transaction counter changed */
+  unsigned char dbUpdate;        /* True if any part of database file changed */
+  unsigned char inNormalWrite;   /* True if in a normal write operation */
+#endif
 #ifdef SQLITE_TEST
   /* In test mode, increase the size of this structure a bit so that 
   ** it is larger than the struct CrashFile defined in test6.c.
   */
   char aPadding[32];
 #endif
 };
 
@@ -22941,16 +23100,34 @@ static int unixLock(sqlite3_file *id, in
       int tErrno = errno;
       rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK);
       if( IS_LOCK_ERROR(rc) ){
         pFile->lastErrno = tErrno;
       }
     }
   }
   
+
+#ifndef NDEBUG
+  /* Set up the transaction-counter change checking flags when
+  ** transitioning from a SHARED to a RESERVED lock.  The change
+  ** from SHARED to RESERVED marks the beginning of a normal
+  ** write operation (not a hot journal rollback).
+  */
+  if( rc==SQLITE_OK
+   && pFile->locktype<=SHARED_LOCK
+   && locktype==RESERVED_LOCK
+  ){
+    pFile->transCntrChng = 0;
+    pFile->dbUpdate = 0;
+    pFile->inNormalWrite = 1;
+  }
+#endif
+
+
   if( rc==SQLITE_OK ){
     pFile->locktype = locktype;
     pLock->locktype = locktype;
   }else if( locktype==EXCLUSIVE_LOCK ){
     pFile->locktype = PENDING_LOCK;
     pLock->locktype = PENDING_LOCK;
   }
 
@@ -22990,16 +23167,33 @@ static int unixUnlock(sqlite3_file *id, 
   h = pFile->h;
   pLock = pFile->pLock;
   assert( pLock->cnt!=0 );
   if( pFile->locktype>SHARED_LOCK ){
     assert( pLock->locktype==pFile->locktype );
     SimulateIOErrorBenign(1);
     SimulateIOError( h=(-1) )
     SimulateIOErrorBenign(0);
+
+#ifndef NDEBUG
+    /* When reducing a lock such that other processes can start
+    ** reading the database file again, make sure that the
+    ** transaction counter was updated if any part of the database
+    ** file changed.  If the transaction counter is not updated,
+    ** other connections to the same file might not realize that
+    ** the file has changed and hence might not know to flush their
+    ** cache.  The use of a stale cache can lead to database corruption.
+    */
+    assert( pFile->inNormalWrite==0
+         || pFile->dbUpdate==0
+         || pFile->transCntrChng==1 );
+    pFile->inNormalWrite = 0;
+#endif
+
+
     if( locktype==SHARED_LOCK ){
       lock.l_type = F_RDLCK;
       lock.l_whence = SEEK_SET;
       lock.l_start = SHARED_FIRST;
       lock.l_len = SHARED_SIZE;
       if( fcntl(h, F_SETLK, &lock)==(-1) ){
         int tErrno = errno;
         rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK);
@@ -23797,17 +23991,17 @@ static int semClose(sqlite3_file *id) {
 **
 ** AFP is the Apple Filing Protocol.  AFP is a network filesystem found
 ** on Apple Macintosh computers - both OS9 and OSX.
 **
 ** Third-party implementations of AFP are available.  But this code here
 ** only works on OSX.
 */
 
-#if defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
 /*
 ** The afpLockingContext structure contains all afp lock specific state
 */
 typedef struct afpLockingContext afpLockingContext;
 struct afpLockingContext {
   unsigned long long sharedByte;
   const char *dbPath;             /* Name of the open file */
 };
@@ -24180,17 +24374,17 @@ static int afpClose(sqlite3_file *id) {
     releaseOpenCnt(pFile->pOpen);
     sqlite3_free(pFile->lockingContext);
     closeUnixFile(id);
     unixLeaveMutex();
   }
   return SQLITE_OK;
 }
 
-#endif /* defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE */
+#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */
 /*
 ** The code above is the AFP lock implementation.  The code is specific
 ** to MacOSX and does not work on other unix platforms.  No alternative
 ** is available.  If you don't compile for a mac, then the "unix-afp"
 ** VFS is not available.
 **
 ********************* End of the AFP lock implementation **********************
 ******************************************************************************/
@@ -24322,16 +24516,39 @@ static int unixWrite(
   sqlite3_file *id, 
   const void *pBuf, 
   int amt,
   sqlite3_int64 offset 
 ){
   int wrote = 0;
   assert( id );
   assert( amt>0 );
+
+#ifndef NDEBUG
+  /* If we are doing a normal write to a database file (as opposed to
+  ** doing a hot-journal rollback or a write to some file other than a
+  ** normal database file) then record the fact that the database
+  ** has changed.  If the transaction counter is modified, record that
+  ** fact too.
+  */
+  if( ((unixFile*)id)->inNormalWrite ){
+    unixFile *pFile = (unixFile*)id;
+    pFile->dbUpdate = 1;  /* The database has been modified */
+    if( offset<=24 && offset+amt>=27 ){
+      char oldCntr[4];
+      SimulateIOErrorBenign(1);
+      seekAndRead(pFile, 24, oldCntr, 4);
+      SimulateIOErrorBenign(0);
+      if( memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){
+        pFile->transCntrChng = 1;  /* The transaction counter has changed */
+      }
+    }
+  }
+#endif
+
   while( amt>0 && (wrote = seekAndWrite((unixFile*)id, offset, pBuf, amt))>0 ){
     amt -= wrote;
     offset += wrote;
     pBuf = &((char*)pBuf)[wrote];
   }
   SimulateIOError(( wrote=(-1), amt=1 ));
   SimulateDiskfullError(( wrote=0, amt=1 ));
   if( amt>0 ){
@@ -24431,19 +24648,21 @@ static int full_fsync(int fd, int fullSy
   ** It'd be better to detect fullfsync support once and avoid 
   ** the fcntl call every time sync is called.
   */
   if( rc ) rc = fsync(fd);
 
 #else 
   if( dataOnly ){
     rc = fdatasync(fd);
-    if( OS_VXWORKS && rc==-1 && errno==ENOTSUP ){
+#if OS_VXWORKS
+    if( rc==-1 && errno==ENOTSUP ){
       rc = fsync(fd);
     }
+#endif
   }else{
     rc = fsync(fd);
   }
 #endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */
 
   if( OS_VXWORKS && rc!= -1 ){
     rc = 0;
   }
@@ -24559,17 +24778,17 @@ static int unixFileSize(sqlite3_file *id
   ** really 1.   Ticket #3260.
   */
   if( *pSize==1 ) *pSize = 0;
 
 
   return SQLITE_OK;
 }
 
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
 /*
 ** Handler for proxy-locking file-control verbs.  Defined below in the
 ** proxying locking division.
 */
 static int proxyFileControl(sqlite3_file*,int,void*);
 #endif
 
 
@@ -24581,22 +24800,33 @@ static int unixFileControl(sqlite3_file 
     case SQLITE_FCNTL_LOCKSTATE: {
       *(int*)pArg = ((unixFile*)id)->locktype;
       return SQLITE_OK;
     }
     case SQLITE_LAST_ERRNO: {
       *(int*)pArg = ((unixFile*)id)->lastErrno;
       return SQLITE_OK;
     }
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#ifndef NDEBUG
+    /* The pager calls this method to signal that it has done
+    ** a rollback and that the database is therefore unchanged and
+    ** it hence it is OK for the transaction change counter to be
+    ** unchanged.
+    */
+    case SQLITE_FCNTL_DB_UNCHANGED: {
+      ((unixFile*)id)->dbUpdate = 0;
+      return SQLITE_OK;
+    }
+#endif
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
     case SQLITE_SET_LOCKPROXYFILE:
     case SQLITE_GET_LOCKPROXYFILE: {
       return proxyFileControl(id,op,pArg);
     }
-#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__) */
+#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */
   }
   return SQLITE_ERROR;
 }
 
 /*
 ** Return the sector size in bytes of the underlying block device for
 ** the specified file. This is almost always 512 bytes, but may be
 ** larger for some devices.
@@ -24729,17 +24959,17 @@ IOMETHODS(
   semIoMethods,             /* sqlite3_io_methods object name */
   semClose,                 /* xClose method */
   semLock,                  /* xLock method */
   semUnlock,                /* xUnlock method */
   semCheckReservedLock      /* xCheckReservedLock method */
 )
 #endif
 
-#if defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
 IOMETHODS(
   afpIoFinder,              /* Finder function name */
   afpIoMethods,             /* sqlite3_io_methods object name */
   afpClose,                 /* xClose method */
   afpLock,                  /* xLock method */
   afpUnlock,                /* xUnlock method */
   afpCheckReservedLock      /* xCheckReservedLock method */
 )
@@ -24749,33 +24979,33 @@ IOMETHODS(
 ** The proxy locking method is a "super-method" in the sense that it
 ** opens secondary file descriptors for the conch and lock files and
 ** it uses proxy, dot-file, AFP, and flock() locking methods on those
 ** secondary files.  For this reason, the division that implements
 ** proxy locking is located much further down in the file.  But we need
 ** to go ahead and define the sqlite3_io_methods and finder function
 ** for proxy locking here.  So we forward declare the I/O methods.
 */
-#if defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
 static int proxyClose(sqlite3_file*);
 static int proxyLock(sqlite3_file*, int);
 static int proxyUnlock(sqlite3_file*, int);
 static int proxyCheckReservedLock(sqlite3_file*, int*);
 IOMETHODS(
   proxyIoFinder,            /* Finder function name */
   proxyIoMethods,           /* sqlite3_io_methods object name */
   proxyClose,               /* xClose method */
   proxyLock,                /* xLock method */
   proxyUnlock,              /* xUnlock method */
   proxyCheckReservedLock    /* xCheckReservedLock method */
 )
 #endif
 
 
-#if defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
 /* 
 ** This "finder" function attempts to determine the best locking strategy 
 ** for the database file "filePath".  It then returns the sqlite3_io_methods
 ** object that implements that strategy.
 **
 ** This is for MacOSX only.
 */
 static const sqlite3_io_methods *autolockIoFinderImpl(
@@ -24826,20 +25056,20 @@ static const sqlite3_io_methods *autoloc
   lockInfo.l_whence = SEEK_SET;
   lockInfo.l_type = F_RDLCK;
   if( fcntl(fd, F_GETLK, &lockInfo)!=-1 ) {
     return &posixIoMethods;
   }else{
     return &dotlockIoMethods;
   }
 }
-static const sqlite3_io_methods (*const autolockIoFinder)(const char*,int)
+static const sqlite3_io_methods *(*const autolockIoFinder)(const char*,int)
         = autolockIoFinderImpl;
 
-#endif /* defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE */
+#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */
 
 /*
 ** An abstract type for a pointer to a IO method finder function:
 */
 typedef const sqlite3_io_methods *(*finder_type)(const char*,int);
 
 
 /****************************************************************************
@@ -24902,17 +25132,17 @@ static int fillInUnixFile(
   }
 
   if( pLockingStyle == &posixIoMethods ){
     unixEnterMutex();
     rc = findLockInfo(pNew, &pNew->pLock, &pNew->pOpen);
     unixLeaveMutex();
   }
 
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
   else if( pLockingStyle == &afpIoMethods ){
     /* AFP locking uses the file path so it needs to be included in
     ** the afpLockingContext.
     */
     afpLockingContext *pCtx;
     pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) );
     if( pCtx==0 ){
       rc = SQLITE_NOMEM;
@@ -24998,17 +25228,17 @@ static int fillInUnixFile(
 ** the file descriptor *pFd using close().
 */
 static int openDirectory(const char *zFilename, int *pFd){
   int ii;
   int fd = -1;
   char zDirname[MAX_PATHNAME+1];
 
   sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename);
-  for(ii=(int)strlen(zDirname); ii>=0 && zDirname[ii]!='/'; ii--);
+  for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--);
   if( ii>0 ){
     zDirname[ii] = '\0';
     fd = open(zDirname, O_RDONLY|O_BINARY, 0);
     if( fd>=0 ){
 #ifdef FD_CLOEXEC
       fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC);
 #endif
       OSTRACE3("OPENDIR %-3d %s\n", fd, zDirname);
@@ -25074,17 +25304,17 @@ static int getTempname(int nBuf, char *z
     for(i=0; i<15; i++, j++){
       zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ];
     }
     zBuf[j] = 0;
   }while( access(zBuf,0)==0 );
   return SQLITE_OK;
 }
 
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
 /*
 ** Routine to transform a unixFile into a proxy-locking unixFile.
 ** Implementation in the proxy-lock division, but used by unixOpen()
 ** if SQLITE_PREFER_PROXY_LOCKING is defined.
 */
 static int proxyTransformUnixFile(unixFile*, const char*);
 #endif
 
@@ -25721,17 +25951,17 @@ static int unixGetLastError(sqlite3_vfs 
 ** will force automatic proxy locking to be disabled for all database
 ** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or
 ** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING).
 */
 
 /*
 ** Proxy locking is only available on MacOSX 
 */
-#if defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
 
 #ifdef SQLITE_TEST
 /* simulate multiple hosts by creating unique hostid file paths */
 SQLITE_API int sqlite3_hostid_num = 0;
 #endif
 
 /*
 ** The proxyLockingContext has the path and file structures for the remote 
@@ -26211,17 +26441,17 @@ static int switchLockProxyPath(unixFile 
 /*
 ** pFile is a file that has been opened by a prior xOpen call.  dbPath
 ** is a string buffer at least MAXPATHLEN+1 characters in size.
 **
 ** This routine find the filename associated with pFile and writes it
 ** int dbPath.
 */
 static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){
-#if defined(__DARWIN__)
+#if defined(__APPLE__)
   if( pFile->pMethod == &afpIoMethods ){
     /* afp style keeps a reference to the db path in the filePath field 
     ** of the struct */
     assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN );
     strcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath);
   }else
 #endif
   if( pFile->pMethod == &dotlockIoMethods ){
@@ -26480,17 +26710,17 @@ static int proxyClose(sqlite3_file *id) 
     sqlite3_free(pCtx);
     return pFile->pMethod->xClose(id);
   }
   return SQLITE_OK;
 }
 
 
 
-#endif /* defined(__DARWIN__) && SQLITE_ENABLE_LOCKING_STYLE */
+#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */
 /*
 ** The proxy locking style is intended for use with AFP filesystems.
 ** And since AFP is only supported on MacOSX, the proxy locking is also
 ** restricted to MacOSX.
 ** 
 **
 ******************* End of the proxy lock implementation **********************
 ******************************************************************************/
@@ -26553,31 +26783,31 @@ SQLITE_API int sqlite3_os_init(void){
   /*
   ** All default VFSes for unix are contained in the following array.
   **
   ** Note that the sqlite3_vfs.pNext field of the VFS object is modified
   ** by the SQLite core when the VFS is registered.  So the following
   ** array cannot be const.
   */
   static sqlite3_vfs aVfs[] = {
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
     UNIXVFS("unix",          autolockIoFinder ),
 #else
     UNIXVFS("unix",          posixIoFinder ),
 #endif
     UNIXVFS("unix-none",     nolockIoFinder ),
     UNIXVFS("unix-dotfile",  dotlockIoFinder ),
 #if OS_VXWORKS
     UNIXVFS("unix-namedsem", semIoFinder ),
 #endif
 #if SQLITE_ENABLE_LOCKING_STYLE
     UNIXVFS("unix-posix",    posixIoFinder ),
     UNIXVFS("unix-flock",    flockIoFinder ),
 #endif
-#if SQLITE_ENABLE_LOCKING_STYLE && defined(__DARWIN__)
+#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
     UNIXVFS("unix-afp",      afpIoFinder ),
     UNIXVFS("unix-proxy",    proxyIoFinder ),
 #endif
   };
   unsigned int i;          /* Loop counter */
 
   /* Register all VFSes defined in the aVfs[] array */
   for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
@@ -28589,17 +28819,17 @@ SQLITE_API int sqlite3_os_end(void){
 ** Test operations are about 100 times more common that set operations.
 ** Clear operations are exceedingly rare.  There are usually between
 ** 5 and 500 set operations per Bitvec object, though the number of sets can
 ** sometimes grow into tens of thousands or larger.  The size of the
 ** Bitvec object is the number of pages in the database file at the
 ** start of a transaction, and is thus usually less than a few thousand,
 ** but can be as large as 2 billion for a really big database.
 **
-** @(#) $Id: bitvec.c,v 1.9 2008/11/19 18:30:35 shane Exp $
+** @(#) $Id: bitvec.c,v 1.10 2009/01/02 21:39:39 drh Exp $
 */
 
 /* Size of the Bitvec structure in bytes. */
 #define BITVEC_SZ        512
 
 /* Round the union size down to the nearest pointer boundary, since that's how 
 ** it will be aligned within the Bitvec struct. */
 #define BITVEC_USIZE     (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*))
@@ -28728,19 +28958,17 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitv
   assert( p!=0 );
   assert( i>0 );
   assert( i<=p->iSize );
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {
     u32 bin = i/p->iDivisor;
     i = i%p->iDivisor;
     if( p->u.apSub[bin]==0 ){
-      sqlite3BeginBenignMalloc();
       p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor );
-      sqlite3EndBenignMalloc();
       if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM;
     }
     p = p->u.apSub[bin];
   }
   if( p->iSize<=BITVEC_NBIT ){
     p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1));
     return SQLITE_OK;
   }
@@ -29543,17 +29771,17 @@ SQLITE_PRIVATE void sqlite3PcacheIterate
 *************************************************************************
 **
 ** This file implements the default page cache implementation (the
 ** sqlite3_pcache interface). It also contains part of the implementation
 ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
 ** If the default page cache implementation is overriden, then neither of
 ** these two features are available.
 **
-** @(#) $Id: pcache1.c,v 1.6 2008/12/10 18:03:46 drh Exp $
+** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $
 */
 
 
 typedef struct PCache1 PCache1;
 typedef struct PgHdr1 PgHdr1;
 typedef struct PgFreeslot PgFreeslot;
 
 /* Pointers to structures of this type are cast and returned as 
@@ -29573,16 +29801,18 @@ struct PCache1 {
   /* Hash table of all pages. The following variables may only be accessed
   ** when the accessor is holding the global mutex (see pcache1EnterMutex() 
   ** and pcache1LeaveMutex()).
   */
   unsigned int nRecyclable;           /* Number of pages in the LRU list */
   unsigned int nPage;                 /* Total number of pages in apHash */
   unsigned int nHash;                 /* Number of slots in apHash[] */
   PgHdr1 **apHash;                    /* Hash table for fast lookup by key */
+
+  unsigned int iMaxKey;               /* Largest key seen since xTruncate() */
 };
 
 /*
 ** Each cache entry is represented by an instance of the following 
 ** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated 
 ** directly after the structure in memory (see the PGHDR1_TO_PAGE() 
 ** macro below).
 */
@@ -30083,16 +30313,19 @@ static void *pcache1Fetch(sqlite3_pcache
     pCache->nPage++;
     pPage->iKey = iKey;
     pPage->pNext = pCache->apHash[h];
     pPage->pCache = pCache;
     pCache->apHash[h] = pPage;
   }
 
 fetch_out:
+  if( pPage && iKey>pCache->iMaxKey ){
+    pCache->iMaxKey = iKey;
+  }
   if( createFlag==1 ) sqlite3EndBenignMalloc();
   pcache1LeaveMutex();
   return (pPage ? PGHDR1_TO_PAGE(pPage) : 0);
 }
 
 
 /*
 ** Implementation of the sqlite3_pcache.xUnpin method.
@@ -30158,30 +30391,37 @@ static void pcache1Rekey(
   }
   *pp = pPage->pNext;
 
   h = iNew%pCache->nHash;
   pPage->iKey = iNew;
   pPage->pNext = pCache->apHash[h];
   pCache->apHash[h] = pPage;
 
+  if( iNew>pCache->iMaxKey ){
+    pCache->iMaxKey = iNew;
+  }
+
   pcache1LeaveMutex();
 }
 
 /*
 ** Implementation of the sqlite3_pcache.xTruncate method. 
 **
 ** Discard all unpinned pages in the cache with a page number equal to
 ** or greater than parameter iLimit. Any pinned pages with a page number
 ** equal to or greater than iLimit are implicitly unpinned.
 */
 static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
   PCache1 *pCache = (PCache1 *)p;
   pcache1EnterMutex();
-  pcache1TruncateUnsafe(pCache, iLimit);
+  if( iLimit<=pCache->iMaxKey ){
+    pcache1TruncateUnsafe(pCache, iLimit);
+    pCache->iMaxKey = iLimit-1;
+  }
   pcache1LeaveMutex();
 }
 
 /*
 ** Implementation of the sqlite3_pcache.xDestroy method. 
 **
 ** Destroy a cache allocated using pcache1Create().
 */
@@ -30292,16 +30532,18 @@ SQLITE_PRIVATE void sqlite3PcacheStats(
 ** second and subsequent inserts make no difference on the output.
 **
 ** This implementation accumulates rowids in a linked list.  For
 ** output, it first sorts the linked list (removing duplicates during
 ** the sort) then returns elements one by one by walking the list.
 **
 ** Big chunks of rowid/next-ptr pairs are allocated at a time, to
 ** reduce the malloc overhead.
+**
+** $Id: rowset.c,v 1.3 2009/01/13 20:14:16 drh Exp $
 */
 
 /*
 ** The number of rowset entries per allocation chunk.
 */
 #define ROWSET_ENTRY_PER_CHUNK  63
 
 /*
@@ -30340,35 +30582,33 @@ struct RowSet {
 };
 
 /*
 ** Turn bulk memory into a RowSet object.  N bytes of memory
 ** are available at pSpace.  The db pointer is used as a memory context
 ** for any subsequent allocations that need to occur.
 ** Return a pointer to the new RowSet object.
 **
-** If N is not sufficient memory to make even a minimum RowSet,
-** then return NULL.  If N is larger than the minimum, use
-** the surplus as an initial allocation of entries available to
-** be filled.
+** It must be the case that N is sufficient to make a Rowset.  If not
+** an assertion fault occurs.
+** 
+** If N is larger than the minimum, use the surplus as an initial
+** allocation of entries available to be filled.
 */
 SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){
   RowSet *p;
-  if( N<sizeof(*p) ){
-    p = 0;
-  }else{
-    p = pSpace;
-    p->pChunk = 0;
-    p->db = db;
-    p->pEntry = 0;
-    p->pLast = 0;
-    p->pFresh = (struct RowSetEntry*)&p[1];
-    p->nFresh = (u16)((N - sizeof(*p))/sizeof(struct RowSetEntry));
-    p->isSorted = 1;
-  }
+  assert( N >= sizeof(*p) );
+  p = pSpace;
+  p->pChunk = 0;
+  p->db = db;
+  p->pEntry = 0;
+  p->pLast = 0;
+  p->pFresh = (struct RowSetEntry*)&p[1];
+  p->nFresh = (u16)((N - sizeof(*p))/sizeof(struct RowSetEntry));
+  p->isSorted = 1;
   return p;
 }
 
 /*
 ** Deallocate all chunks from a RowSet.
 */
 SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){
   struct RowSetChunk *pChunk, *pNextChunk;
@@ -30526,40 +30766,33 @@ SQLITE_PRIVATE int sqlite3RowSetNext(Row
 ** 
 ** The pager is used to access a database disk file.  It implements
 ** atomic commit and rollback through the use of a journal file that
 ** is separate from the database file.  The pager also implements file
 ** locking to prevent two processes from writing the same database
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.514 2008/12/10 22:15:00 drh Exp $
+** @(#) $Id: pager.c,v 1.551 2009/01/14 23:03:41 drh Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 
 /*
 ** Macros for troubleshooting.  Normally turned off
 */
 #if 0
+int sqlite3PagerTrace=1;  /* True to enable tracing */
 #define sqlite3DebugPrintf printf
-#define PAGERTRACE1(X)       sqlite3DebugPrintf(X)
-#define PAGERTRACE2(X,Y)     sqlite3DebugPrintf(X,Y)
-#define PAGERTRACE3(X,Y,Z)   sqlite3DebugPrintf(X,Y,Z)
-#define PAGERTRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W)
-#define PAGERTRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W,V)
-#else
-#define PAGERTRACE1(X)
-#define PAGERTRACE2(X,Y)
-#define PAGERTRACE3(X,Y,Z)
-#define PAGERTRACE4(X,Y,Z,W)
-#define PAGERTRACE5(X,Y,Z,W,V)
-#endif
-
-/*
-** The following two macros are used within the PAGERTRACEX() macros above
+#define PAGERTRACE(X)     if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; }
+#else
+#define PAGERTRACE(X)
+#endif
+
+/*
+** The following two macros are used within the PAGERTRACE() macros above
 ** to print out file-descriptors. 
 **
 ** PAGERID() takes a pointer to a Pager struct as its argument. The
 ** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file
 ** struct as its argument.
 */
 #define PAGERID(p) ((int)(p->fd))
 #define FILEHANDLEID(fd) ((int)fd)
@@ -30614,33 +30847,16 @@ SQLITE_PRIVATE int sqlite3RowSetNext(Row
 */
 #define PAGER_UNLOCK      0
 #define PAGER_SHARED      1   /* same as SHARED_LOCK */
 #define PAGER_RESERVED    2   /* same as RESERVED_LOCK */
 #define PAGER_EXCLUSIVE   4   /* same as EXCLUSIVE_LOCK */
 #define PAGER_SYNCED      5
 
 /*
-** If the SQLITE_BUSY_RESERVED_LOCK macro is set to true at compile-time,
-** then failed attempts to get a reserved lock will invoke the busy callback.
-** This is off by default.  To see why, consider the following scenario:
-** 
-** Suppose thread A already has a shared lock and wants a reserved lock.
-** Thread B already has a reserved lock and wants an exclusive lock.  If
-** both threads are using their busy callbacks, it might be a long time
-** be for one of the threads give up and allows the other to proceed.
-** But if the thread trying to get the reserved lock gives up quickly
-** (if it never invokes its busy callback) then the contention will be
-** resolved quickly.
-*/
-#ifndef SQLITE_BUSY_RESERVED_LOCK
-# define SQLITE_BUSY_RESERVED_LOCK 0
-#endif
-
-/*
 ** This macro rounds values up so that if the value is an address it
 ** is guaranteed to be an address that is aligned to an 8-byte boundary.
 */
 #define FORCE_ALIGNMENT(X)   (((X)+7)&~7)
 
 /*
 ** A macro used for invoking the codec if there is one
 */
@@ -30648,94 +30864,130 @@ SQLITE_PRIVATE int sqlite3RowSetNext(Row
 # define CODEC1(P,D,N,X) if( P->xCodec!=0 ){ P->xCodec(P->pCodecArg,D,N,X); }
 # define CODEC2(P,D,N,X) ((char*)(P->xCodec!=0?P->xCodec(P->pCodecArg,D,N,X):D))
 #else
 # define CODEC1(P,D,N,X) /* NO-OP */
 # define CODEC2(P,D,N,X) ((char*)D)
 #endif
 
 /*
+** The maximum allowed sector size. 16MB. If the xSectorsize() method 
+** returns a value larger than this, then MAX_SECTOR_SIZE is used instead.
+** This could conceivably cause corruption following a power failure on
+** such a system. This is currently an undocumented limit.
+*/
+#define MAX_SECTOR_SIZE 0x0100000
+
+/*
+** An instance of the following structure is allocated for each active
+** savepoint and statement transaction in the system. All such structures
+** are stored in the Pager.aSavepoint[] array, which is allocated and
+** resized using sqlite3Realloc().
+**
+** When a savepoint is created, the PagerSavepoint.iHdrOffset field is
+** set to 0. If a journal-header is written into the main journal while
+** the savepoint is active, then iHdrOffset is set to the byte offset 
+** immediately following the last journal record written into the main
+** journal before the journal-header. This is required during savepoint
+** rollback (see pagerPlaybackSavepoint()).
+*/
+typedef struct PagerSavepoint PagerSavepoint;
+struct PagerSavepoint {
+  i64 iOffset;                 /* Starting offset in main journal */
+  i64 iHdrOffset;              /* See above */
+  Bitvec *pInSavepoint;        /* Set of pages in this savepoint */
+  Pgno nOrig;                  /* Original number of pages in file */
+  Pgno iSubRec;                /* Index of first record in sub-journal */
+};
+
+/*
 ** A open page cache is an instance of the following structure.
 **
 ** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or
 ** or SQLITE_FULL. Once one of the first three errors occurs, it persists
 ** and is returned as the result of every major pager API call.  The
 ** SQLITE_FULL return code is slightly different. It persists only until the
 ** next successful rollback is performed on the pager cache. Also,
 ** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup()
 ** APIs, they may still be used successfully.
+**
+** Managing the size of the database file in pages is a little complicated.
+** The variable Pager.dbSize contains the number of pages that the database
+** image currently contains. As the database image grows or shrinks this
+** variable is updated. The variable Pager.dbFileSize contains the number
+** of pages in the database file. This may be different from Pager.dbSize
+** if some pages have been appended to the database image but not yet written
+** out from the cache to the actual file on disk. Or if the image has been
+** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable
+** contains the number of pages in the database image when the current
+** transaction was opened. The contents of all three of these variables is
+** only guaranteed to be correct if the boolean Pager.dbSizeValid is true.
 */
 struct Pager {
   sqlite3_vfs *pVfs;          /* OS functions to use for IO */
   u8 journalOpen;             /* True if journal file descriptors is valid */
   u8 journalStarted;          /* True if header of journal is synced */
   u8 useJournal;              /* Use a rollback journal on this file */
   u8 noReadlock;              /* Do not bother to obtain readlocks */
-  u8 stmtOpen;                /* True if the statement subjournal is open */
-  u8 stmtInUse;               /* True we are in a statement subtransaction */
-  u8 stmtAutoopen;            /* Open stmt journal when main journal is opened*/
   u8 noSync;                  /* Do not sync the journal if true */
   u8 fullSync;                /* Do extra syncs of the journal for robustness */
   u8 sync_flags;              /* One of SYNC_NORMAL or SYNC_FULL */
   u8 state;                   /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */
   u8 tempFile;                /* zFilename is a temporary file */
   u8 readOnly;                /* True for a read-only database */
   u8 needSync;                /* True if an fsync() is needed on the journal */
   u8 dirtyCache;              /* True if cached pages have changed */
-  u8 alwaysRollback;          /* Disable DontRollback() for all pages */
   u8 memDb;                   /* True to inhibit all file I/O */
   u8 setMaster;               /* True if a m-j name has been written to jrnl */
   u8 doNotSync;               /* Boolean. While true, do not spill the cache */
   u8 exclusiveMode;           /* Boolean. True if locking_mode==EXCLUSIVE */
   u8 journalMode;             /* On of the PAGER_JOURNALMODE_* values */
   u8 dbModified;              /* True if there are any changes to the Db */
   u8 changeCountDone;         /* Set after incrementing the change-counter */
   u8 dbSizeValid;             /* Set when dbSize is correct */
+  Pgno dbSize;                /* Number of pages in the database */
+  Pgno dbOrigSize;            /* dbSize before the current transaction */
+  Pgno dbFileSize;            /* Number of pages in the database file */
   u32 vfsFlags;               /* Flags for sqlite3_vfs.xOpen() */
   int errCode;                /* One of several kinds of errors */
-  Pgno dbSize;                /* Number of pages in the file */
-  Pgno origDbSize;            /* dbSize before the current change */
-  Pgno stmtSize;              /* Size of database (in pages) at stmt_begin() */
   int nRec;                   /* Number of pages written to the journal */
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   int stmtNRec;               /* Number of records in stmt subjournal */
   int nExtra;                 /* Add this many bytes to each in-memory page */
   int pageSize;               /* Number of bytes in a page */
   int nPage;                  /* Total number of in-memory pages */
   int mxPage;                 /* Maximum number of pages to hold in cache */
   Pgno mxPgno;                /* Maximum allowed size of the database */
   Bitvec *pInJournal;         /* One bit for each page in the database file */
-  Bitvec *pInStmt;            /* One bit for each page in the database */
   Bitvec *pAlwaysRollback;    /* One bit for each page marked always-rollback */
   char *zFilename;            /* Name of the database file */
   char *zJournal;             /* Name of the journal file */
   char *zDirectory;           /* Directory hold database and journal files */
   sqlite3_file *fd, *jfd;     /* File descriptors for database and journal */
-  sqlite3_file *stfd;         /* File descriptor for the statement subjournal*/
+  sqlite3_file *sjfd;         /* File descriptor for the sub-journal*/
   int (*xBusyHandler)(void*); /* Function to call when busy */
   void *pBusyHandlerArg;      /* Context argument for xBusyHandler */
   i64 journalOff;             /* Current byte offset in the journal file */
   i64 journalHdr;             /* Byte offset to previous journal header */
-  i64 stmtHdrOff;             /* First journal header written this statement */
-  i64 stmtCksum;              /* cksumInit when statement was started */
-  i64 stmtJSize;              /* Size of journal at stmt_begin() */
   u32 sectorSize;             /* Assumed sector size during rollback */
 #ifdef SQLITE_TEST
   int nHit, nMiss;            /* Cache hits and missing */
   int nRead, nWrite;          /* Database pages read/written */
 #endif
   void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */
 #ifdef SQLITE_HAS_CODEC
   void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */
   void *pCodecArg;            /* First argument to xCodec() */
 #endif
   char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
   char dbFileVers[16];        /* Changes whenever database file changes */
   i64 journalSizeLimit;       /* Size limit for persistent journal files */
   PCache *pPCache;            /* Pointer to page cache object */
+  PagerSavepoint *aSavepoint; /* Array of active savepoints */
+  int nSavepoint;             /* Number of elements in aSavepoint[] */
 };
 
 /*
 ** The following global variables hold counters used for
 ** testing purposes only.  These variables do not exist in
 ** a non-testing build.  These variables are not thread-safe.
 */
 #ifdef SQLITE_TEST
@@ -30749,17 +31001,17 @@ SQLITE_API int sqlite3_pager_writej_coun
 
 
 
 /*
 ** Journal files begin with the following magic string.  The data
 ** was obtained from /dev/random.  It is used only as a sanity check.
 **
 ** Since version 2.8.0, the journal format contains additional sanity
-** checking information.  If the power fails while the journal is begin
+** checking information.  If the power fails while the journal is being
 ** written, semi-random garbage data might appear in the journal
 ** file after power is restored.  If an attempt is then made
 ** to roll the journal back, the database could be corrupted.  The additional
 ** sanity checking data is an attempt to discover the garbage in the
 ** journal and ignore it.
 **
 ** The sanity checking information for the new journal format consists
 ** of a 32-bit checksum on each page of data.  The checksum covers both
@@ -30812,25 +31064,40 @@ static const unsigned char aJournalMagic
 #define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1))
 
 /*
 ** The maximum legal page number is (2^31 - 1).
 */
 #define PAGER_MAX_PGNO 2147483647
 
 /*
-** Return true if page *pPg has already been written to the statement
-** journal (or statement snapshot has been created, if *pPg is part
-** of an in-memory database).
-*/
-static int pageInStatement(PgHdr *pPg){
+** Return true if it is necessary to write page *pPg into the sub-journal.
+** A page needs to be written into the sub-journal if there exists one
+** or more open savepoints for which:
+**
+**   * The page-number is less than or equal to PagerSavepoint.nOrig, and
+**   * The bit corresponding to the page-number is not set in
+**     PagerSavepoint.pInSavepoint.
+*/
+static int subjRequiresPage(PgHdr *pPg){
+  Pgno pgno = pPg->pgno;
   Pager *pPager = pPg->pPager;
-  return sqlite3BitvecTest(pPager->pInStmt, pPg->pgno);
-}
-
+  int i;
+  for(i=0; i<pPager->nSavepoint; i++){
+    PagerSavepoint *p = &pPager->aSavepoint[i];
+    if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/*
+** Return true if the page is already in the journal file.
+*/
 static int pageInJournal(PgHdr *pPg){
   return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno);
 }
 
 /*
 ** Read a 32-bit integer from the given file descriptor.  Store the integer
 ** that is read in *pRes.  Return SQLITE_OK if everything worked, or an
 ** error code is something goes wrong.
@@ -30888,17 +31155,17 @@ static int osUnlock(sqlite3_file *pFd, i
 static int jrnlBufferSize(Pager *pPager){
   int dc;           /* Device characteristics */
   int nSector;      /* Sector size */
   int szPage;        /* Page size */
   sqlite3_file *fd = pPager->fd;
 
   if( fd->pMethods ){
     dc = sqlite3OsDeviceCharacteristics(fd);
-    nSector = sqlite3OsSectorSize(fd);
+    nSector = pPager->sectorSize;
     szPage = pPager->pageSize;
   }
 
   assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
   assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));
 
   if( !fd->pMethods || 
        (dc & (SQLITE_IOCAP_ATOMIC|(szPage>>8)) && nSector<=szPage) ){
@@ -31067,26 +31334,29 @@ static int readMasterJournal(sqlite3_fil
 ** Input Offset              Output Offset
 ** ---------------------------------------
 ** 0                         0
 ** 512                       512
 ** 100                       512
 ** 2000                      2048
 ** 
 */
-static void seekJournalHdr(Pager *pPager){
+static i64 journalHdrOffset(Pager *pPager){
   i64 offset = 0;
   i64 c = pPager->journalOff;
   if( c ){
     offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager);
   }
   assert( offset%JOURNAL_HDR_SZ(pPager)==0 );
   assert( offset>=c );
   assert( (offset-c)<JOURNAL_HDR_SZ(pPager) );
-  pPager->journalOff = offset;
+  return offset;
+}
+static void seekJournalHdr(Pager *pPager){
+  pPager->journalOff = journalHdrOffset(pPager);
 }
 
 /*
 ** Write zeros over the header of the journal file.  This has the
 ** effect of invalidating the journal file and committing the
 ** transaction.
 */
 static int zeroJournalHdr(Pager *pPager, int doTruncate){
@@ -31138,23 +31408,30 @@ static int zeroJournalHdr(Pager *pPager,
 ** 
 ** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space.
 */
 static int writeJournalHdr(Pager *pPager){
   int rc = SQLITE_OK;
   char *zHeader = pPager->pTmpSpace;
   u32 nHeader = pPager->pageSize;
   u32 nWrite;
+  int ii;
 
   if( nHeader>JOURNAL_HDR_SZ(pPager) ){
     nHeader = JOURNAL_HDR_SZ(pPager);
   }
 
-  if( pPager->stmtHdrOff==0 ){
-    pPager->stmtHdrOff = pPager->journalOff;
+  /* If there are active savepoints and any of them were created since the
+  ** most recent journal header was written, update the PagerSavepoint.iHdrOff
+  ** fields now.
+  */
+  for(ii=0; ii<pPager->nSavepoint; ii++){
+    if( pPager->aSavepoint[ii].iHdrOffset==0 ){
+      pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff;
+    }
   }
 
   seekJournalHdr(pPager);
   pPager->journalHdr = pPager->journalOff;
 
   memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic));
 
   /* 
@@ -31185,17 +31462,17 @@ static int writeJournalHdr(Pager *pPager
   }else{
     put32bits(&zHeader[sizeof(aJournalMagic)], 0);
   }
 
   /* The random check-hash initialiser */ 
   sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
   put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
   /* The initial database size */
-  put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbSize);
+  put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize);
   /* The assumed sector size for this process */
   put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize);
 
   /* Initializing the tail of the buffer is not necessary.  Everything
   ** works find if the following memset() is omitted.  But initializing
   ** the memory prevents valgrind from complaining, so we are willing to
   ** take the performance hit.
   */
@@ -31214,39 +31491,41 @@ static int writeJournalHdr(Pager *pPager
   }
 
   return rc;
 }
 
 /*
 ** The journal file must be open when this is called. A journal header file
 ** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal
-** file. See comments above function writeJournalHdr() for a description of
-** the journal header format.
+** file. The current location in the journal file is given by
+** pPager->journalOff.  See comments above function writeJournalHdr() for
+** a description of the journal header format.
 **
 ** If the header is read successfully, *nRec is set to the number of
 ** page records following this header and *dbSize is set to the size of the
 ** database before the transaction began, in pages. Also, pPager->cksumInit
 ** is set to the value read from the journal header. SQLITE_OK is returned
 ** in this case.
 **
 ** If the journal header file appears to be corrupted, SQLITE_DONE is
-** returned and *nRec and *dbSize are not set.  If JOURNAL_HDR_SZ bytes
+** returned and *nRec and *dbSize are undefined.  If JOURNAL_HDR_SZ bytes
 ** cannot be read from the journal file an error code is returned.
 */
 static int readJournalHdr(
   Pager *pPager, 
   i64 journalSize,
   u32 *pNRec, 
   u32 *pDbSize
 ){
   int rc;
   unsigned char aMagic[8]; /* A buffer to hold the magic header */
   i64 jrnlOff;
-  int iPageSize;
+  u32 iPageSize;
+  u32 iSectorSize;
 
   seekJournalHdr(pPager);
   if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){
     return SQLITE_DONE;
   }
   jrnlOff = pPager->journalOff;
 
   rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), jrnlOff);
@@ -31261,38 +31540,51 @@ static int readJournalHdr(
   if( rc ) return rc;
 
   rc = read32bits(pPager->jfd, jrnlOff+4, &pPager->cksumInit);
   if( rc ) return rc;
 
   rc = read32bits(pPager->jfd, jrnlOff+8, pDbSize);
   if( rc ) return rc;
 
-  rc = read32bits(pPager->jfd, jrnlOff+16, (u32 *)&iPageSize);
-  if( rc==SQLITE_OK 
-   && iPageSize>=512 
-   && iPageSize<=SQLITE_MAX_PAGE_SIZE 
-   && ((iPageSize-1)&iPageSize)==0 
-  ){
-    u16 pagesize = (u16)iPageSize;
-    rc = sqlite3PagerSetPagesize(pPager, &pagesize);
-  }
-  if( rc ) return rc;
-
-  /* Update the assumed sector-size to match the value used by 
-  ** the process that created this journal. If this journal was
-  ** created by a process other than this one, then this routine
-  ** is being called from within pager_playback(). The local value
-  ** of Pager.sectorSize is restored at the end of that routine.
-  */
-  rc = read32bits(pPager->jfd, jrnlOff+12, &pPager->sectorSize);
-  if( rc ) return rc;
-  if( (pPager->sectorSize & (pPager->sectorSize-1))!=0
-         || pPager->sectorSize>0x1000000 ){
-    return SQLITE_DONE;
+  if( pPager->journalOff==0 ){
+    rc = read32bits(pPager->jfd, jrnlOff+16, &iPageSize);
+    if( rc ) return rc;
+
+    if( iPageSize<512 
+     || iPageSize>SQLITE_MAX_PAGE_SIZE 
+     || ((iPageSize-1)&iPageSize)!=0 
+    ){
+      /* If the page-size in the journal-header is invalid, then the process
+      ** that wrote the journal-header must have crashed before the header
+      ** was synced. In this case stop reading the journal file here.
+      */
+      rc = SQLITE_DONE;
+    }else{
+      u16 pagesize = (u16)iPageSize;
+      rc = sqlite3PagerSetPagesize(pPager, &pagesize);
+      assert( rc!=SQLITE_OK || pagesize==(u16)iPageSize );
+    }
+    if( rc ) return rc;
+  
+    /* Update the assumed sector-size to match the value used by 
+    ** the process that created this journal. If this journal was
+    ** created by a process other than this one, then this routine
+    ** is being called from within pager_playback(). The local value
+    ** of Pager.sectorSize is restored at the end of that routine.
+    */
+    rc = read32bits(pPager->jfd, jrnlOff+12, &iSectorSize);
+    if( rc ) return rc;
+    if( (iSectorSize&(iSectorSize-1))
+      || iSectorSize<512
+      || iSectorSize>MAX_SECTOR_SIZE
+    ){
+      return SQLITE_DONE;
+    }
+    pPager->sectorSize = iSectorSize;
   }
 
   pPager->journalOff += JOURNAL_HDR_SZ(pPager);
   return SQLITE_OK;
 }
 
 
 /*
@@ -31393,62 +31685,94 @@ static PgHdr *pager_lookup(Pager *pPager
 ** to access those pages will likely result in a coredump.
 */
 static void pager_reset(Pager *pPager){
   if( pPager->errCode ) return;
   sqlite3PcacheClear(pPager->pPCache);
 }
 
 /*
+** Free all structures in the Pager.aSavepoint[] array and set both
+** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal
+** if it is open and the pager is not in exclusive mode.
+*/
+static void releaseAllSavepoint(Pager *pPager){
+  int ii;
+  for(ii=0; ii<pPager->nSavepoint; ii++){
+    sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
+  }
+  if( !pPager->exclusiveMode ){
+    sqlite3OsClose(pPager->sjfd);
+  }
+  sqlite3_free(pPager->aSavepoint);
+  pPager->aSavepoint = 0;
+  pPager->nSavepoint = 0;
+  pPager->stmtNRec = 0;
+}
+
+/*
+** Set the bit number pgno in the PagerSavepoint.pInSavepoint bitvecs of
+** all open savepoints.
+*/
+static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
+  int ii;                   /* Loop counter */
+  int rc = SQLITE_OK;       /* Result code */
+
+  for(ii=0; ii<pPager->nSavepoint; ii++){
+    PagerSavepoint *p = &pPager->aSavepoint[ii];
+    if( pgno<=p->nOrig ){
+      rc |= sqlite3BitvecSet(p->pInSavepoint, pgno);
+      assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
+    }
+  }
+  return rc;
+}
+
+/*
 ** Unlock the database file. 
 **
 ** If the pager is currently in error state, discard the contents of 
 ** the cache and reset the Pager structure internal state. If there is
 ** an open journal-file, then the next time a shared-lock is obtained
 ** on the pager file (by this or any other process), it will be
 ** treated as a hot-journal and rolled back.
 */
 static void pager_unlock(Pager *pPager){
   if( !pPager->exclusiveMode ){
-    int rc = osUnlock(pPager->fd, NO_LOCK);
-    if( rc ) pPager->errCode = rc;
-    pPager->dbSizeValid = 0;
-    IOTRACE(("UNLOCK %p\n", pPager))
+    int rc;
 
     /* Always close the journal file when dropping the database lock.
     ** Otherwise, another connection with journal_mode=delete might
     ** delete the file out from under us.
     */
     if( pPager->journalOpen ){
       sqlite3OsClose(pPager->jfd);
       pPager->journalOpen = 0;
       sqlite3BitvecDestroy(pPager->pInJournal);
       pPager->pInJournal = 0;
       sqlite3BitvecDestroy(pPager->pAlwaysRollback);
       pPager->pAlwaysRollback = 0;
     }
 
+    rc = osUnlock(pPager->fd, NO_LOCK);
+    if( rc ) pPager->errCode = rc;
+    pPager->dbSizeValid = 0;
+    IOTRACE(("UNLOCK %p\n", pPager))
+
     /* If Pager.errCode is set, the contents of the pager cache cannot be
     ** trusted. Now that the pager file is unlocked, the contents of the
     ** cache can be discarded and the error code safely cleared.
     */
     if( pPager->errCode ){
       if( rc==SQLITE_OK ) pPager->errCode = SQLITE_OK;
       pager_reset(pPager);
-      if( pPager->stmtOpen ){
-        sqlite3OsClose(pPager->stfd);
-        sqlite3BitvecDestroy(pPager->pInStmt);
-        pPager->pInStmt = 0;
-      }
-      pPager->stmtOpen = 0;
-      pPager->stmtInUse = 0;
+      releaseAllSavepoint(pPager);
       pPager->journalOff = 0;
       pPager->journalStarted = 0;
-      pPager->stmtAutoopen = 0;
-      pPager->origDbSize = 0;
+      pPager->dbOrigSize = 0;
     }
 
     pPager->state = PAGER_UNLOCK;
     pPager->changeCountDone = 0;
   }
 }
 
 /*
@@ -31483,21 +31807,17 @@ static void pagerUnlockAndRollback(Pager
 ** a file is an expensive operation.
 */
 static int pager_end_transaction(Pager *pPager, int hasMaster){
   int rc = SQLITE_OK;
   int rc2 = SQLITE_OK;
   if( pPager->state<PAGER_RESERVED ){
     return SQLITE_OK;
   }
-  sqlite3PagerStmtCommit(pPager);
-  if( pPager->stmtOpen && !pPager->exclusiveMode ){
-    sqlite3OsClose(pPager->stfd);
-    pPager->stmtOpen = 0;
-  }
+  releaseAllSavepoint(pPager);
   if( pPager->journalOpen ){
     if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){
       int isMemoryJournal = sqlite3IsMemJournal(pPager->jfd);
       sqlite3OsClose(pPager->jfd);
       pPager->journalOpen = 0;
       if( !isMemoryJournal ){
         rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0);
       }
@@ -31532,23 +31852,25 @@ static int pager_end_transaction(Pager *
     pPager->nRec = 0;
   }else{
     assert( pPager->pInJournal==0 );
   }
 
   if( !pPager->exclusiveMode ){
     rc2 = osUnlock(pPager->fd, SHARED_LOCK);
     pPager->state = PAGER_SHARED;
+    pPager->changeCountDone = 0;
   }else if( pPager->state==PAGER_SYNCED ){
     pPager->state = PAGER_EXCLUSIVE;
   }
-  pPager->origDbSize = 0;
+  pPager->dbOrigSize = 0;
   pPager->setMaster = 0;
   pPager->needSync = 0;
   /* lruListSetFirstSynced(pPager); */
+  sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
   if( !MEMDB ){
     pPager->dbSizeValid = 0;
   }
   pPager->dbModified = 0;
 
   return (rc==SQLITE_OK?rc2:rc);
 }
 
@@ -31578,66 +31900,82 @@ static u32 pager_cksum(Pager *pPager, co
   while( i>0 ){
     cksum += aData[i];
     i -= 200;
   }
   return cksum;
 }
 
 /*
-** Read a single page from the journal file opened on file descriptor
-** jfd.  Playback this one page.
+** Read a single page from either the journal file (if isMainJrnl==1) or
+** from the sub-journal (if isMainJrnl==0) and playback that page.
+** The page begins at offset *pOffset into the file.  The  *pOffset
+** value is increased to the start of the next page in the journal.
 **
 ** The isMainJrnl flag is true if this is the main rollback journal and
 ** false for the statement journal.  The main rollback journal uses
 ** checksums - the statement journal does not.
+**
+** If pDone is not NULL, then it is a record of pages that have already
+** been played back.  If the page at *pOffset has already been played back
+** (if the corresponding pDone bit is set) then skip the playback.
+** Make sure the pDone bit corresponding to the *pOffset page is set
+** prior to returning.
 */
 static int pager_playback_one_page(
-  Pager *pPager,       /* The pager being played back */
-  sqlite3_file *jfd,   /* The file that is the journal being rolled back */
-  i64 offset,          /* Offset of the page within the journal */
-  int isMainJrnl       /* True for main rollback journal. False for Stmt jrnl */
+  Pager *pPager,                /* The pager being played back */
+  int isMainJrnl,               /* 1 -> main journal. 0 -> sub-journal. */
+  i64 *pOffset,                 /* Offset of record to playback */
+  int isSavepnt,                /* True for a savepoint rollback */
+  Bitvec *pDone                 /* Bitvec of pages already played back */
 ){
   int rc;
   PgHdr *pPg;                   /* An existing page in the cache */
   Pgno pgno;                    /* The page number of a page in journal */
   u32 cksum;                    /* Checksum used for sanity checking */
-  u8 *aData = (u8 *)pPager->pTmpSpace;   /* Temp storage for a page */
-
-  /* isMainJrnl should be true for the main journal and false for
-  ** statement journals.  Verify that this is always the case
-  */
-  assert( jfd == (isMainJrnl ? pPager->jfd : pPager->stfd) );
-  assert( aData );
-
-  rc = read32bits(jfd, offset, &pgno);
+  u8 *aData;                    /* Temporary storage for the page */
+  sqlite3_file *jfd;            /* The file descriptor for the journal file */
+
+  assert( (isMainJrnl&~1)==0 );      /* isMainJrnl is 0 or 1 */
+  assert( (isSavepnt&~1)==0 );       /* isSavepnt is 0 or 1 */
+  assert( isMainJrnl || pDone );     /* pDone always used on sub-journals */
+  assert( isSavepnt || pDone==0 );   /* pDone never used on non-savepoint */
+
+  aData = (u8*)pPager->pTmpSpace;
+  assert( aData );         /* Temp storage must have already been allocated */
+
+  jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
+
+  rc = read32bits(jfd, *pOffset, &pgno);
   if( rc!=SQLITE_OK ) return rc;
-  rc = sqlite3OsRead(jfd, aData, pPager->pageSize, offset+4);
+  rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4);
   if( rc!=SQLITE_OK ) return rc;
-  pPager->journalOff += pPager->pageSize + 4;
+  *pOffset += pPager->pageSize + 4 + isMainJrnl*4;
 
   /* Sanity checking on the page.  This is more important that I originally
   ** thought.  If a power failure occurs while the journal is being written,
   ** it could cause invalid data to be written into the journal.  We need to
   ** detect this invalid data (with high probability) and ignore it.
   */
   if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){
     return SQLITE_DONE;
   }
-  if( pgno>(unsigned)pPager->dbSize ){
+  if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){
     return SQLITE_OK;
   }
   if( isMainJrnl ){
-    rc = read32bits(jfd, offset+pPager->pageSize+4, &cksum);
+    rc = read32bits(jfd, (*pOffset)-4, &cksum);
     if( rc ) return rc;
-    pPager->journalOff += 4;
-    if( pager_cksum(pPager, aData)!=cksum ){
+    if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){
       return SQLITE_DONE;
     }
   }
+  if( pDone && (rc = sqlite3BitvecSet(pDone, pgno)) ){
+    return rc;
+  }
 
   assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE );
 
   /* If the pager is in RESERVED state, then there must be a copy of this
   ** page in the pager cache. In this case just update the pager cache,
   ** not the database file. The page is left marked dirty in this case.
   **
   ** An exception to the above rule: If the database is in no-sync mode
@@ -31664,39 +32002,85 @@ static int pager_playback_one_page(
   ** in the main journal either because the page is not in cache or else
   ** the page is marked as needSync==0.
   **
   ** 2008-04-14:  When attempting to vacuum a corrupt database file, it
   ** is possible to fail a statement on a database that does not yet exist.
   ** Do not attempt to write if database file has never been opened.
   */
   pPg = pager_lookup(pPager, pgno);
-  PAGERTRACE4("PLAYBACK %d page %d hash(%08x)\n",
-               PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData));
+  PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
+               PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData),
+               (isMainJrnl?"main-journal":"sub-journal")
+  ));
   if( (pPager->state>=PAGER_EXCLUSIVE)
    && (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC))
    && (pPager->fd->pMethods)
   ){
     i64 ofst = (pgno-1)*(i64)pPager->pageSize;
     rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
+    if( pgno>pPager->dbFileSize ){
+      pPager->dbFileSize = pgno;
+    }
+  }else if( !isMainJrnl && pPg==0 ){
+    /* If this is a rollback of a savepoint and data was not written to
+    ** the database and the page is not in-memory, there is a potential
+    ** problem. When the page is next fetched by the b-tree layer, it 
+    ** will be read from the database file, which may or may not be 
+    ** current. 
+    **
+    ** There are a couple of different ways this can happen. All are quite
+    ** obscure. When running in synchronous mode, this can only happen 
+    ** if the page is on the free-list at the start of the transaction, then
+    ** populated, then moved using sqlite3PagerMovepage().
+    **
+    ** The solution is to add an in-memory page to the cache containing
+    ** the data just read from the sub-journal. Mark the page as dirty 
+    ** and if the pager requires a journal-sync, then mark the page as 
+    ** requiring a journal-sync before it is written.
+    */
+    assert( isSavepnt );
+    if( (rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1)) ){
+      return rc;
+    }
+    pPg->flags &= ~PGHDR_NEED_READ;
+    sqlite3PcacheMakeDirty(pPg);
   }
   if( pPg ){
     /* No page should ever be explicitly rolled back that is in use, except
     ** for page 1 which is held in use in order to keep the lock on the
     ** database active. However such a page may be rolled back as a result
     ** of an internal error resulting in an automatic call to
     ** sqlite3PagerRollback().
     */
     void *pData;
     pData = pPg->pData;
     memcpy(pData, aData, pPager->pageSize);
     if( pPager->xReiniter ){
       pPager->xReiniter(pPg);
     }
-    if( isMainJrnl ){
+    if( isMainJrnl && (!isSavepnt || pPager->journalOff<=pPager->journalHdr) ){
+      /* If the contents of this page were just restored from the main 
+      ** journal file, then its content must be as they were when the 
+      ** transaction was first opened. In this case we can mark the page
+      ** as clean, since there will be no need to write it out to the.
+      **
+      ** There is one exception to this rule. If the page is being rolled
+      ** back as part of a savepoint (or statement) rollback from an 
+      ** unsynced portion of the main journal file, then it is not safe
+      ** to mark the page as clean. This is because marking the page as
+      ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is
+      ** already in the journal file (recorded in Pager.pInJournal) and
+      ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to
+      ** again within this transaction, it will be marked as dirty but
+      ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially
+      ** be written out into the database file before its journal file
+      ** segment is synced. If a crash occurs during or following this,
+      ** database corruption may ensue.
+      */
       sqlite3PcacheMakeClean(pPg);
     }
 #ifdef SQLITE_CHECK_PAGES
     pPg->pageHash = pager_pagehash(pPg);
 #endif
     /* If this was page 1, then restore the value of Pager.dbFileVers.
     ** Do this before any decoding. */
     if( pgno==1 ){
@@ -31705,16 +32089,56 @@ static int pager_playback_one_page(
 
     /* Decode the page just read from disk */
     CODEC1(pPager, pData, pPg->pgno, 3);
     sqlite3PcacheRelease(pPg);
   }
   return rc;
 }
 
+#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST)
+/*
+** This routine looks ahead into the main journal file and determines
+** whether or not the next record (the record that begins at file
+** offset pPager->journalOff) is a well-formed page record consisting
+** of a valid page number, pPage->pageSize bytes of content, followed
+** by a valid checksum.
+**
+** The pager never needs to know this in order to do its job.   This
+** routine is only used from with assert() and testcase() macros.
+*/
+static int pagerNextJournalPageIsValid(Pager *pPager){
+  Pgno pgno;           /* The page number of the page */
+  u32 cksum;           /* The page checksum */
+  int rc;              /* Return code from read operations */
+  sqlite3_file *fd;    /* The file descriptor from which we are reading */
+  u8 *aData;           /* Content of the page */
+
+  /* Read the page number header */
+  fd = pPager->jfd;
+  rc = read32bits(fd, pPager->journalOff, &pgno);
+  if( rc!=SQLITE_OK ){ return 0; }                                  /*NO_TEST*/
+  if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ return 0; }         /*NO_TEST*/
+  if( pgno>(Pgno)pPager->dbSize ){ return 0; }                      /*NO_TEST*/
+
+  /* Read the checksum */
+  rc = read32bits(fd, pPager->journalOff+pPager->pageSize+4, &cksum);
+  if( rc!=SQLITE_OK ){ return 0; }                                  /*NO_TEST*/
+
+  /* Read the data and verify the checksum */
+  aData = (u8*)pPager->pTmpSpace;
+  rc = sqlite3OsRead(fd, aData, pPager->pageSize, pPager->journalOff+4);
+  if( rc!=SQLITE_OK ){ return 0; }                                  /*NO_TEST*/
+  if( pager_cksum(pPager, aData)!=cksum ){ return 0; }              /*NO_TEST*/
+
+  /* Reach this point only if the page is valid */
+  return 1;
+}
+#endif /* !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) */
+
 /*
 ** Parameter zMaster is the name of a master journal file. A single journal
 ** file that referred to the master journal file has just been rolled back.
 ** This routine checks if it is possible to delete the master journal file,
 ** and does so if it is.
 **
 ** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not 
 ** available for use within this function.
@@ -31812,23 +32236,22 @@ delmaster_out:
   if( master_open ){
     sqlite3OsClose(pMaster);
   }
   sqlite3_free(pMaster);
   return rc;
 }
 
 
-static void pager_truncate_cache(Pager *pPager);
-
-/*
-** Truncate the main file of the given pager to the number of pages
-** indicated. Also truncate the cached representation of the file.
-**
-** Might might be the case that the file on disk is smaller than nPage.
+/*
+** If the main database file is open and an exclusive lock is held, 
+** truncate the main file of the given pager to the specified number 
+** of pages.
+**
+** It might might be the case that the file on disk is smaller than nPage.
 ** This can happen, for example, if we are in the middle of a transaction
 ** which has extended the file size and the new pages are still all held
 ** in cache, then an INSERT or UPDATE does a statement rollback.  Some
 ** operating system implementations can get confused if you try to
 ** truncate a file to some size that is larger than it currently is,
 ** so detect this case and write a single zero byte to the end of the new
 ** file instead.
 */
@@ -31839,43 +32262,45 @@ static int pager_truncate(Pager *pPager,
     rc = sqlite3OsFileSize(pPager->fd, &currentSize);
     newSize = pPager->pageSize*(i64)nPage;
     if( rc==SQLITE_OK && currentSize!=newSize ){
       if( currentSize>newSize ){
         rc = sqlite3OsTruncate(pPager->fd, newSize);
       }else{
         rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
       }
-    }
-  }
-  if( rc==SQLITE_OK ){
-    pPager->dbSize = nPage;
-    pager_truncate_cache(pPager);
+      if( rc==SQLITE_OK ){
+        pPager->dbFileSize = nPage;
+      }
+    }
   }
   return rc;
 }
 
 /*
 ** Set the sectorSize for the given pager.
 **
 ** The sector size is at least as big as the sector size reported
-** by sqlite3OsSectorSize().  The minimum sector size is 512.
+** by sqlite3OsSectorSize(). The minimum sector size is 512.
 */
 static void setSectorSize(Pager *pPager){
   assert(pPager->fd->pMethods||pPager->tempFile);
   if( !pPager->tempFile ){
     /* Sector size doesn't matter for temporary files. Also, the file
     ** may not have been opened yet, in whcih case the OsSectorSize()
     ** call will segfault.
     */
     pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
   }
   if( pPager->sectorSize<512 ){
     pPager->sectorSize = 512;
   }
+  if( pPager->sectorSize>MAX_SECTOR_SIZE ){
+    pPager->sectorSize = MAX_SECTOR_SIZE;
+  }
 }
 
 /*
 ** Playback the journal and thus restore the database file to
 ** the state it was in before we started making changes.  
 **
 ** The journal file format is as follows: 
 **
@@ -31993,36 +32418,48 @@ static int pager_playback(Pager *pPager,
 
     /* If nRec is 0 and this rollback is of a transaction created by this
     ** process and if this is the final header in the journal, then it means
     ** that this part of the journal was being filled but has not yet been
     ** synced to disk.  Compute the number of pages based on the remaining
     ** size of the file.
     **
     ** The third term of the test was added to fix ticket #2565.
-    */
+    ** When rolling back a hot journal, nRec==0 always means that the next
+    ** chunk of the journal contains zero pages to be rolled back.  But
+    ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in
+    ** the journal, it means that the journal might contain additional
+    ** pages that need to be rolled back and that the number of pages 
+    ** should be computed based on the journal file size.
+    */
+    testcase( nRec==0 && !isHot
+         && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)!=pPager->journalOff
+         && ((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager))>0
+         && pagerNextJournalPageIsValid(pPager)
+    );
     if( nRec==0 && !isHot &&
         pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){
       nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager));
     }
 
     /* If this is the first header read from the journal, truncate the
     ** database file back to its original size.
     */
     if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){
       rc = pager_truncate(pPager, mxPg);
       if( rc!=SQLITE_OK ){
         goto end_playback;
       }
+      pPager->dbSize = mxPg;
     }
 
     /* Copy original pages out of the journal and back into the database file.
     */
     for(u=0; u<nRec; u++){
-      rc = pager_playback_one_page(pPager, pPager->jfd, pPager->journalOff, 1);
+      rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 0, 0);
       if( rc!=SQLITE_OK ){
         if( rc==SQLITE_DONE ){
           rc = SQLITE_OK;
           pPager->journalOff = szJ;
           break;
         }else{
           /* If we are unable to rollback, then the database is probably
           ** going to end up being corrupt.  It is corrupt to us, anyhow.
@@ -32033,16 +32470,26 @@ static int pager_playback(Pager *pPager,
         }
       }
     }
   }
   /*NOTREACHED*/
   assert( 0 );
 
 end_playback:
+  /* Following a rollback, the database file should be back in its original
+  ** state prior to the start of the transaction, so invoke the
+  ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the
+  ** assertion that the transaction counter was modified.
+  */
+  assert(
+    pPager->fd->pMethods==0 ||
+    sqlite3OsFileControl(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0)>=SQLITE_OK
+  );
+
   if( rc==SQLITE_OK ){
     zMaster = pPager->pTmpSpace;
     rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
   }
   if( rc==SQLITE_OK ){
     rc = pager_end_transaction(pPager, zMaster[0]!='\0');
   }
   if( rc==SQLITE_OK && zMaster[0] && res ){
@@ -32056,111 +32503,117 @@ end_playback:
   ** back a journal created by a process with a different sector size
   ** value. Reset it to the correct value for this process.
   */
   setSectorSize(pPager);
   return rc;
 }
 
 /*
-** Playback the statement journal.
-**
-** This is similar to playing back the transaction journal but with
-** a few extra twists.
-**
-**    (1)  The number of pages in the database file at the start of
-**         the statement is stored in pPager->stmtSize, not in the
-**         journal file itself.
-**
-**    (2)  In addition to playing back the statement journal, also
-**         playback all pages of the transaction journal beginning
-**         at offset pPager->stmtJSize.
-*/
-static int pager_stmt_playback(Pager *pPager){
-  i64 szJ;                 /* Size of the full journal */
-  i64 hdrOff;
-  int nRec;                /* Number of Records */
-  int i;                   /* Loop counter */
-  int rc;
-
-  szJ = pPager->journalOff;
-
-  /* Set hdrOff to be the offset just after the end of the last journal
-  ** page written before the first journal-header for this statement
-  ** transaction was written, or the end of the file if no journal
-  ** header was written.
-  */
-  hdrOff = pPager->stmtHdrOff;
-  assert( pPager->fullSync || !hdrOff );
-  if( !hdrOff ){
-    hdrOff = szJ;
-  }
-  
-  /* Truncate the database back to its original size.
-  */
-  rc = pager_truncate(pPager, pPager->stmtSize);
+** Playback savepoint pSavepoint.  Or, if pSavepoint==NULL, then playback
+** the entire master journal file.
+**
+** The case pSavepoint==NULL occurs when a ROLLBACK TO command is invoked
+** on a SAVEPOINT that is a transaction savepoint.
+*/
+static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){
+  i64 szJ;                 /* Effective size of the main journal */
+  i64 iHdrOff;             /* End of first segment of main-journal records */
+  Pgno ii;                 /* Loop counter */
+  int rc = SQLITE_OK;      /* Return code */
+  Bitvec *pDone = 0;       /* Bitvec to ensure pages played back only once */
+
+  /* Allocate a bitvec to use to store the set of pages rolled back */
+  if( pSavepoint ){
+    pDone = sqlite3BitvecCreate(pSavepoint->nOrig);
+    if( !pDone ){
+      return SQLITE_NOMEM;
+    }
+  }
+
+  /* Truncate the database back to the size it was before the 
+  ** savepoint being reverted was opened.
+  */
+  pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize;
   assert( pPager->state>=PAGER_SHARED );
 
-  /* Figure out how many records are in the statement journal.
-  */
-  assert( pPager->stmtInUse && pPager->journalOpen );
-  nRec = pPager->stmtNRec;
-  
-  /* Copy original pages out of the statement journal and back into the
-  ** database file.  Note that the statement journal omits checksums from
-  ** each record since power-failure recovery is not important to statement
-  ** journals.
-  */
-  for(i=0; i<nRec; i++){
-    i64 offset = i*(4+pPager->pageSize);
-    rc = pager_playback_one_page(pPager, pPager->stfd, offset, 0);
-    assert( rc!=SQLITE_DONE );
-    if( rc!=SQLITE_OK ) goto end_stmt_playback;
-  }
-
-  /* Now roll some pages back from the transaction journal. Pager.stmtJSize
-  ** was the size of the journal file when this statement was started, so
-  ** everything after that needs to be rolled back, either into the
-  ** database, the memory cache, or both.
-  **
-  ** If it is not zero, then Pager.stmtHdrOff is the offset to the start
-  ** of the first journal header written during this statement transaction.
-  */
-  pPager->journalOff = pPager->stmtJSize;
-  pPager->cksumInit = (int)(pPager->stmtCksum & 0xffffffff);
-  while( pPager->journalOff < hdrOff ){
-    rc = pager_playback_one_page(pPager, pPager->jfd, pPager->journalOff, 1);
-    assert( rc!=SQLITE_DONE );
-    if( rc!=SQLITE_OK ) goto end_stmt_playback;
-  }
-
-  while( pPager->journalOff < szJ ){
-    u32 nJRec;         /* Number of Journal Records */
+  /* Use pPager->journalOff as the effective size of the main rollback
+  ** journal.  The actual file might be larger than this in
+  ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST.  But anything
+  ** past pPager->journalOff is off-limits to us.
+  */
+  szJ = pPager->journalOff;
+
+  /* Begin by rolling back records from the main journal starting at
+  ** PagerSavepoint.iOffset and continuing to the next journal header.
+  ** There might be records in the main journal that have a page number
+  ** greater than the current database size (pPager->dbSize) but those
+  ** will be skipped automatically.  Pages are added to pDone as they
+  ** are played back.
+  */
+  if( pSavepoint ){
+    iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ;
+    pPager->journalOff = pSavepoint->iOffset;
+    while( rc==SQLITE_OK && pPager->journalOff<iHdrOff ){
+      rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 1, pDone);
+      assert( rc!=SQLITE_DONE );
+    }
+  }else{
+    pPager->journalOff = 0;
+  }
+
+  /* Continue rolling back records out of the main journal starting at
+  ** the first journal header seen and continuing until the effective end
+  ** of the main journal file.  Continue to skip out-of-range pages and
+  ** continue adding pages rolled back to pDone.
+  */
+  while( rc==SQLITE_OK && pPager->journalOff<szJ ){
+    u32 nJRec = 0;     /* Number of Journal Records */
     u32 dummy;
     rc = readJournalHdr(pPager, szJ, &nJRec, &dummy);
-    if( rc!=SQLITE_OK ){
-      assert( rc!=SQLITE_DONE );
-      goto end_stmt_playback;
-    }
-    if( nJRec==0 ){
-      nJRec = (int)((szJ - pPager->journalOff) / (pPager->pageSize+8));
-    }
-    for(i=nJRec-1; i>=0 && pPager->journalOff < szJ; i--){
-      rc = pager_playback_one_page(pPager, pPager->jfd, pPager->journalOff, 1);
+    assert( rc!=SQLITE_DONE );
+
+    /*
+    ** The "pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff"
+    ** test is related to ticket #2565.  See the discussion in the
+    ** pager_playback() function for additional information.
+    */
+    assert( !(nJRec==0
+         && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)!=pPager->journalOff
+         && ((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager))>0
+         && pagerNextJournalPageIsValid(pPager))
+    );
+    if( nJRec==0 
+     && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff
+    ){
+      nJRec = (szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager);
+    }
+    for(ii=0; rc==SQLITE_OK && ii<nJRec && pPager->journalOff<szJ; ii++){
+      rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 1, pDone);
       assert( rc!=SQLITE_DONE );
-      if( rc!=SQLITE_OK ) goto end_stmt_playback;
-    }
-  }
-
-  pPager->journalOff = szJ;
-  
-end_stmt_playback:
-  if( rc==SQLITE_OK) {
+    }
+  }
+  assert( rc!=SQLITE_OK || pPager->journalOff==szJ );
+
+  /* Finally,  rollback pages from the sub-journal.  Page that were
+  ** previously rolled back out of the main journal (and are hence in pDone)
+  ** will be skipped.  Out-of-range pages are also skipped.
+  */
+  if( pSavepoint ){
+    i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize);
+    for(ii=pSavepoint->iSubRec; rc==SQLITE_OK&&ii<(u32)pPager->stmtNRec; ii++){
+      assert( offset == ii*(4+pPager->pageSize) );
+      rc = pager_playback_one_page(pPager, 0, &offset, 1, pDone);
+      assert( rc!=SQLITE_DONE );
+    }
+  }
+
+  sqlite3BitvecDestroy(pDone);
+  if( rc==SQLITE_OK ){
     pPager->journalOff = szJ;
-    /* pager_reload_cache(pPager); */
   }
   return rc;
 }
 
 /*
 ** Change the maximum number of in-memory pages that are allowed.
 */
 SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){
@@ -32322,17 +32775,17 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
   if( !pPager ){
     sqlite3_free(zPathname);
     return SQLITE_NOMEM;
   }
   pPager->pPCache = (PCache *)&pPager[1];
   pPtr = ((u8 *)&pPager[1]) + pcacheSize;
   pPager->vfsFlags = vfsFlags;
   pPager->fd = (sqlite3_file*)&pPtr[pVfs->szOsFile*0];
-  pPager->stfd = (sqlite3_file*)&pPtr[pVfs->szOsFile];
+  pPager->sjfd = (sqlite3_file*)&pPtr[pVfs->szOsFile];
   pPager->jfd = (sqlite3_file*)&pPtr[pVfs->szOsFile+journalFileSize];
   pPager->zFilename = (char*)&pPtr[pVfs->szOsFile+2*journalFileSize];
   pPager->zDirectory = &pPager->zFilename[nPathname+1];
   pPager->zJournal = &pPager->zDirectory[nPathname+1];
   pPager->pVfs = pVfs;
   if( zPathname ){
     memcpy(pPager->zFilename, zPathname, nPathname+1);
     sqlite3_free(zPathname);
@@ -32353,19 +32806,19 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
       ** choose a default page size in case we have to create the
       ** database file. The default page size is the maximum of:
       **
       **    + SQLITE_DEFAULT_PAGE_SIZE,
       **    + The value returned by sqlite3OsSectorSize()
       **    + The largest page size that can be written atomically.
       */
       if( rc==SQLITE_OK && !readOnly ){
-        int iSectorSize = sqlite3OsSectorSize(pPager->fd);
-        if( szPageDflt<iSectorSize ){
-          szPageDflt = iSectorSize;
+        setSectorSize(pPager);
+        if( szPageDflt<pPager->sectorSize ){
+          szPageDflt = pPager->sectorSize;
         }
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
         {
           int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
           int ii;
           assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
           assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));
           assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536);
@@ -32405,17 +32858,17 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
     sqlite3OsClose(pPager->fd);
     sqlite3_free(pPager);
     return ((rc==SQLITE_OK)?SQLITE_NOMEM:rc);
   }
   nExtra = FORCE_ALIGNMENT(nExtra);
   sqlite3PcacheOpen(szPageDflt, nExtra, !memDb,
                     !memDb?pagerStress:0, (void *)pPager, pPager->pPCache);
 
-  PAGERTRACE3("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename);
+  PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename));
   IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename))
 
   /* Fill in Pager.zDirectory[] */
   memcpy(pPager->zDirectory, pPager->zFilename, nPathname+1);
   for(i=sqlite3Strlen30(pPager->zDirectory); 
       i>0 && pPager->zDirectory[i-1]!='/'; i--){}
   if( i>0 ) pPager->zDirectory[i-1] = 0;
 
@@ -32627,17 +33080,18 @@ SQLITE_PRIVATE int sqlite3PagerPagecount
       return rc;
     }
     if( n>0 && n<pPager->pageSize ){
       n = 1;
     }else{
       n /= pPager->pageSize;
     }
     if( pPager->state!=PAGER_UNLOCK ){
-      pPager->dbSize = (int)n;
+      pPager->dbSize = (Pgno)n;
+      pPager->dbFileSize = (Pgno)n;
       pPager->dbSizeValid = 1;
     }
   }
   if( n==(PENDING_BYTE/pPager->pageSize) ){
     n++;
   }
   if( n>pPager->mxPgno ){
     pPager->mxPgno = (Pgno)n;
@@ -32649,32 +33103,16 @@ SQLITE_PRIVATE int sqlite3PagerPagecount
 }
 
 /*
 ** Forward declaration
 */
 static int syncJournal(Pager*);
 
 /*
-** This routine is used to truncate the cache when a database
-** is truncated.  Drop from the cache all pages whose pgno is
-** larger than pPager->dbSize and is unreferenced.
-**
-** Referenced pages larger than pPager->dbSize are zeroed.
-**
-** Actually, at the point this routine is called, it would be
-** an error to have a referenced page.  But rather than delete
-** that page and guarantee a subsequent segfault, it seems better
-** to zero it and hope that we error out sanely.
-*/
-static void pager_truncate_cache(Pager *pPager){
-  sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
-}
-
-/*
 ** Try to obtain a lock on a file.  Invoke the busy callback if the lock
 ** is currently not available.  Repeat until the busy callback returns
 ** false or until the lock succeeds.
 **
 ** Return SQLITE_OK on success and an error code if we cannot obtain
 ** the lock.
 */
 static int pager_wait_on_lock(Pager *pPager, int locktype){
@@ -32697,39 +33135,44 @@ static int pager_wait_on_lock(Pager *pPa
     if( rc==SQLITE_OK ){
       pPager->state = (u8)locktype;
       IOTRACE(("LOCK %p %d\n", pPager, locktype))
     }
   }
   return rc;
 }
 
-/*
-** Truncate the file to the number of pages specified.
-*/
-SQLITE_PRIVATE int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
-  int rc = SQLITE_OK;
-  assert( pPager->state>=PAGER_SHARED );
-
-  sqlite3PagerPagecount(pPager, 0);
-  if( pPager->errCode ){
-    rc = pPager->errCode;
-  }else if( nPage<pPager->dbSize ){
-    rc = syncJournal(pPager);
-    if( rc==SQLITE_OK ){
-      /* Get an exclusive lock on the database before truncating. */
-      rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pager_truncate(pPager, nPage);
-    }
-  }
-
-  return rc;
-}
+#ifndef SQLITE_OMIT_AUTOVACUUM
+/*
+** Truncate the in-memory database file image to nPage pages. This 
+** function does not actually modify the database file on disk. It 
+** just sets the internal state of the pager object so that the 
+** truncation will be done when the current transaction is committed.
+*/
+SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){
+  assert( pPager->dbSizeValid );
+  assert( pPager->dbSize>=nPage );
+  pPager->dbSize = nPage;
+}
+
+/*
+** Return the current size of the database file image in pages. This
+** function differs from sqlite3PagerPagecount() in two ways:
+**
+**  a) It may only be called when at least one reference to a database
+**     page is held. This guarantees that the database size is already
+**     known and a call to sqlite3OsFileSize() is not required.
+**
+**  b) The return value is not adjusted for the locking page.
+*/
+SQLITE_PRIVATE Pgno sqlite3PagerImageSize(Pager *pPager){
+  assert( pPager->dbSizeValid );
+  return pPager->dbSize;
+}
+#endif  /* ifndef SQLITE_OMIT_AUTOVACUUM */
 
 /*
 ** Shutdown the page cache.  Free all memory and close all files.
 **
 ** If a transaction was in progress when this routine is called, that
 ** transaction is rolled back.  All outstanding pages are invalidated
 ** and their memory is freed.  Any attempt to use a page associated
 ** with this page cache after this function returns will likely
@@ -32743,30 +33186,35 @@ SQLITE_PRIVATE int sqlite3PagerTruncate(
 SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){
 
   disable_simulated_io_errors();
   sqlite3BeginBenignMalloc();
   pPager->errCode = 0;
   pPager->exclusiveMode = 0;
   pager_reset(pPager);
   if( !MEMDB ){
+    /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() 
+    ** call which may be made from within pagerUnlockAndRollback(). If it
+    ** is not -1, then the unsynced portion of an open journal file may
+    ** be played back into the database. If a power failure occurs while
+    ** this is happening, the database may become corrupt.
+    */
+    pPager->journalHdr = -1;
     pagerUnlockAndRollback(pPager);
   }
   enable_simulated_io_errors();
   sqlite3EndBenignMalloc();
-  PAGERTRACE2("CLOSE %d\n", PAGERID(pPager));
+  PAGERTRACE(("CLOSE %d\n", PAGERID(pPager)));
   IOTRACE(("CLOSE %p\n", pPager))
   if( pPager->journalOpen ){
     sqlite3OsClose(pPager->jfd);
   }
   sqlite3BitvecDestroy(pPager->pInJournal);
   sqlite3BitvecDestroy(pPager->pAlwaysRollback);
-  if( pPager->stmtOpen ){
-    sqlite3OsClose(pPager->stfd);
-  }
+  releaseAllSavepoint(pPager);
   sqlite3OsClose(pPager->fd);
   /* Temp files are automatically deleted by the OS
   ** if( pPager->tempFile ){
   **   sqlite3OsDelete(pPager->zFilename);
   ** }
   */
 
   sqlite3PageFree(pPager->pTmpSpace);
@@ -32826,42 +33274,70 @@ static int syncJournal(Pager *pPager){
   */
   if( pPager->needSync ){
     assert( !pPager->tempFile );
     if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){
       int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
       assert( pPager->journalOpen );
 
       if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
+        i64 jrnlOff = journalHdrOffset(pPager);
+        u8 zMagic[8];
+
+        /* This block deals with an obscure problem. If the last connection
+        ** that wrote to this database was operating in persistent-journal
+        ** mode, then the journal file may at this point actually be larger
+        ** than Pager.journalOff bytes. If the next thing in the journal
+        ** file happens to be a journal-header (written as part of the
+        ** previous connections transaction), and a crash or power-failure 
+        ** occurs after nRec is updated but before this connection writes 
+        ** anything else to the journal file (or commits/rolls back its 
+        ** transaction), then SQLite may become confused when doing the 
+        ** hot-journal rollback following recovery. It may roll back all
+        ** of this connections data, then proceed to rolling back the old,
+        ** out-of-date data that follows it. Database corruption.
+        **
+        ** To work around this, if the journal file does appear to contain
+        ** a valid header following Pager.journalOff, then write a 0x00
+        ** byte to the start of it to prevent it from being recognized.
+        */
+        rc = sqlite3OsRead(pPager->jfd, zMagic, 8, jrnlOff);
+        if( rc==SQLITE_OK && 0==memcmp(zMagic, aJournalMagic, 8) ){
+          static const u8 zerobyte = 0;
+          rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, jrnlOff);
+        }
+        if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
+          return rc;
+        }
+
         /* Write the nRec value into the journal file header. If in
         ** full-synchronous mode, sync the journal first. This ensures that
         ** all data has really hit the disk before nRec is updated to mark
         ** it as a candidate for rollback.
         **
         ** This is not required if the persistent media supports the
         ** SAFE_APPEND property. Because in this case it is not possible 
         ** for garbage data to be appended to the file, the nRec field
         ** is populated with 0xFFFFFFFF when the journal header is written
         ** and never needs to be updated.
         */
-        i64 jrnlOff;
         if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
-          PAGERTRACE2("SYNC journal of %d\n", PAGERID(pPager));
+          PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
           IOTRACE(("JSYNC %p\n", pPager))
           rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags);
           if( rc!=0 ) return rc;
         }
 
         jrnlOff = pPager->journalHdr + sizeof(aJournalMagic);
         IOTRACE(("JHDR %p %lld %d\n", pPager, jrnlOff, 4));
         rc = write32bits(pPager->jfd, jrnlOff, pPager->nRec);
         if( rc ) return rc;
       }
       if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
-        PAGERTRACE2("SYNC journal of %d\n", PAGERID(pPager));
+        PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
         IOTRACE(("JSYNC %p\n", pPager))
         rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| 
           (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
         );
         if( rc!=0 ) return rc;
       }
       pPager->journalStarted = 1;
     }
@@ -32915,49 +33391,81 @@ static int pager_write_pagelist(PgHdr *p
     /* If the file has not yet been opened, open it now. */
     if( !pPager->fd->pMethods ){
       assert(pPager->tempFile);
       rc = sqlite3PagerOpentemp(pPager, pPager->fd, pPager->vfsFlags);
       if( rc ) return rc;
     }
 
     /* If there are dirty pages in the page cache with page numbers greater
-    ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to
+    ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
     ** make the file smaller (presumably by auto-vacuum code). Do not write
     ** any such pages to the file.
     */
     if( pList->pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
       i64 offset = (pList->pgno-1)*(i64)pPager->pageSize;
       char *pData = CODEC2(pPager, pList->pData, pList->pgno, 6);
-      PAGERTRACE4("STORE %d page %d hash(%08x)\n",
-                   PAGERID(pPager), pList->pgno, pager_pagehash(pList));
+
+      PAGERTRACE(("STORE %d page %d hash(%08x)\n",
+                   PAGERID(pPager), pList->pgno, pager_pagehash(pList)));
       IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno));
       rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset);
       PAGER_INCR(sqlite3_pager_writedb_count);
       PAGER_INCR(pPager->nWrite);
       if( pList->pgno==1 ){
         memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
       }
+      if( pList->pgno>pPager->dbFileSize ){
+        pPager->dbFileSize = pList->pgno;
+      }
     }
 #ifndef NDEBUG
     else{
-      PAGERTRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
+      PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno));
     }
 #endif
     if( rc ) return rc;
 #ifdef SQLITE_CHECK_PAGES
     pList->pageHash = pager_pagehash(pList);
 #endif
     pList = pList->pDirty;
   }
 
   return SQLITE_OK;
 }
 
 /*
+** Add the page to the sub-journal. It is the callers responsibility to
+** use subjRequiresPage() to check that it is really required before 
+** calling this function.
+*/
+static int subjournalPage(PgHdr *pPg){
+  int rc;
+  void *pData = pPg->pData;
+  Pager *pPager = pPg->pPager;
+  i64 offset = pPager->stmtNRec*(4+pPager->pageSize);
+  char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7);
+
+  PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno));
+
+  assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
+  rc = write32bits(pPager->sjfd, offset, pPg->pgno);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4);
+  }
+  if( rc==SQLITE_OK ){
+    pPager->stmtNRec++;
+    assert( pPager->nSavepoint>0 );
+    rc = addToSavepointBitvecs(pPager, pPg->pgno);
+  }
+  return rc;
+}
+
+
+/*
 ** This function is called by the pcache layer when it has reached some
 ** soft memory limit. The argument is a pointer to a purgeable Pager 
 ** object. This function attempts to make a single dirty page that has no
 ** outstanding references (if one exists) clean so that it can be recycled 
 ** by the pcache layer.
 */
 static int pagerStress(void *p, PgHdr *pPg){
   Pager *pPager = (Pager *)p;
@@ -32976,24 +33484,30 @@ static int pagerStress(void *p, PgHdr *p
         !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND)
       ){
         pPager->nRec = 0;
         rc = writeJournalHdr(pPager);
       }
     }
     if( rc==SQLITE_OK ){
       pPg->pDirty = 0;
-      rc = pager_write_pagelist(pPg);
+      if( pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) ){
+        rc = subjournalPage(pPg);
+      }
+      if( rc==SQLITE_OK ){
+        rc = pager_write_pagelist(pPg);
+      }
     }
     if( rc!=SQLITE_OK ){
       pager_error(pPager, rc);
     }
   }
 
   if( rc==SQLITE_OK ){
+    PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno));
     sqlite3PcacheMakeClean(pPg);
   }
   return rc;
 }
 
 
 /*
 ** Return 1 if there is a hot journal on the given pager.
@@ -33057,18 +33571,18 @@ static int readDbPage(Pager *pPager, PgH
   PAGER_INCR(sqlite3_pager_readdb_count);
   PAGER_INCR(pPager->nRead);
   IOTRACE(("PGIN %p %d\n", pPager, pgno));
   if( pgno==1 ){
     memcpy(&pPager->dbFileVers, &((u8*)pPg->pData)[24],
                                               sizeof(pPager->dbFileVers));
   }
   CODEC1(pPager, pPg->pData, pPg->pgno, 3);
-  PAGERTRACE4("FETCH %d page %d hash(%08x)\n",
-               PAGERID(pPager), pPg->pgno, pager_pagehash(pPg));
+  PAGERTRACE(("FETCH %d page %d hash(%08x)\n",
+               PAGERID(pPager), pPg->pgno, pager_pagehash(pPg)));
   return rc;
 }
 
 
 /*
 ** This function is called to obtain the shared lock required before
 ** data may be read from the pager cache. If the shared lock has already
 ** been obtained, this function is a no-op.
@@ -33110,18 +33624,20 @@ static int pagerSharedLock(Pager *pPager
     assert( !MEMDB );
     assert( sqlite3PcacheRefCount(pPager->pPCache)==0 );
     if( !pPager->noReadlock ){
       rc = pager_wait_on_lock(pPager, SHARED_LOCK);
       if( rc!=SQLITE_OK ){
         assert( pPager->state==PAGER_UNLOCK );
         return pager_error(pPager, rc);
       }
-      assert( pPager->state>=SHARED_LOCK );
-    }
+    }else if( pPager->state==PAGER_UNLOCK ){
+      pPager->state = PAGER_SHARED;
+    }
+    assert( pPager->state>=SHARED_LOCK );
 
     /* If a journal file exists, and there is no RESERVED lock on the
     ** database file, then it either needs to be played back or deleted.
     */
     if( !isErrorReset ){
       rc = hasHotJournal(pPager, &isHotJournal);
       if( rc!=SQLITE_OK ){
         goto failed;
@@ -33180,18 +33696,21 @@ static int pagerSharedLock(Pager *pPager
       }
       pPager->journalOpen = 1;
       pPager->journalStarted = 0;
       pPager->journalOff = 0;
       pPager->setMaster = 0;
       pPager->journalHdr = 0;
  
       /* Playback and delete the journal.  Drop the database write
-      ** lock and reacquire the read lock.
-      */
+      ** lock and reacquire the read lock. Purge the cache before
+      ** playing back the hot-journal so that we don't end up with
+      ** an inconsistent cache.
+      */
+      sqlite3PcacheClear(pPager->pPCache);
       rc = pager_playback(pPager, 1);
       if( rc!=SQLITE_OK ){
         rc = pager_error(pPager, rc);
         goto failed;
       }
       assert(pPager->state==PAGER_SHARED || 
           (pPager->exclusiveMode && pPager->state>PAGER_SHARED)
       );
@@ -33232,20 +33751,17 @@ static int pagerSharedLock(Pager *pPager
       }else{
         memset(dbFileVers, 0, sizeof(dbFileVers));
       }
 
       if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){
         pager_reset(pPager);
       }
     }
-    assert( pPager->exclusiveMode || pPager->state<=PAGER_SHARED );
-    if( pPager->state==PAGER_UNLOCK ){
-      pPager->state = PAGER_SHARED;
-    }
+    assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED );
   }
 
  failed:
   if( rc!=SQLITE_OK ){
     /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */
     pager_unlock(pPager);
   }
   return rc;
@@ -33457,16 +33973,37 @@ SQLITE_PRIVATE int sqlite3PagerUnref(DbP
     Pager *pPager = pPg->pPager;
     sqlite3PcacheRelease(pPg);
     pagerUnlockIfUnused(pPager);
   }
   return SQLITE_OK;
 }
 
 /*
+** If the main journal file has already been opened, ensure that the
+** sub-journal file is open too. If the main journal is not open,
+** this function is a no-op.
+**
+** SQLITE_OK is returned if everything goes according to plan. An 
+** SQLITE_IOERR_XXX error code is returned if the call to 
+** sqlite3OsOpen() fails.
+*/
+static int openSubJournal(Pager *pPager){
+  int rc = SQLITE_OK;
+  if( pPager->journalOpen && !pPager->sjfd->pMethods ){
+    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){
+      sqlite3MemJournalOpen(pPager->sjfd);
+    }else{
+      rc = sqlite3PagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL);
+    }
+  }
+  return rc;
+}
+
+/*
 ** Create a journal file for pPager.  There should already be a RESERVED
 ** or EXCLUSIVE lock on the database file when this routine is called.
 **
 ** Return SQLITE_OK if everything.  Return an error code and release the
 ** write lock if anything goes wrong.
 */
 static int pager_open_journal(Pager *pPager){
   sqlite3_vfs *pVfs = pPager->pVfs;
@@ -33515,22 +34052,22 @@ static int pager_open_journal(Pager *pPa
   pPager->journalOpen = 1;
   pPager->journalStarted = 0;
   pPager->needSync = 0;
   pPager->nRec = 0;
   if( pPager->errCode ){
     rc = pPager->errCode;
     goto failed_to_open_journal;
   }
-  pPager->origDbSize = pPager->dbSize;
+  pPager->dbOrigSize = pPager->dbSize;
 
   rc = writeJournalHdr(pPager);
 
-  if( pPager->stmtAutoopen && rc==SQLITE_OK ){
-    rc = sqlite3PagerStmtBegin(pPager);
+  if( pPager->nSavepoint && rc==SQLITE_OK ){
+    rc = openSubJournal(pPager);
   }
   if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && rc!=SQLITE_IOERR_NOMEM ){
     rc = pager_end_transaction(pPager, 0);
     if( rc==SQLITE_OK ){
       rc = SQLITE_FULL;
     }
   }
   return rc;
@@ -33582,37 +34119,37 @@ SQLITE_PRIVATE int sqlite3PagerBegin(DbP
       if( exFlag ){
         rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
       }
     }
     if( rc!=SQLITE_OK ){
       return rc;
     }
     pPager->dirtyCache = 0;
-    PAGERTRACE2("TRANSACTION %d\n", PAGERID(pPager));
+    PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager)));
     if( pPager->useJournal && !pPager->tempFile
            && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){
       rc = pager_open_journal(pPager);
     }
   }else if( pPager->journalOpen && pPager->journalOff==0 ){
     /* This happens when the pager was in exclusive-access mode the last
     ** time a (read or write) transaction was successfully concluded
     ** by this connection. Instead of deleting the journal file it was 
     ** kept open and either was truncated to 0 bytes or its header was
     ** overwritten with zeros.
     */
     assert( pPager->nRec==0 );
-    assert( pPager->origDbSize==0 );
+    assert( pPager->dbOrigSize==0 );
     assert( pPager->pInJournal==0 );
     sqlite3PagerPagecount(pPager, 0);
     pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize );
     if( !pPager->pInJournal ){
       rc = SQLITE_NOMEM;
     }else{
-      pPager->origDbSize = pPager->dbSize;
+      pPager->dbOrigSize = pPager->dbSize;
       rc = writeJournalHdr(pPager);
     }
   }
   assert( !pPager->journalOpen || pPager->journalOff>0 || rc!=SQLITE_OK );
   return rc;
 }
 
 /*
@@ -33662,17 +34199,17 @@ static int pager_write(PgHdr *pPg){
   if( rc ){
     return rc;
   }
 
   /* Mark the page as dirty.  If the page has already been written
   ** to the journal then we can return right away.
   */
   sqlite3PcacheMakeDirty(pPg);
-  if( pageInJournal(pPg) && (pageInStatement(pPg) || pPager->stmtInUse==0) ){
+  if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){
     pPager->dirtyCache = 1;
     pPager->dbModified = 1;
   }else{
 
     /* If we get this far, it means that the page needs to be
     ** written to the transaction journal or the ckeckpoint journal
     ** or both.
     **
@@ -33693,17 +34230,17 @@ static int pager_write(PgHdr *pPg){
     pPager->dirtyCache = 1;
     pPager->dbModified = 1;
   
     /* The transaction journal now exists and we have a RESERVED or an
     ** EXCLUSIVE lock on the main database file.  Write the current page to
     ** the transaction journal if it is not there already.
     */
     if( !pageInJournal(pPg) && pPager->journalOpen ){
-      if( pPg->pgno<=pPager->origDbSize ){
+      if( pPg->pgno<=pPager->dbOrigSize ){
         u32 cksum;
         char *pData2;
 
         /* We should never write to the journal file the page that
         ** contains the database locks.  The following assert verifies
         ** that we do not. */
         assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) );
         pData2 = CODEC2(pPager, pData, pPg->pgno, 7);
@@ -33716,72 +34253,67 @@ static int pager_write(PgHdr *pPg){
         }
         if( rc==SQLITE_OK ){
           rc = write32bits(pPager->jfd, pPager->journalOff, cksum);
           pPager->journalOff += 4;
         }
         IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, 
                  pPager->journalOff, pPager->pageSize));
         PAGER_INCR(sqlite3_pager_writej_count);
-        PAGERTRACE5("JOURNAL %d page %d needSync=%d hash(%08x)\n",
+        PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n",
              PAGERID(pPager), pPg->pgno, 
-             ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg));
+             ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg)));
+
+        /* Even if an IO or diskfull error occurred while journalling the
+        ** page in the block above, set the need-sync flag for the page.
+        ** Otherwise, when the transaction is rolled back, the logic in
+        ** playback_one_page() will think that the page needs to be restored
+        ** in the database file. And if an IO error occurs while doing so,
+        ** then corruption may follow.
+        */
+        if( !pPager->noSync ){
+          pPg->flags |= PGHDR_NEED_SYNC;
+          pPager->needSync = 1;
+        }
 
         /* An error has occured writing to the journal file. The 
         ** transaction will be rolled back by the layer above.
         */
         if( rc!=SQLITE_OK ){
           return rc;
         }
 
         pPager->nRec++;
         assert( pPager->pInJournal!=0 );
-        sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
-        if( !pPager->noSync ){
-          pPg->flags |= PGHDR_NEED_SYNC;
-        }
-        if( pPager->stmtInUse ){
-          sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
+        rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
+        testcase( rc==SQLITE_NOMEM );
+        assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
+        rc |= addToSavepointBitvecs(pPager, pPg->pgno);
+        if( rc!=SQLITE_OK ){
+          assert( rc==SQLITE_NOMEM );
+          return rc;
         }
       }else{
         if( !pPager->journalStarted && !pPager->noSync ){
           pPg->flags |= PGHDR_NEED_SYNC;
-        }
-        PAGERTRACE4("APPEND %d page %d needSync=%d\n",
+          pPager->needSync = 1;
+        }
+        PAGERTRACE(("APPEND %d page %d needSync=%d\n",
                 PAGERID(pPager), pPg->pgno,
-               ((pPg->flags&PGHDR_NEED_SYNC)?1:0));
-      }
-      if( pPg->flags&PGHDR_NEED_SYNC ){
-        pPager->needSync = 1;
+               ((pPg->flags&PGHDR_NEED_SYNC)?1:0)));
       }
     }
   
     /* If the statement journal is open and the page is not in it,
     ** then write the current page to the statement journal.  Note that
     ** the statement journal format differs from the standard journal format
     ** in that it omits the checksums and the header.
     */
-    if( pPager->stmtInUse 
-     && !pageInStatement(pPg) 
-     && pPg->pgno<=pPager->stmtSize 
-    ){
-      i64 offset = pPager->stmtNRec*(4+pPager->pageSize);
-      char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7);
-      assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize );
-      rc = write32bits(pPager->stfd, offset, pPg->pgno);
-      if( rc==SQLITE_OK ){
-        rc = sqlite3OsWrite(pPager->stfd, pData2, pPager->pageSize, offset+4);
-      }
-      PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
-      if( rc!=SQLITE_OK ){
-        return rc;
-      }
-      pPager->stmtNRec++;
-      assert( pPager->pInStmt!=0 );
-      sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
+    if( subjRequiresPage(pPg) ){
+      rc = subjournalPage(pPg);
     }
   }
 
   /* Update the database size and return.
   */
   assert( pPager->state>=PAGER_SHARED );
   if( pPager->dbSize<pPg->pgno ){
     pPager->dbSize = pPg->pgno;
@@ -33846,40 +34378,43 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbP
       PgHdr *pPage;
       if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){
         if( pg!=PAGER_MJ_PGNO(pPager) ){
           rc = sqlite3PagerGet(pPager, pg, &pPage);
           if( rc==SQLITE_OK ){
             rc = pager_write(pPage);
             if( pPage->flags&PGHDR_NEED_SYNC ){
               needSync = 1;
+              assert(pPager->needSync);
             }
             sqlite3PagerUnref(pPage);
           }
         }
       }else if( (pPage = pager_lookup(pPager, pg))!=0 ){
         if( pPage->flags&PGHDR_NEED_SYNC ){
           needSync = 1;
         }
         sqlite3PagerUnref(pPage);
       }
     }
 
-    /* If the PgHdr.needSync flag is set for any of the nPage pages 
+    /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages 
     ** starting at pg1, then it needs to be set for all of them. Because
     ** writing to any of these nPage pages may damage the others, the
     ** journal file must contain sync()ed copies of all of them
     ** before any of them can be written out to the database file.
     */
     if( needSync ){
       assert( !MEMDB && pPager->noSync==0 );
       for(ii=0; ii<nPage && needSync; ii++){
         PgHdr *pPage = pager_lookup(pPager, pg1+ii);
-        if( pPage ) pPage->flags |= PGHDR_NEED_SYNC;
-        sqlite3PagerUnref(pPage);
+        if( pPage ){
+          pPage->flags |= PGHDR_NEED_SYNC;
+          sqlite3PagerUnref(pPage);
+        }
       }
       assert(pPager->needSync);
     }
 
     assert( pPager->doNotSync==1 );
     pPager->doNotSync = 0;
   }else{
     rc = pager_write(pDbPage);
@@ -33908,57 +34443,57 @@ SQLITE_PRIVATE int sqlite3PagerIswriteab
 ** The overlying software layer calls this routine when all of the data
 ** on the given page is unused.  The pager marks the page as clean so
 ** that it does not get written to disk.
 **
 ** Tests show that this optimization, together with the
 ** sqlite3PagerDontRollback() below, more than double the speed
 ** of large INSERT operations and quadruple the speed of large DELETEs.
 **
-** When this routine is called, set the alwaysRollback flag to true.
-** Subsequent calls to sqlite3PagerDontRollback() for the same page
-** will thereafter be ignored.  This is necessary to avoid a problem
-** where a page with data is added to the freelist during one part of
-** a transaction then removed from the freelist during a later part
-** of the same transaction and reused for some other purpose.  When it
-** is first added to the freelist, this routine is called.  When reused,
-** the sqlite3PagerDontRollback() routine is called.  But because the
-** page contains critical data, we still need to be sure it gets
-** rolled back in spite of the sqlite3PagerDontRollback() call.
+** When this routine is called, set the bit corresponding to pDbPage in
+** the Pager.pAlwaysRollback bitvec.  Subsequent calls to
+** sqlite3PagerDontRollback() for the same page will thereafter be ignored.
+** This is necessary to avoid a problem where a page with data is added to
+** the freelist during one part of a transaction then removed from the
+** freelist during a later part of the same transaction and reused for some
+** other purpose.  When it is first added to the freelist, this routine is
+** called.  When reused, the sqlite3PagerDontRollback() routine is called.
+** But because the page contains critical data, we still need to be sure it
+** gets rolled back in spite of the sqlite3PagerDontRollback() call.
 */
 SQLITE_PRIVATE int sqlite3PagerDontWrite(DbPage *pDbPage){
   PgHdr *pPg = pDbPage;
   Pager *pPager = pPg->pPager;
   int rc;
 
-  if( pPg->pgno>pPager->origDbSize ){
+  if( pPg->pgno>pPager->dbOrigSize ){
     return SQLITE_OK;
   }
   if( pPager->pAlwaysRollback==0 ){
     assert( pPager->pInJournal );
-    pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->origDbSize);
+    pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->dbOrigSize);
     if( !pPager->pAlwaysRollback ){
       return SQLITE_NOMEM;
     }
   }
   rc = sqlite3BitvecSet(pPager->pAlwaysRollback, pPg->pgno);
 
-  if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && !pPager->stmtInUse ){
+  if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){
     assert( pPager->state>=PAGER_SHARED );
-    if( pPager->dbSize==pPg->pgno && pPager->origDbSize<pPager->dbSize ){
+    if( pPager->dbSize==pPg->pgno && pPager->dbOrigSize<pPager->dbSize ){
       /* If this pages is the last page in the file and the file has grown
       ** during the current transaction, then do NOT mark the page as clean.
       ** When the database file grows, we must make sure that the last page
       ** gets written at least once so that the disk file will be the correct
       ** size. If you do not write this page and the size of the file
       ** on the disk ends up being too small, that can lead to database
       ** corruption during the next transaction.
       */
     }else{
-      PAGERTRACE3("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager));
+      PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager)));
       IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno))
       pPg->flags |= PGHDR_DONT_WRITE;
 #ifdef SQLITE_CHECK_PAGES
       pPg->pageHash = pager_pagehash(pPg);
 #endif
     }
   }
   return rc;
@@ -33972,74 +34507,85 @@ SQLITE_PRIVATE int sqlite3PagerDontWrite
 **
 ** If we have not yet actually read the content of this page (if
 ** the PgHdr.needRead flag is set) then this routine acts as a promise
 ** that we will never need to read the page content in the future.
 ** so the needRead flag can be cleared at this point.
 */
 SQLITE_PRIVATE void sqlite3PagerDontRollback(DbPage *pPg){
   Pager *pPager = pPg->pPager;
+  TESTONLY( int rc; )  /* Return value from sqlite3BitvecSet() */
 
   assert( pPager->state>=PAGER_RESERVED );
 
   /* If the journal file is not open, or DontWrite() has been called on
-  ** this page (DontWrite() sets the alwaysRollback flag), then this
+  ** this page (DontWrite() sets the Pager.pAlwaysRollback bit), then this
   ** function is a no-op.
   */
   if( pPager->journalOpen==0 
    || sqlite3BitvecTest(pPager->pAlwaysRollback, pPg->pgno)
-   || pPg->pgno>pPager->origDbSize
+   || pPg->pgno>pPager->dbOrigSize
   ){
     return;
   }
 
 #ifdef SQLITE_SECURE_DELETE
   if( sqlite3BitvecTest(pPager->pInJournal, pPg->pgno)!=0
-   || pPg->pgno>pPager->origDbSize ){
+   || pPg->pgno>pPager->dbOrigSize ){
     return;
   }
 #endif
 
   /* If SECURE_DELETE is disabled, then there is no way that this
   ** routine can be called on a page for which sqlite3PagerDontWrite()
   ** has not been previously called during the same transaction.
   ** And if DontWrite() has previously been called, the following
   ** conditions must be met.
   **
   ** (Later:)  Not true.  If the database is corrupted by having duplicate
   ** pages on the freelist (ex: corrupt9.test) then the following is not
   ** necessarily true:
   */
-  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ); */
+  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->dbOrigSize ); */
 
   assert( pPager->pInJournal!=0 );
-  sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
   pPg->flags &= ~PGHDR_NEED_READ;
-  if( pPager->stmtInUse ){
-    assert( pPager->stmtSize >= pPager->origDbSize );
-    sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
-  }
-  PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
+
+  /* Failure to set the bits in the InJournal bit-vectors is benign.
+  ** It merely means that we might do some extra work to journal a page
+  ** that does not need to be journaled.  Nevertheless, be sure to test the
+  ** case where a malloc error occurs while trying to set a bit in a 
+  ** bit vector.
+  */
+  sqlite3BeginBenignMalloc();
+  TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
+  testcase( rc==SQLITE_NOMEM );
+  TESTONLY( rc = ) addToSavepointBitvecs(pPager, pPg->pgno);
+  testcase( rc==SQLITE_NOMEM );
+  sqlite3EndBenignMalloc();
+
+
+  PAGERTRACE(("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager)));
   IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
 }
 
 
 /*
 ** This routine is called to increment the database file change-counter,
 ** stored at byte 24 of the pager file.
 */
 static int pager_incr_changecounter(Pager *pPager, int isDirect){
   PgHdr *pPgHdr;
   u32 change_counter;
   int rc = SQLITE_OK;
 
 #ifndef SQLITE_ENABLE_ATOMIC_WRITE
   assert( isDirect==0 );  /* isDirect is only true for atomic writes */
 #endif
-  if( !pPager->changeCountDone ){
+  if( !pPager->changeCountDone && pPager->dbSize>0 ){
     /* Open page 1 of the file for writing. */
     rc = sqlite3PagerGet(pPager, 1, &pPgHdr);
     if( rc!=SQLITE_OK ) return rc;
 
     if( !isDirect ){
       rc = sqlite3PagerWrite(pPgHdr);
       if( rc!=SQLITE_OK ){
         sqlite3PagerUnref(pPgHdr);
@@ -34050,16 +34596,17 @@ static int pager_incr_changecounter(Page
     /* Increment the value just read and write it back to byte 24. */
     change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers);
     change_counter++;
     put32bits(((char*)pPgHdr->pData)+24, change_counter);
 
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
     if( isDirect && pPager->fd->pMethods ){
       const void *zBuf = pPgHdr->pData;
+      assert( pPager->dbFileSize>0 );
       rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0);
     }
 #endif
 
     /* Release the page reference. */
     sqlite3PagerUnref(pPgHdr);
     pPager->changeCountDone = 1;
   }
@@ -34088,28 +34635,24 @@ SQLITE_PRIVATE int sqlite3PagerSync(Page
 ** This routine ensures that the journal is synced, all dirty pages written
 ** to the database file and the database file synced. The only thing that
 ** remains to commit the transaction is to delete the journal file (or
 ** master journal file if specified).
 **
 ** Note that if zMaster==NULL, this does not overwrite a previous value
 ** passed to an sqlite3PagerCommitPhaseOne() call.
 **
-** If parameter nTrunc is non-zero, then the pager file is truncated to
-** nTrunc pages (this is used by auto-vacuum databases).
-**
 ** If the final parameter - noSync - is true, then the database file itself
 ** is not synced. The caller must call sqlite3PagerSync() directly to
 ** sync the database file before calling CommitPhaseTwo() to delete the
 ** journal file in this case.
 */
 SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(
   Pager *pPager, 
   const char *zMaster, 
-  Pgno nTrunc,
   int noSync
 ){
   int rc = SQLITE_OK;
 
   if( pPager->errCode ){
     return pPager->errCode;
   }
 
@@ -34117,18 +34660,18 @@ SQLITE_PRIVATE int sqlite3PagerCommitPha
   */
   if( pPager->dbModified==0 &&
         (pPager->journalMode!=PAGER_JOURNALMODE_DELETE ||
           pPager->exclusiveMode!=0) ){
     assert( pPager->dirtyCache==0 || pPager->journalOpen==0 );
     return SQLITE_OK;
   }
 
-  PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", 
-      pPager->zFilename, zMaster, nTrunc);
+  PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
+      pPager->zFilename, zMaster, pPager->dbSize));
 
   /* If this is an in-memory db, or no pages have been written to, or this
   ** function has already been called, it is a no-op.
   */
   if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dirtyCache ){
     PgHdr *pPg;
 
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
@@ -34144,17 +34687,17 @@ SQLITE_PRIVATE int sqlite3PagerCommitPha
     ** be created for this transaction.
     */
     int useAtomicWrite;
     pPg = sqlite3PcacheDirtyList(pPager->pPCache);
     useAtomicWrite = (
         !zMaster && 
         pPager->journalOpen &&
         pPager->journalOff==jrnlBufferSize(pPager) && 
-        nTrunc==0 && 
+        pPager->dbSize>=pPager->dbFileSize && 
         (pPg==0 || pPg->pDirty==0)
     );
     assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF );
     if( useAtomicWrite ){
       /* Update the nRec field in the journal file. */
       int offset = pPager->journalHdr + sizeof(aJournalMagic);
       assert(pPager->nRec==1);
       rc = write32bits(pPager->jfd, offset, pPager->nRec);
@@ -34181,73 +34724,73 @@ SQLITE_PRIVATE int sqlite3PagerCommitPha
     ** EXCLUSIVE lock. The next time the process tries to commit the
     ** transaction the m-j name will have already been written.
     */
     if( !pPager->setMaster ){
       rc = pager_incr_changecounter(pPager, 0);
       if( rc!=SQLITE_OK ) goto sync_exit;
       if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){
 #ifndef SQLITE_OMIT_AUTOVACUUM
-        if( nTrunc!=0 ){
+        if( pPager->dbSize<pPager->dbOrigSize ){
           /* If this transaction has made the database smaller, then all pages
           ** being discarded by the truncation must be written to the journal
           ** file.
           */
           Pgno i;
           Pgno iSkip = PAGER_MJ_PGNO(pPager);
-          for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){
+          Pgno dbSize = pPager->dbSize;
+          pPager->dbSize = pPager->dbOrigSize;
+          for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){
             if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){
               rc = sqlite3PagerGet(pPager, i, &pPg);
               if( rc!=SQLITE_OK ) goto sync_exit;
               rc = sqlite3PagerWrite(pPg);
               sqlite3PagerUnref(pPg);
               if( rc!=SQLITE_OK ) goto sync_exit;
             }
           } 
+          pPager->dbSize = dbSize;
         }
 #endif
         rc = writeMasterJournal(pPager, zMaster);
         if( rc!=SQLITE_OK ) goto sync_exit;
         rc = syncJournal(pPager);
       }
     }
     if( rc!=SQLITE_OK ) goto sync_exit;
 
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( nTrunc!=0 ){
-      rc = sqlite3PagerTruncate(pPager, nTrunc);
-      if( rc!=SQLITE_OK ) goto sync_exit;
-    }
-#endif
-
     /* Write all dirty pages to the database file */
     pPg = sqlite3PcacheDirtyList(pPager->pPCache);
     rc = pager_write_pagelist(pPg);
     if( rc!=SQLITE_OK ){
       assert( rc!=SQLITE_IOERR_BLOCKED );
       /* The error might have left the dirty list all fouled up here,
       ** but that does not matter because if the if the dirty list did
       ** get corrupted, then the transaction will roll back and
       ** discard the dirty list.  There is an assert in
       ** pager_get_all_dirty_pages() that verifies that no attempt
       ** is made to use an invalid dirty list.
       */
       goto sync_exit;
     }
     sqlite3PcacheCleanAll(pPager->pPCache);
 
+    if( pPager->dbSize<pPager->dbFileSize ){
+      assert( pPager->state>=PAGER_EXCLUSIVE );
+      rc = pager_truncate(pPager, pPager->dbSize);
+      if( rc!=SQLITE_OK ) goto sync_exit;
+    }
+
     /* Sync the database file. */
     if( !pPager->noSync && !noSync ){
       rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
     }
     IOTRACE(("DBSYNC %p\n", pPager))
 
     pPager->state = PAGER_SYNCED;
-  }else if( MEMDB && nTrunc!=0 ){
-    rc = sqlite3PagerTruncate(pPager, nTrunc);
   }
 
 sync_exit:
   if( rc==SQLITE_IOERR_BLOCKED ){
     /* pager_incr_changecounter() may attempt to obtain an exclusive
      * lock to spill the cache and return IOERR_BLOCKED. But since 
      * there is no chance the cache is inconsistent, it is
      * better to return SQLITE_BUSY.
@@ -34275,17 +34818,17 @@ SQLITE_PRIVATE int sqlite3PagerCommitPha
     return SQLITE_ERROR;
   }
   if( pPager->dbModified==0 &&
         (pPager->journalMode!=PAGER_JOURNALMODE_DELETE ||
           pPager->exclusiveMode!=0) ){
     assert( pPager->dirtyCache==0 || pPager->journalOpen==0 );
     return SQLITE_OK;
   }
-  PAGERTRACE2("COMMIT %d\n", PAGERID(pPager));
+  PAGERTRACE(("COMMIT %d\n", PAGERID(pPager)));
   assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dirtyCache );
   rc = pager_end_transaction(pPager, pPager->setMaster);
   rc = pager_error(pPager, rc);
   return rc;
 }
 
 /*
 ** Rollback all changes.  The database falls back to PAGER_SHARED mode.
@@ -34296,17 +34839,17 @@ SQLITE_PRIVATE int sqlite3PagerCommitPha
 ** the correct locking protocol or unless some other
 ** process is writing trash into the journal file (SQLITE_CORRUPT) or
 ** unless a prior malloc() failed (SQLITE_NOMEM).  Appropriate error
 ** codes are returned for all these occasions.  Otherwise,
 ** SQLITE_OK is returned.
 */
 SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){
   int rc = SQLITE_OK;
-  PAGERTRACE2("ROLLBACK %d\n", PAGERID(pPager));
+  PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager)));
   if( !pPager->dirtyCache || !pPager->journalOpen ){
     rc = pager_end_transaction(pPager, pPager->setMaster);
   }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
     if( pPager->state>=PAGER_EXCLUSIVE ){
       pager_playback(pPager, 0);
     }
     rc = pPager->errCode;
   }else{
@@ -34376,102 +34919,109 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pa
   return a;
 }
 SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){
   return MEMDB;
 }
 #endif
 
 /*
-** Set the statement rollback point.
-**
-** This routine should be called with the transaction journal already
-** open.  A new statement journal is created that can be used to rollback
-** changes of a single SQL command within a larger transaction.
-*/
-static int pagerStmtBegin(Pager *pPager){
-  int rc;
-  assert( !pPager->stmtInUse );
-  assert( pPager->state>=PAGER_SHARED );
-  assert( pPager->dbSizeValid );
-  PAGERTRACE2("STMT-BEGIN %d\n", PAGERID(pPager));
-  if( !pPager->journalOpen ){
-    pPager->stmtAutoopen = 1;
-    return SQLITE_OK;
-  }
-  assert( pPager->journalOpen );
-  assert( pPager->pInStmt==0 );
-  pPager->pInStmt = sqlite3BitvecCreate(pPager->dbSize);
-  if( pPager->pInStmt==0 ){
-    /* sqlite3OsLock(pPager->fd, SHARED_LOCK); */
-    return SQLITE_NOMEM;
-  }
-  pPager->stmtJSize = pPager->journalOff;
-  pPager->stmtSize = pPager->dbSize;
-  pPager->stmtHdrOff = 0;
-  pPager->stmtCksum = pPager->cksumInit;
-  if( !pPager->stmtOpen ){
-    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){
-      sqlite3MemJournalOpen(pPager->stfd);
-    }else{
-      rc = sqlite3PagerOpentemp(pPager, pPager->stfd, SQLITE_OPEN_SUBJOURNAL);
-      if( rc ){
-        goto stmt_begin_failed;
-      }
-    }
-    pPager->stmtOpen = 1;
-    pPager->stmtNRec = 0;
-  }
-  pPager->stmtInUse = 1;
-  return SQLITE_OK;
- 
-stmt_begin_failed:
-  if( pPager->pInStmt ){
-    sqlite3BitvecDestroy(pPager->pInStmt);
-    pPager->pInStmt = 0;
-  }
-  return rc;
-}
-SQLITE_PRIVATE int sqlite3PagerStmtBegin(Pager *pPager){
-  int rc;
-  rc = pagerStmtBegin(pPager);
-  return rc;
-}
-
-/*
-** Commit a statement.
-*/
-SQLITE_PRIVATE int sqlite3PagerStmtCommit(Pager *pPager){
-  if( pPager->stmtInUse ){
-    PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
-    sqlite3BitvecDestroy(pPager->pInStmt);
-    pPager->pInStmt = 0;
-    pPager->stmtNRec = 0;
-    pPager->stmtInUse = 0;
-    if( sqlite3IsMemJournal(pPager->stfd) ){
-      sqlite3OsTruncate(pPager->stfd, 0);
-    }
-  }
-  pPager->stmtAutoopen = 0;
-  return SQLITE_OK;
-}
-
-/*
-** Rollback a statement.
-*/
-SQLITE_PRIVATE int sqlite3PagerStmtRollback(Pager *pPager){
-  int rc;
-  if( pPager->stmtInUse ){
-    PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
-    rc = pager_stmt_playback(pPager);
-    sqlite3PagerStmtCommit(pPager);
-  }else{
-    rc = SQLITE_OK;
-  }
-  pPager->stmtAutoopen = 0;
+** Ensure that there are at least nSavepoint savepoints open.
+*/
+SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){
+  int rc = SQLITE_OK;
+
+  if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){
+    int ii;
+    PagerSavepoint *aNew;
+
+    /* Either there is no active journal or the sub-journal is open or 
+    ** the journal is always stored in memory */
+    assert( pPager->nSavepoint==0 || pPager->sjfd->pMethods ||
+            pPager->journalMode==PAGER_JOURNALMODE_MEMORY );
+
+    /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM
+    ** if the allocation fails. Otherwise, zero the new portion in case a 
+    ** malloc failure occurs while populating it in the for(...) loop below.
+    */
+    aNew = (PagerSavepoint *)sqlite3Realloc(
+        pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint
+    );
+    if( !aNew ){
+      return SQLITE_NOMEM;
+    }
+    memset(&aNew[pPager->nSavepoint], 0,
+        (nSavepoint - pPager->nSavepoint) * sizeof(PagerSavepoint)
+    );
+    pPager->aSavepoint = aNew;
+    ii = pPager->nSavepoint;
+    pPager->nSavepoint = nSavepoint;
+
+    /* Populate the PagerSavepoint structures just allocated. */
+    for(/* no-op */; ii<nSavepoint; ii++){
+      assert( pPager->dbSizeValid );
+      aNew[ii].nOrig = pPager->dbSize;
+      if( pPager->journalOpen && pPager->journalOff>0 ){
+        aNew[ii].iOffset = pPager->journalOff;
+      }else{
+        aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager);
+      }
+      aNew[ii].iSubRec = pPager->stmtNRec;
+      aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize);
+      if( !aNew[ii].pInSavepoint ){
+        return SQLITE_NOMEM;
+      }
+    }
+
+    /* Open the sub-journal, if it is not already opened. */
+    rc = openSubJournal(pPager);
+  }
+
+  return rc;
+}
+
+/*
+** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE.
+** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with
+** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes
+** that have occured since savepoint iSavepoint was created.
+**
+** In either case, all savepoints with an index greater than iSavepoint 
+** are destroyed.
+**
+** If there are less than (iSavepoint+1) active savepoints when this 
+** function is called it is a no-op.
+*/ 
+SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){
+  int rc = SQLITE_OK;
+
+  assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
+
+  if( iSavepoint<pPager->nSavepoint ){
+    int ii;
+    int nNew = iSavepoint + (op==SAVEPOINT_ROLLBACK);
+    for(ii=nNew; ii<pPager->nSavepoint; ii++){
+      sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
+    }
+    pPager->nSavepoint = nNew;
+
+    if( op==SAVEPOINT_ROLLBACK && pPager->jfd->pMethods ){
+      PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1];
+      rc = pagerPlaybackSavepoint(pPager, pSavepoint);
+      assert(rc!=SQLITE_DONE);
+    }
+  
+    /* If this is a release of the outermost savepoint, truncate 
+    ** the sub-journal. */
+    if( nNew==0 && op==SAVEPOINT_RELEASE && pPager->sjfd->pMethods ){
+      assert( rc==SQLITE_OK );
+      rc = sqlite3OsTruncate(pPager->sjfd, 0);
+      pPager->stmtNRec = 0;
+    }
+  }
   return rc;
 }
 
 /*
 ** Return the full pathname of the database file.
 */
 SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager){
   return pPager->zFilename;
@@ -34550,35 +35100,57 @@ SQLITE_PRIVATE void sqlite3PagerSetCodec
 ** If the fourth argument, isCommit, is non-zero, then this page is being
 ** moved as part of a database reorganization just before the transaction 
 ** is being committed. In this case, it is guaranteed that the database page 
 ** pPg refers to will not be written to again within this transaction.
 */
 SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
   PgHdr *pPgOld;  /* The page being overwritten. */
   Pgno needSyncPgno = 0;
+  int rc;
 
   assert( pPg->nRef>0 );
 
-  PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
-      PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno);
+  /* If the page being moved is dirty and has not been saved by the latest
+  ** savepoint, then save the current contents of the page into the 
+  ** sub-journal now. This is required to handle the following scenario:
+  **
+  **   BEGIN;
+  **     <journal page X, then modify it in memory>
+  **     SAVEPOINT one;
+  **       <Move page X to location Y>
+  **     ROLLBACK TO one;
+  **
+  ** If page X were not written to the sub-journal here, it would not
+  ** be possible to restore its contents when the "ROLLBACK TO one"
+  ** statement were processed.
+  */
+  if( pPg->flags&PGHDR_DIRTY 
+   && subjRequiresPage(pPg)
+   && SQLITE_OK!=(rc = subjournalPage(pPg))
+  ){
+    return rc;
+  }
+
+  PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", 
+      PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno));
   IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno))
 
   pager_get_content(pPg);
 
   /* If the journal needs to be sync()ed before page pPg->pgno can
   ** be written to, store pPg->pgno in local variable needSyncPgno.
   **
   ** If the isCommit flag is set, there is no need to remember that
   ** the journal needs to be sync()ed before database page pPg->pgno 
   ** can be written to. The caller has already promised not to write to it.
   */
   if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){
     needSyncPgno = pPg->pgno;
-    assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize );
+    assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
     assert( pPg->flags&PGHDR_DIRTY );
     assert( pPager->needSync );
   }
 
   /* If the cache contains a page with page-number pgno, remove it
   ** from its hash chain. Also, if the PgHdr.needSync was set for 
   ** page pgno before the 'move' operation, it needs to be retained 
   ** for the page moved there.
@@ -34612,22 +35184,21 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(
     ** array. Otherwise, if the page is loaded and written again in
     ** this transaction, it may be written to the database file before
     ** it is synced into the journal file. This way, it may end up in
     ** the journal file twice, but that is not a problem.
     **
     ** The sqlite3PagerGet() call may cause the journal to sync. So make
     ** sure the Pager.needSync flag is set too.
     */
-    int rc;
     PgHdr *pPgHdr;
     assert( pPager->needSync );
     rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
     if( rc!=SQLITE_OK ){
-      if( pPager->pInJournal && needSyncPgno<=pPager->origDbSize ){
+      if( pPager->pInJournal && needSyncPgno<=pPager->dbOrigSize ){
         sqlite3BitvecClear(pPager->pInJournal, needSyncPgno);
       }
       return rc;
     }
     pPager->needSync = 1;
     assert( pPager->noSync==0 && !MEMDB );
     pPgHdr->flags |= PGHDR_NEED_SYNC;
     sqlite3PcacheMakeDirty(pPgHdr);
@@ -34751,17 +35322,17 @@ SQLITE_PRIVATE i64 sqlite3PagerJournalSi
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btreeInt.h,v 1.37 2008/12/10 16:45:51 drh Exp $
+** $Id: btreeInt.h,v 1.38 2008/12/27 15:23:13 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
 **
 **     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
 **     "Sorting And Searching", pages 473-480. Addison-Wesley
 **     Publishing Company, Reading, Massachusetts.
 **
@@ -35108,17 +35679,16 @@ struct BtShared {
   BtCursor *pCursor;    /* A list of all open cursors */
   MemPage *pPage1;      /* First page of the database */
   u8 inStmt;            /* True if we are in a statement subtransaction */
   u8 readOnly;          /* True if the underlying file is readonly */
   u8 pageSizeFixed;     /* True if the page size can no longer be changed */
 #ifndef SQLITE_OMIT_AUTOVACUUM
   u8 autoVacuum;        /* True if auto-vacuum is enabled */
   u8 incrVacuum;        /* True if incr-vacuum is enabled */
-  Pgno nTrunc;          /* Non-zero if the db will be truncated (incr vacuum) */
 #endif
   u16 pageSize;         /* Total number of bytes on a page */
   u16 usableSize;       /* Number of usable bytes on each page */
   u16 maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
   u16 minLocal;         /* Minimum local payload in non-LEAFDATA tables */
   u16 maxLeaf;          /* Maximum local payload in a LEAFDATA table */
   u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */
   u8 inTransaction;     /* Transaction state */
@@ -35684,17 +36254,17 @@ SQLITE_PRIVATE void sqlite3BtreeMutexArr
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.548 2008/12/16 13:46:30 drh Exp $
+** $Id: btree.c,v 1.558 2009/01/10 16:15:21 drh Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** See the header comment on "btreeInt.h" for additional information.
 ** Including a description of file format and an overview of operation.
 */
 
 /*
 ** The header string that appears at the beginning of every
@@ -35708,30 +36278,16 @@ static const char zMagicHeader[] = SQLIT
 */
 #if 0
 int sqlite3BtreeTrace=0;  /* True to enable tracing */
 # define TRACE(X)  if(sqlite3BtreeTrace){printf X;fflush(stdout);}
 #else
 # define TRACE(X)
 #endif
 
-/*
-** Sometimes we need a small amount of code such as a variable initialization
-** to setup for a later assert() statement.  We do not want this code to
-** appear when assert() is disabled.  The following macro is therefore
-** used to contain that setup code.  The "VVA" acronym stands for
-** "Verification, Validation, and Accreditation".  In other words, the
-** code within VVA_ONLY() will only run during verification processes.
-*/
-#ifndef NDEBUG
-# define VVA_ONLY(X)  X
-#else
-# define VVA_ONLY(X)
-#endif
-
 
 
 #ifndef SQLITE_OMIT_SHARED_CACHE
 /*
 ** A list of BtShared objects that are eligible for participation
 ** in shared cache.  This variable has file scope during normal builds,
 ** but the test harness needs to access it so we make it global for 
 ** test builds.
@@ -36855,16 +37411,17 @@ static int getAndInitPage(
 }
 
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to sqlite3BtreeGetPage.
 */
 static void releasePage(MemPage *pPage){
   if( pPage ){
+    assert( pPage->nOverflow==0 || sqlite3PagerPageRefcount(pPage->pDbPage)>1 );
     assert( pPage->aData );
     assert( pPage->pBt );
     assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage );
     assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData );
     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
     sqlite3PagerUnref(pPage->pDbPage);
   }
 }
@@ -37733,16 +38290,24 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTran
       assert( !pBt->pExclusive );
       pBt->pExclusive = p;
     }
 #endif
   }
 
 
 trans_begun:
+  if( rc==SQLITE_OK && wrflag ){
+    /* This call makes sure that the pager has the correct number of
+    ** open savepoints. If the second parameter is greater than 0 and
+    ** the sub-journal is not already open, then it will be opened here.
+    */
+    rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint);
+  }
+
   btreeIntegrity(p);
   sqlite3BtreeLeave(p);
   return rc;
 }
 
 #ifndef SQLITE_OMIT_AUTOVACUUM
 
 /*
@@ -37945,25 +38510,20 @@ static int allocateBtreePage(BtShared *,
 ** is no longer in use.
 **
 ** If the nFin parameter is non-zero, the implementation assumes
 ** that the caller will keep calling incrVacuumStep() until
 ** it returns SQLITE_DONE or an error, and that nFin is the
 ** number of pages the database file will contain after this 
 ** process is complete.
 */
-static int incrVacuumStep(BtShared *pBt, Pgno nFin){
-  Pgno iLastPg;             /* Last page in the database */
+static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
   Pgno nFreeList;           /* Number of pages still on the free-list */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  iLastPg = pBt->nTrunc;
-  if( iLastPg==0 ){
-    iLastPg = pagerPagecount(pBt);
-  }
 
   if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){
     int rc;
     u8 eType;
     Pgno iPtrPage;
 
     nFreeList = get4byte(&pBt->pPage1->aData[36]);
     if( nFreeList==0 || nFin==iLastPg ){
@@ -38027,19 +38587,22 @@ static int incrVacuumStep(BtShared *pBt,
       }
       releasePage(pLastPg);
       if( rc!=SQLITE_OK ){
         return rc;
       }
     }
   }
 
-  pBt->nTrunc = iLastPg - 1;
-  while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){
-    pBt->nTrunc--;
+  if( nFin==0 ){
+    iLastPg--;
+    while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){
+      iLastPg--;
+    }
+    sqlite3PagerTruncateImage(pBt->pPager, iLastPg);
   }
   return SQLITE_OK;
 }
 
 /*
 ** A write-transaction must be opened before calling this function.
 ** It performs a single unit of work towards an incremental vacuum.
 **
@@ -38053,87 +38616,78 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuu
 
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
   assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE );
   if( !pBt->autoVacuum ){
     rc = SQLITE_DONE;
   }else{
     invalidateAllOverflowCache(pBt);
-    rc = incrVacuumStep(pBt, 0);
+    rc = incrVacuumStep(pBt, 0, sqlite3PagerImageSize(pBt->pPager));
   }
   sqlite3BtreeLeave(p);
   return rc;
 }
 
 /*
 ** This routine is called prior to sqlite3PagerCommit when a transaction
 ** is commited for an auto-vacuum database.
 **
 ** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages
 ** the database file should be truncated to during the commit process. 
 ** i.e. the database has been reorganized so that only the first *pnTrunc
 ** pages are in use.
 */
-static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){
+static int autoVacuumCommit(BtShared *pBt){
   int rc = SQLITE_OK;
   Pager *pPager = pBt->pPager;
   VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) );
 
   assert( sqlite3_mutex_held(pBt->mutex) );
   invalidateAllOverflowCache(pBt);
   assert(pBt->autoVacuum);
   if( !pBt->incrVacuum ){
-    Pgno nFin = 0;
-
-    if( pBt->nTrunc==0 ){
-      Pgno nFree;
-      Pgno nPtrmap;
-      const int pgsz = pBt->pageSize;
-      Pgno nOrig = pagerPagecount(pBt);
-
-      if( PTRMAP_ISPAGE(pBt, nOrig) ){
-        return SQLITE_CORRUPT_BKPT;
-      }
-      if( nOrig==PENDING_BYTE_PAGE(pBt) ){
-        nOrig--;
-      }
-      nFree = get4byte(&pBt->pPage1->aData[36]);
-      nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
-      nFin = nOrig - nFree - nPtrmap;
-      if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
-        nFin--;
-      }
-      while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
-        nFin--;
-      }
-    }
-
-    while( rc==SQLITE_OK ){
-      rc = incrVacuumStep(pBt, nFin);
-    }
-    if( rc==SQLITE_DONE ){
-      assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc);
+    Pgno nFin;
+    Pgno nFree;
+    Pgno nPtrmap;
+    Pgno iFree;
+    const int pgsz = pBt->pageSize;
+    Pgno nOrig = pagerPagecount(pBt);
+
+    if( PTRMAP_ISPAGE(pBt, nOrig) ){
+      return SQLITE_CORRUPT_BKPT;
+    }
+    if( nOrig==PENDING_BYTE_PAGE(pBt) ){
+      nOrig--;
+    }
+    nFree = get4byte(&pBt->pPage1->aData[36]);
+    nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
+    nFin = nOrig - nFree - nPtrmap;
+    if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+      nFin--;
+    }
+    while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
+      nFin--;
+    }
+
+    for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){
+      rc = incrVacuumStep(pBt, nFin, iFree);
+    }
+    if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){
       rc = SQLITE_OK;
-      if( pBt->nTrunc && nFin ){
-        rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
-        put4byte(&pBt->pPage1->aData[32], 0);
-        put4byte(&pBt->pPage1->aData[36], 0);
-        pBt->nTrunc = nFin;
-      }
+      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
+      put4byte(&pBt->pPage1->aData[32], 0);
+      put4byte(&pBt->pPage1->aData[36], 0);
+      sqlite3PagerTruncateImage(pBt->pPager, nFin);
     }
     if( rc!=SQLITE_OK ){
       sqlite3PagerRollback(pPager);
     }
   }
 
-  if( rc==SQLITE_OK ){
-    *pnTrunc = pBt->nTrunc;
-    pBt->nTrunc = 0;
-  }
   assert( nRef==sqlite3PagerRefcount(pPager) );
   return rc;
 }
 
 #endif /* ifndef SQLITE_OMIT_AUTOVACUUM */
 
 /*
 ** This routine does the first phase of a two-phase commit.  This routine
@@ -38160,29 +38714,28 @@ static int autoVacuumCommit(BtShared *pB
 **
 ** Once this is routine has returned, the only thing required to commit
 ** the write-transaction for this database file is to delete the journal.
 */
 SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
   int rc = SQLITE_OK;
   if( p->inTrans==TRANS_WRITE ){
     BtShared *pBt = p->pBt;
-    Pgno nTrunc = 0;
     sqlite3BtreeEnter(p);
     pBt->db = p->db;
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
-      rc = autoVacuumCommit(pBt, &nTrunc); 
+      rc = autoVacuumCommit(pBt);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
         return rc;
       }
     }
 #endif
-    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, nTrunc, 0);
+    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
     sqlite3BtreeLeave(p);
   }
   return rc;
 }
 
 /*
 ** Commit the transaction currently in progress.
 **
@@ -38342,20 +38895,16 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(
   }
 #endif
   btreeIntegrity(p);
   unlockAllTables(p);
 
   if( p->inTrans==TRANS_WRITE ){
     int rc2;
 
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    pBt->nTrunc = 0;
-#endif
-
     assert( TRANS_WRITE==pBt->inTransaction );
     rc2 = sqlite3PagerRollback(pBt->pPager);
     if( rc2!=SQLITE_OK ){
       rc = rc2;
     }
 
     /* The rollback may have destroyed the pPage1->aData value.  So
     ** call sqlite3BtreeGetPage() on page 1 again to make
@@ -38399,39 +38948,48 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(
 ** error occurs within the statement, the effect of that one statement
 ** can be rolled back without having to rollback the entire transaction.
 */
 SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p){
   int rc;
   BtShared *pBt = p->pBt;
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
-  if( (p->inTrans!=TRANS_WRITE) || pBt->inStmt ){
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
+  assert( p->inTrans==TRANS_WRITE );
+  assert( !pBt->inStmt );
+  assert( pBt->readOnly==0 );
+  if( NEVER(p->inTrans!=TRANS_WRITE || pBt->inStmt || pBt->readOnly) ){
+    rc = SQLITE_INTERNAL;
   }else{
     assert( pBt->inTransaction==TRANS_WRITE );
-    rc = pBt->readOnly ? SQLITE_OK : sqlite3PagerStmtBegin(pBt->pPager);
+    /* At the pager level, a statement transaction is a savepoint with
+    ** an index greater than all savepoints created explicitly using
+    ** SQL statements. It is illegal to open, release or rollback any
+    ** such savepoints while the statement transaction savepoint is active.
+    */
+    rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint+1);
     pBt->inStmt = 1;
   }
   sqlite3BtreeLeave(p);
   return rc;
 }
 
-
 /*
 ** Commit the statment subtransaction currently in progress.  If no
 ** subtransaction is active, this is a no-op.
 */
 SQLITE_PRIVATE int sqlite3BtreeCommitStmt(Btree *p){
   int rc;
   BtShared *pBt = p->pBt;
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
-  if( pBt->inStmt && !pBt->readOnly ){
-    rc = sqlite3PagerStmtCommit(pBt->pPager);
+  assert( pBt->readOnly==0 );
+  if( pBt->inStmt ){
+    int iStmtpoint = p->db->nSavepoint;
+    rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_RELEASE, iStmtpoint);
   }else{
     rc = SQLITE_OK;
   }
   pBt->inStmt = 0;
   sqlite3BtreeLeave(p);
   return rc;
 }
 
@@ -38443,25 +39001,60 @@ SQLITE_PRIVATE int sqlite3BtreeCommitStm
 ** to use a cursor that was open at the beginning of this operation
 ** will result in an error.
 */
 SQLITE_PRIVATE int sqlite3BtreeRollbackStmt(Btree *p){
   int rc = SQLITE_OK;
   BtShared *pBt = p->pBt;
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
-  if( pBt->inStmt && !pBt->readOnly ){
-    rc = sqlite3PagerStmtRollback(pBt->pPager);
+  assert( pBt->readOnly==0 );
+  if( pBt->inStmt ){
+    int iStmtpoint = p->db->nSavepoint;
+    rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_ROLLBACK, iStmtpoint);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_RELEASE, iStmtpoint);
+    }
     pBt->inStmt = 0;
   }
   sqlite3BtreeLeave(p);
   return rc;
 }
 
 /*
+** The second argument to this function, op, is always SAVEPOINT_ROLLBACK
+** or SAVEPOINT_RELEASE. This function either releases or rolls back the
+** savepoint identified by parameter iSavepoint, depending on the value 
+** of op.
+**
+** Normally, iSavepoint is greater than or equal to zero. However, if op is
+** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the 
+** contents of the entire transaction are rolled back. This is different
+** from a normal transaction rollback, as no locks are released and the
+** transaction remains open.
+*/
+SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){
+  int rc = SQLITE_OK;
+  if( p && p->inTrans==TRANS_WRITE ){
+    BtShared *pBt = p->pBt;
+    assert( pBt->inStmt==0 );
+    assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
+    assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
+    sqlite3BtreeEnter(p);
+    pBt->db = p->db;
+    rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint);
+    if( rc==SQLITE_OK ){
+      rc = newDatabase(pBt);
+    }
+    sqlite3BtreeLeave(p);
+  }
+  return rc;
+}
+
+/*
 ** Create a new cursor for the BTree whose root is on the page
 ** iTable.  The act of acquiring a cursor gets a read lock on 
 ** the database file.
 **
 ** If wrFlag==0, then the cursor can only be used for reading.
 ** If wrFlag==1, then the cursor can be used for reading or for
 ** writing if other conditions for writing are also met.  These
 ** are the conditions that must be met in order for writing to
@@ -38495,32 +39088,30 @@ static int btreeCursor(
 ){
   int rc;
   Pgno nPage;
   BtShared *pBt = p->pBt;
 
   assert( sqlite3BtreeHoldsMutex(p) );
   assert( wrFlag==0 || wrFlag==1 );
   if( wrFlag ){
-    if( pBt->readOnly ){
+    assert( !pBt->readOnly );
+    if( NEVER(pBt->readOnly) ){
       return SQLITE_READONLY;
     }
     if( checkReadLocks(p, iTable, 0, 0) ){
       return SQLITE_LOCKED;
     }
   }
 
   if( pBt->pPage1==0 ){
     rc = lockBtreeWithRetry(p);
     if( rc!=SQLITE_OK ){
       return rc;
     }
-    if( pBt->readOnly && wrFlag ){
-      return SQLITE_READONLY;
-    }
   }
   pCur->pgnoRoot = (Pgno)iTable;
   rc = sqlite3PagerPagecount(pBt->pPager, (int *)&nPage); 
   if( rc!=SQLITE_OK ){
     return rc;
   }
   if( iTable==1 && nPage==0 ){
     rc = SQLITE_EMPTY;
@@ -39391,29 +39982,30 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCu
 ** must be NULL.  For index tables, pIdxKey is used and intKey
 ** is ignored.
 **
 ** If an exact match is not found, then the cursor is always
 ** left pointing at a leaf page which would hold the entry if it
 ** were present.  The cursor might point to an entry that comes
 ** before or after the key.
 **
-** The result of comparing the key with the entry to which the
-** cursor is written to *pRes if pRes!=NULL.  The meaning of
-** this value is as follows:
+** An integer is written into *pRes which is the result of
+** comparing the key with the entry to which the cursor is 
+** pointing.  The meaning of the integer written into
+** *pRes is as follows:
 **
 **     *pRes<0      The cursor is left pointing at an entry that
-**                  is smaller than pKey or if the table is empty
+**                  is smaller than intKey/pIdxKey or if the table is empty
 **                  and the cursor is therefore left point to nothing.
 **
 **     *pRes==0     The cursor is left pointing at an entry that
-**                  exactly matches pKey.
+**                  exactly matches intKey/pIdxKey.
 **
 **     *pRes>0      The cursor is left pointing at an entry that
-**                  is larger than pKey.
+**                  is larger than intKey/pIdxKey.
 **
 */
 SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
   BtCursor *pCur,          /* The cursor to be moved */
   UnpackedRecord *pIdxKey, /* Unpacked index key */
   i64 intKey,              /* The table key */
   int biasRight,           /* If true, bias the search to the high end */
   int *pRes                /* Write search results here */
@@ -39452,26 +40044,26 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnp
   assert( pCur->apPage[0]->intKey || pIdxKey );
   for(;;){
     int lwr, upr;
     Pgno chldPg;
     MemPage *pPage = pCur->apPage[pCur->iPage];
     int c = -1;  /* pRes return if table is empty must be -1 */
     lwr = 0;
     upr = pPage->nCell-1;
-    if( !pPage->intKey && pIdxKey==0 ){
+    if( (!pPage->intKey && pIdxKey==0) || upr<0 ){
       rc = SQLITE_CORRUPT_BKPT;
       goto moveto_finish;
     }
     if( biasRight ){
       pCur->aiIdx[pCur->iPage] = (u16)upr;
     }else{
       pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2);
     }
-    if( lwr<=upr ) for(;;){
+    for(;;){
       void *pCellKey;
       i64 nCellKey;
       int idx = pCur->aiIdx[pCur->iPage];
       pCur->info.nSize = 0;
       pCur->validNKey = 1;
       if( pPage->intKey ){
         u8 *pCell;
         pCell = findCell(pPage, idx) + pPage->childPtrSize;
@@ -39508,17 +40100,17 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnp
       }
       if( c==0 ){
         pCur->info.nKey = nCellKey;
         if( pPage->intKey && !pPage->leaf ){
           lwr = idx;
           upr = lwr - 1;
           break;
         }else{
-          if( pRes ) *pRes = 0;
+          *pRes = 0;
           rc = SQLITE_OK;
           goto moveto_finish;
         }
       }
       if( c<0 ){
         lwr = idx+1;
       }else{
         upr = idx-1;
@@ -39954,39 +40546,26 @@ static int allocateBtreePage(
     }while( searchList );
   }else{
     /* There are no pages on the freelist, so create a new page at the
     ** end of the file */
     int nPage = pagerPagecount(pBt);
     *pPgno = nPage + 1;
 
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pBt->nTrunc ){
-      /* An incr-vacuum has already run within this transaction. So the
-      ** page to allocate is not from the physical end of the file, but
-      ** at pBt->nTrunc. 
-      */
-      *pPgno = pBt->nTrunc+1;
-      if( *pPgno==PENDING_BYTE_PAGE(pBt) ){
-        (*pPgno)++;
-      }
-    }
     if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){
       /* If *pPgno refers to a pointer-map page, allocate two new pages
       ** at the end of the file instead of one. The first allocated page
       ** becomes a new pointer-map page, the second is used by the caller.
       */
       TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
       assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
       (*pPgno)++;
       if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; }
     }
-    if( pBt->nTrunc ){
-      pBt->nTrunc = *pPgno;
-    }
 #endif
 
     assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
     rc = sqlite3BtreeGetPage(pBt, *pPgno, ppPage, 0);
     if( rc ) return rc;
     rc = sqlite3PagerWrite((*ppPage)->pDbPage);
     if( rc!=SQLITE_OK ){
       releasePage(*ppPage);
@@ -40603,16 +41182,17 @@ static int balance_quick(BtCursor *pCur)
   ** not important, as they will be recalculated when the page is rolled
   ** back. But here, in balance_quick(), it is possible that pPage has 
   ** not yet been marked dirty or written into the journal file. Therefore
   ** it will not be rolled back and so it is important to make sure that
   ** the page data and contents of MemPage are consistent.
   */
   pPage->isInit = 0;
   sqlite3BtreeInitPage(pPage);
+  assert( pPage->nOverflow==0 );
 
   /* If everything else succeeded, balance the parent page, in 
   ** case the divider cell inserted caused it to become overfull.
   */
   if( rc==SQLITE_OK ){
     releasePage(pPage);
     pCur->iPage--;
     rc = balance(pCur, 0);
@@ -40651,18 +41231,18 @@ static int balance_quick(BtCursor *pCur)
 ** be rolled back.
 */
 static int balance_nonroot(BtCursor *pCur){
   MemPage *pPage;              /* The over or underfull page to balance */
   MemPage *pParent;            /* The parent of pPage */
   BtShared *pBt;               /* The whole database */
   int nCell = 0;               /* Number of cells in apCell[] */
   int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
-  int nOld;                    /* Number of pages in apOld[] */
-  int nNew;                    /* Number of pages in apNew[] */
+  int nOld = 0;                /* Number of pages in apOld[] */
+  int nNew = 0;                /* Number of pages in apNew[] */
   int nDiv;                    /* Number of cells in apDiv[] */
   int i, j, k;                 /* Loop counters */
   int idx;                     /* Index of pPage in pParent->aCell[] */
   int nxDiv;                   /* Next divider slot in pParent->aCell[] */
   int rc;                      /* The return code */
   int leafCorrection;          /* 4 if pPage is a leaf.  0 if not */
   int leafData;                /* True if pPage is a leaf of a LEAFDATA tree */
   int usableSpace;             /* Bytes in pPage beyond the header */
@@ -40695,17 +41275,17 @@ static int balance_nonroot(BtCursor *pCu
   */
   assert( pCur->iPage>0 );
   assert( pPage->isInit );
   assert( sqlite3PagerIswriteable(pPage->pDbPage) || pPage->nOverflow==1 );
   pBt = pPage->pBt;
   pParent = pCur->apPage[pCur->iPage-1];
   assert( pParent );
   if( SQLITE_OK!=(rc = sqlite3PagerWrite(pParent->pDbPage)) ){
-    return rc;
+    goto balance_cleanup;
   }
 
   TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno));
 
 #ifndef SQLITE_OMIT_QUICKBALANCE
   /*
   ** A special case:  If a new entry has just been inserted into a
   ** table (that is, a btree with integer keys and all data at the leaves)
@@ -40726,34 +41306,28 @@ static int balance_nonroot(BtCursor *pCu
     ** TODO: Check the siblings to the left of pPage. It may be that
     ** they are not full and no new page is required.
     */
     return balance_quick(pCur);
   }
 #endif
 
   if( SQLITE_OK!=(rc = sqlite3PagerWrite(pPage->pDbPage)) ){
-    return rc;
+    goto balance_cleanup;
   }
 
   /*
   ** Find the cell in the parent page whose left child points back
   ** to pPage.  The "idx" variable is the index of that cell.  If pPage
   ** is the rightmost child of pParent then set idx to pParent->nCell 
   */
   idx = pCur->aiIdx[pCur->iPage-1];
   assertParentIndex(pParent, idx, pPage->pgno);
 
   /*
-  ** Initialize variables so that it will be safe to jump
-  ** directly to balance_cleanup at any moment.
-  */
-  nOld = nNew = 0;
-
-  /*
   ** Find sibling pages to pPage and the cells in pParent that divide
   ** the siblings.  An attempt is made to find NN siblings on either
   ** side of pPage.  More siblings are taken from one side, however, if
   ** pPage there are fewer than NN siblings on the other side.  If pParent
   ** has NB or fewer children then all children of pParent are taken.
   */
   nxDiv = idx - NN;
   if( nxDiv + NB > pParent->nCell ){
@@ -41204,37 +41778,36 @@ static int balance_nonroot(BtCursor *pCu
   /*
   ** Balance the parent page.  Note that the current page (pPage) might
   ** have been added to the freelist so it might no longer be initialized.
   ** But the parent page will always be initialized.
   */
   assert( pParent->isInit );
   sqlite3ScratchFree(apCell);
   apCell = 0;
+  TRACE(("BALANCE: finished with %d: old=%d new=%d cells=%d\n",
+          pPage->pgno, nOld, nNew, nCell));
+  pPage->nOverflow = 0;
   releasePage(pPage);
   pCur->iPage--;
   rc = balance(pCur, 0);
   
   /*
   ** Cleanup before returning.
   */
 balance_cleanup:
   sqlite3PageFree(aSpace2);
   sqlite3ScratchFree(apCell);
   for(i=0; i<nOld; i++){
     releasePage(apOld[i]);
   }
   for(i=0; i<nNew; i++){
     releasePage(apNew[i]);
   }
-  pPage->nOverflow = 0;
-
-  /* releasePage(pParent); */
-  TRACE(("BALANCE: finished with %d: old=%d new=%d cells=%d\n",
-          pPage->pgno, nOld, nNew, nCell));
+  pCur->apPage[pCur->iPage]->nOverflow = 0;
 
   return rc;
 }
 
 /*
 ** This routine is called for the root page of a btree when the root
 ** page contains no cells.  This is an opportunity to make the tree
 ** shallower by one level.
@@ -41384,16 +41957,19 @@ static int balance_deeper(BtCursor *pCur
     put4byte(&pPage->aData[pPage->hdrOffset+8], pgnoChild);
     TRACE(("BALANCE: copy root %d into %d\n", pPage->pgno, pChild->pgno));
     if( ISAUTOVACUUM ){
       rc = ptrmapPut(pBt, pChild->pgno, PTRMAP_BTREE, pPage->pgno);
 #ifndef SQLITE_OMIT_AUTOVACUUM
       if( rc==SQLITE_OK ){
         rc = setChildPtrmaps(pChild);
       }
+      if( rc ){
+        pChild->nOverflow = 0;
+      }
 #endif
     }
   }
 
   if( rc==SQLITE_OK ){
     pCur->iPage++;
     pCur->apPage[1] = pChild;
     pCur->aiIdx[0] = 0;
@@ -41418,27 +41994,28 @@ static int balance(BtCursor *pCur, int i
   int rc = SQLITE_OK;
   MemPage *pPage = pCur->apPage[pCur->iPage];
 
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   if( pCur->iPage==0 ){
     rc = sqlite3PagerWrite(pPage->pDbPage);
     if( rc==SQLITE_OK && pPage->nOverflow>0 ){
       rc = balance_deeper(pCur);
+      assert( pCur->apPage[0]==pPage );
       assert( pPage->nOverflow==0 || rc!=SQLITE_OK );
     }
     if( rc==SQLITE_OK && pPage->nCell==0 ){
       rc = balance_shallower(pCur);
+      assert( pCur->apPage[0]==pPage );
       assert( pPage->nOverflow==0 || rc!=SQLITE_OK );
     }
   }else{
     if( pPage->nOverflow>0 || 
         (!isInsert && pPage->nFree>pPage->pBt->usableSize*2/3) ){
       rc = balance_nonroot(pCur);
-      assert( pPage->nOverflow==0 || rc!=SQLITE_OK );
     }
   }
   return rc;
 }
 
 /*
 ** This routine checks all cursors that point to table pgnoRoot.
 ** If any of those cursors were opened with wrFlag==0 in a different
@@ -41530,25 +42107,19 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
   int idx;
   MemPage *pPage;
   Btree *p = pCur->pBtree;
   BtShared *pBt = p->pBt;
   unsigned char *oldCell;
   unsigned char *newCell = 0;
 
   assert( cursorHoldsMutex(pCur) );
-  if( pBt->inTransaction!=TRANS_WRITE ){
-    /* Must start a transaction before doing an insert */
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-    return rc;
-  }
+  assert( pBt->inTransaction==TRANS_WRITE );
   assert( !pBt->readOnly );
-  if( !pCur->wrFlag ){
-    return SQLITE_PERM;   /* Cursor not open for writing */
-  }
+  assert( pCur->wrFlag );
   if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur, nKey) ){
     return SQLITE_LOCKED; /* The table pCur points to has a read lock */
   }
   if( pCur->eState==CURSOR_FAULT ){
     return pCur->skip;
   }
 
   /* Save the positions of any other cursors open on this table */
@@ -41603,18 +42174,17 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
   }
   rc = insertCell(pPage, idx, newCell, szNew, 0, 0);
   if( rc==SQLITE_OK ){
     rc = balance(pCur, 1);
   }
 
   /* Must make sure nOverflow is reset to zero even if the balance()
   ** fails.  Internal data structure corruption will result otherwise. */
-  assert( pPage->nOverflow==0 || rc!=SQLITE_OK );
-  pPage->nOverflow = 0;
+  pCur->apPage[pCur->iPage]->nOverflow = 0;
 
   if( rc==SQLITE_OK ){
     moveToRoot(pCur);
   }
 end_insert:
   return rc;
 }
 
@@ -41628,31 +42198,25 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(Bt
   unsigned char *pCell;
   int rc;
   Pgno pgnoChild = 0;
   Btree *p = pCur->pBtree;
   BtShared *pBt = p->pBt;
 
   assert( cursorHoldsMutex(pCur) );
   assert( pPage->isInit );
-  if( pBt->inTransaction!=TRANS_WRITE ){
-    /* Must start a transaction before doing a delete */
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-    return rc;
-  }
+  assert( pBt->inTransaction==TRANS_WRITE );
   assert( !pBt->readOnly );
   if( pCur->eState==CURSOR_FAULT ){
     return pCur->skip;
   }
-  if( pCur->aiIdx[pCur->iPage]>=pPage->nCell ){
+  if( NEVER(pCur->aiIdx[pCur->iPage]>=pPage->nCell) ){
     return SQLITE_ERROR;  /* The cursor is not pointing to anything */
   }
-  if( !pCur->wrFlag ){
-    return SQLITE_PERM;   /* Did not open this cursor for writing */
-  }
+  assert( pCur->wrFlag );
   if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur, pCur->info.nKey) ){
     return SQLITE_LOCKED; /* The table pCur points to has a read lock */
   }
 
   /* Restore the current cursor position (a no-op if the cursor is not in 
   ** CURSOR_REQUIRESEEK state) and save the positions of any other cursors 
   ** open on the same table. Then call sqlite3PagerWrite() on the page
   ** that the entry will be deleted from.
@@ -41837,21 +42401,17 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(Bt
 */
 static int btreeCreateTable(Btree *p, int *piTable, int flags){
   BtShared *pBt = p->pBt;
   MemPage *pRoot;
   Pgno pgnoRoot;
   int rc;
 
   assert( sqlite3BtreeHoldsMutex(p) );
-  if( pBt->inTransaction!=TRANS_WRITE ){
-    /* Must start a transaction first */
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-    return rc;
-  }
+  assert( pBt->inTransaction==TRANS_WRITE );
   assert( !pBt->readOnly );
 
 #ifdef SQLITE_OMIT_AUTOVACUUM
   rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0);
   if( rc ){
     return rc;
   }
 #else
@@ -42037,19 +42597,18 @@ cleardatabasepage_out:
 ** integer value pointed to by pnChange is incremented by the number of
 ** entries in the table.
 */
 SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){
   int rc;
   BtShared *pBt = p->pBt;
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
-  if( p->inTrans!=TRANS_WRITE ){
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-  }else if( (rc = checkReadLocks(p, iTable, 0, 1))!=SQLITE_OK ){
+  assert( p->inTrans==TRANS_WRITE );
+  if( (rc = checkReadLocks(p, iTable, 0, 1))!=SQLITE_OK ){
     /* nothing to do */
   }else if( SQLITE_OK!=(rc = saveAllCursors(pBt, iTable, 0)) ){
     /* nothing to do */
   }else{
     rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -42076,19 +42635,17 @@ SQLITE_PRIVATE int sqlite3BtreeClearTabl
 ** meta[3] is updated by this procedure.
 */
 static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){
   int rc;
   MemPage *pPage = 0;
   BtShared *pBt = p->pBt;
 
   assert( sqlite3BtreeHoldsMutex(p) );
-  if( p->inTrans!=TRANS_WRITE ){
-    return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-  }
+  assert( p->inTrans==TRANS_WRITE );
 
   /* It is illegal to drop a table if any cursors are open on the
   ** database. This is because in auto-vacuum mode the backend may
   ** need to move another root-page to fill a gap left by the deleted
   ** root page. If an open cursor was using this page a problem would 
   ** occur.
   */
   if( pBt->pCursor ){
@@ -42269,32 +42826,29 @@ SQLITE_PRIVATE int sqlite3BtreeGetMeta(B
 */
 SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){
   BtShared *pBt = p->pBt;
   unsigned char *pP1;
   int rc;
   assert( idx>=1 && idx<=15 );
   sqlite3BtreeEnter(p);
   pBt->db = p->db;
-  if( p->inTrans!=TRANS_WRITE ){
-    rc = pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR;
-  }else{
-    assert( pBt->pPage1!=0 );
-    pP1 = pBt->pPage1->aData;
-    rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
-    if( rc==SQLITE_OK ){
-      put4byte(&pP1[36 + idx*4], iMeta);
+  assert( p->inTrans==TRANS_WRITE );
+  assert( pBt->pPage1!=0 );
+  pP1 = pBt->pPage1->aData;
+  rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
+  if( rc==SQLITE_OK ){
+    put4byte(&pP1[36 + idx*4], iMeta);
 #ifndef SQLITE_OMIT_AUTOVACUUM
-      if( idx==7 ){
-        assert( pBt->autoVacuum || iMeta==0 );
-        assert( iMeta==0 || iMeta==1 );
-        pBt->incrVacuum = (u8)iMeta;
-      }
-#endif
-    }
+    if( idx==7 ){
+      assert( pBt->autoVacuum || iMeta==0 );
+      assert( iMeta==0 || iMeta==1 );
+      pBt->incrVacuum = (u8)iMeta;
+    }
+#endif
   }
   sqlite3BtreeLeave(p);
   return rc;
 }
 
 /*
 ** Return the flag byte at the beginning of the page that the cursor
 ** is currently pointing to.
@@ -42302,18 +42856,19 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMet
 SQLITE_PRIVATE int sqlite3BtreeFlags(BtCursor *pCur){
   /* TODO: What about CURSOR_REQUIRESEEK state? Probably need to call
   ** restoreCursorPosition() here.
   */
   MemPage *pPage;
   restoreCursorPosition(pCur);
   pPage = pCur->apPage[pCur->iPage];
   assert( cursorHoldsMutex(pCur) );
+  assert( pPage!=0 );
   assert( pPage->pBt==pCur->pBt );
-  return pPage ? pPage->aData[pPage->hdrOffset] : 0;
+  return pPage->aData[pPage->hdrOffset];
 }
 
 
 /*
 ** Return the pager associated with a BTree.  This routine is used for
 ** testing and debugging only.
 */
 SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){
@@ -42518,17 +43073,17 @@ static int checkTreePage(
   if( checkRef(pCheck, iPage, zParentContext) ) return 0;
   if( (rc = sqlite3BtreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){
     if( rc==SQLITE_NOMEM ) pCheck->mallocFailed = 1;
     checkAppendMsg(pCheck, zContext,
        "unable to get the page. error code=%d", rc);
     return 0;
   }
   if( (rc = sqlite3BtreeInitPage(pPage))!=0 ){
-    if( rc==SQLITE_NOMEM ) pCheck->mallocFailed = 1;
+    assert( rc==SQLITE_CORRUPT );  /* The only possible error from InitPage */
     checkAppendMsg(pCheck, zContext, 
                    "sqlite3BtreeInitPage() returns error code %d", rc);
     releasePage(pPage);
     return 0;
   }
 
   /* Check out all the cells.
   */
@@ -42687,21 +43242,16 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegri
   }
   sCheck.pBt = pBt;
   sCheck.pPager = pBt->pPager;
   sCheck.nPage = pagerPagecount(sCheck.pBt);
   sCheck.mxErr = mxErr;
   sCheck.nErr = 0;
   sCheck.mallocFailed = 0;
   *pnErr = 0;
-#ifndef SQLITE_OMIT_AUTOVACUUM
-  if( pBt->nTrunc!=0 ){
-    sCheck.nPage = pBt->nTrunc;
-  }
-#endif
   if( sCheck.nPage==0 ){
     unlockBtreeIfUnused(pBt);
     sqlite3BtreeLeave(p);
     return 0;
   }
   sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) );
   if( !sCheck.anRef ){
     unlockBtreeIfUnused(pBt);
@@ -42750,20 +43300,22 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegri
     }
     if( sCheck.anRef[i]!=0 && 
        (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){
       checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i);
     }
 #endif
   }
 
-  /* Make sure this analysis did not leave any unref() pages
+  /* Make sure this analysis did not leave any unref() pages.
+  ** This is an internal consistency check; an integrity check
+  ** of the integrity check.
   */
   unlockBtreeIfUnused(pBt);
-  if( nRef != sqlite3PagerRefcount(pBt->pPager) ){
+  if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){
     checkAppendMsg(&sCheck, 0, 
       "Outstanding page count goes from %d to %d during this analysis",
       nRef, sqlite3PagerRefcount(pBt->pPager)
     );
   }
 
   /* Clean  up and report errors.
   */
@@ -42842,20 +43394,19 @@ static int btreeCopyFile(Btree *pTo, Btr
   BtShared *pBtTo = pTo->pBt;
   BtShared *pBtFrom = pFrom->pBt;
   pBtTo->db = pTo->db;
   pBtFrom->db = pFrom->db;
 
   nToPageSize = pBtTo->pageSize;
   nFromPageSize = pBtFrom->pageSize;
 
-  if( pTo->inTrans!=TRANS_WRITE || pFrom->inTrans!=TRANS_WRITE ){
-    return SQLITE_ERROR;
-  }
-  if( pBtTo->pCursor ){
+  assert( pTo->inTrans==TRANS_WRITE );
+  assert( pFrom->inTrans==TRANS_WRITE );
+  if( NEVER(pBtTo->pCursor) ){
     return SQLITE_BUSY;
   }
 
   nToPage = pagerPagecount(pBtTo);
   nFromPage = pagerPagecount(pBtFrom);
   iSkip = PENDING_BYTE_PAGE(pBtTo);
 
   /* Variable nNewPage is the number of pages required to store the
@@ -42949,86 +43500,80 @@ static int btreeCopyFile(Btree *pTo, Btr
   ** a size that is not an integer multiple of nToPageSize - the current
   ** page size used by the pager associated with B-Tree pTo.
   **
   ** For example, say the page-size of pTo is 2048 bytes and the original 
   ** number of pages is 5 (10 KB file). If pFrom has a page size of 1024 
   ** bytes and 9 pages, then the file needs to be truncated to 9KB.
   */
   if( rc==SQLITE_OK ){
-    if( nFromPageSize!=nToPageSize ){
-      sqlite3_file *pFile = sqlite3PagerFile(pBtTo->pPager);
-      i64 iSize = (i64)nFromPageSize * (i64)nFromPage;
-      i64 iNow = (i64)((nToPage>nNewPage)?nToPage:nNewPage) * (i64)nToPageSize; 
-      i64 iPending = ((i64)PENDING_BYTE_PAGE(pBtTo)-1) *(i64)nToPageSize;
-  
-      assert( iSize<=iNow );
-  
-      /* Commit phase one syncs the journal file associated with pTo 
-      ** containing the original data. It does not sync the database file
-      ** itself. After doing this it is safe to use OsTruncate() and other
-      ** file APIs on the database file directly.
-      */
-      pBtTo->db = pTo->db;
-      rc = sqlite3PagerCommitPhaseOne(pBtTo->pPager, 0, 0, 1);
-      if( iSize<iNow && rc==SQLITE_OK ){
-        rc = sqlite3OsTruncate(pFile, iSize);
-      }
-  
-      /* The loop that copied data from database pFrom to pTo did not
-      ** populate the locking page of database pTo. If the page-size of
-      ** pFrom is smaller than that of pTo, this means some data will
-      ** not have been copied. 
-      **
-      ** This block copies the missing data from database pFrom to pTo 
-      ** using file APIs. This is safe because at this point we know that
-      ** all of the original data from pTo has been synced into the 
-      ** journal file. At this point it would be safe to do anything at
-      ** all to the database file except truncate it to zero bytes.
-      */
-      if( rc==SQLITE_OK && nFromPageSize<nToPageSize && iSize>iPending){
-        i64 iOff;
-        for(
-          iOff=iPending; 
-          rc==SQLITE_OK && iOff<(iPending+nToPageSize); 
-          iOff += nFromPageSize
-        ){
-          DbPage *pFromPage = 0;
-          Pgno iFrom = (Pgno)(iOff/nFromPageSize)+1;
-  
-          if( iFrom==PENDING_BYTE_PAGE(pBtFrom) || iFrom>nFromPage ){
-            continue;
-          }
-  
-          rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage);
-          if( rc==SQLITE_OK ){
-            char *zFrom = sqlite3PagerGetData(pFromPage);
-            rc = sqlite3OsWrite(pFile, zFrom, nFromPageSize, iOff);
-            sqlite3PagerUnref(pFromPage);
-          }
-        }
-      }
-  
-      /* Sync the database file */
-      if( rc==SQLITE_OK ){
-        rc = sqlite3PagerSync(pBtTo->pPager);
-      }
-    }else{
-      rc = sqlite3PagerTruncate(pBtTo->pPager, nNewPage);
-    }
-    if( rc==SQLITE_OK ){
-      pBtTo->pageSizeFixed = 0;
-    }
-  }
-
-  if( rc ){
+    sqlite3_file *pFile = sqlite3PagerFile(pBtTo->pPager);
+    i64 iSize = (i64)nFromPageSize * (i64)nFromPage;
+    i64 iNow = (i64)((nToPage>nNewPage)?nToPage:nNewPage) * (i64)nToPageSize; 
+    i64 iPending = ((i64)PENDING_BYTE_PAGE(pBtTo)-1) *(i64)nToPageSize;
+
+    assert( iSize<=iNow );
+
+    /* Commit phase one syncs the journal file associated with pTo 
+    ** containing the original data. It does not sync the database file
+    ** itself. After doing this it is safe to use OsTruncate() and other
+    ** file APIs on the database file directly.
+    */
+    pBtTo->db = pTo->db;
+    rc = sqlite3PagerCommitPhaseOne(pBtTo->pPager, 0, 1);
+    if( iSize<iNow && rc==SQLITE_OK ){
+      rc = sqlite3OsTruncate(pFile, iSize);
+    }
+
+    /* The loop that copied data from database pFrom to pTo did not
+    ** populate the locking page of database pTo. If the page-size of
+    ** pFrom is smaller than that of pTo, this means some data will
+    ** not have been copied. 
+    **
+    ** This block copies the missing data from database pFrom to pTo 
+    ** using file APIs. This is safe because at this point we know that
+    ** all of the original data from pTo has been synced into the 
+    ** journal file. At this point it would be safe to do anything at
+    ** all to the database file except truncate it to zero bytes.
+    */
+    if( rc==SQLITE_OK && nFromPageSize<nToPageSize && iSize>iPending){
+      i64 iOff;
+      for(
+        iOff=iPending; 
+        rc==SQLITE_OK && iOff<(iPending+nToPageSize); 
+        iOff += nFromPageSize
+      ){
+        DbPage *pFromPage = 0;
+        Pgno iFrom = (Pgno)(iOff/nFromPageSize)+1;
+
+        if( iFrom==PENDING_BYTE_PAGE(pBtFrom) || iFrom>nFromPage ){
+          continue;
+        }
+
+        rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage);
+        if( rc==SQLITE_OK ){
+          char *zFrom = sqlite3PagerGetData(pFromPage);
+          rc = sqlite3OsWrite(pFile, zFrom, nFromPageSize, iOff);
+          sqlite3PagerUnref(pFromPage);
+        }
+      }
+    }
+  }
+
+  /* Sync the database file */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3PagerSync(pBtTo->pPager);
+  }
+  if( rc==SQLITE_OK ){
+    pBtTo->pageSizeFixed = 0;
+  }else{
     sqlite3BtreeRollback(pTo);
   }
 
-  return rc;  
+  return rc;
 }
 SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){
   int rc;
   sqlite3BtreeEnter(pTo);
   sqlite3BtreeEnter(pFrom);
   rc = btreeCopyFile(pTo, pFrom);
   sqlite3BtreeLeave(pFrom);
   sqlite3BtreeLeave(pTo);
@@ -43045,25 +43590,26 @@ SQLITE_PRIVATE int sqlite3BtreeIsInTrans
   return (p && (p->inTrans==TRANS_WRITE));
 }
 
 /*
 ** Return non-zero if a statement transaction is active.
 */
 SQLITE_PRIVATE int sqlite3BtreeIsInStmt(Btree *p){
   assert( sqlite3BtreeHoldsMutex(p) );
-  return (p->pBt && p->pBt->inStmt);
+  return ALWAYS(p->pBt) && p->pBt->inStmt;
 }
 
 /*
 ** Return non-zero if a read (or write) transaction is active.
 */
 SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){
+  assert( p );
   assert( sqlite3_mutex_held(p->db->mutex) );
-  return (p && (p->inTrans!=TRANS_NONE));
+  return p->inTrans!=TRANS_NONE;
 }
 
 /*
 ** This function returns a pointer to a blob of memory associated with
 ** a single shared-btree. The memory is used by client code for its own
 ** purposes (for example, to store a high-level schema associated with 
 ** the shared-btree). The btree layer manages reference counting issues.
 **
@@ -43201,17 +43747,17 @@ SQLITE_PRIVATE void sqlite3BtreeCacheOve
 **
 *************************************************************************
 **
 ** This file contains code use to manipulate "Mem" structure.  A "Mem"
 ** stores a single value in the VDBE.  Mem is an opaque structure visible
 ** only within the VDBE.  Interface routines refer to a Mem using the
 ** name sqlite_value
 **
-** $Id: vdbemem.c,v 1.133 2008/12/10 19:26:24 drh Exp $
+** $Id: vdbemem.c,v 1.134 2009/01/05 22:30:39 drh Exp $
 */
 
 /*
 ** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*)
 ** P if required.
 */
 #define expandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0)
 
@@ -43647,17 +44193,16 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetNul
 }
 
 /*
 ** Delete any previous value and set the value to be a BLOB of length
 ** n containing all zeros.
 */
 SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){
   sqlite3VdbeMemRelease(pMem);
-  MemSetTypeFlag(pMem, MEM_Blob);
   pMem->flags = MEM_Blob|MEM_Zero;
   pMem->type = SQLITE_BLOB;
   pMem->n = 0;
   if( n<0 ) n = 0;
   pMem->u.nZero = n;
   pMem->enc = SQLITE_UTF8;
 }
 
@@ -44289,17 +44834,17 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sql
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains code used for creating, destroying, and populating
 ** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.)  Prior
 ** to version 2.8.7, all this code was combined into the vdbe.c source file.
 ** But that file was getting too big so this subroutines were split out.
 **
-** $Id: vdbeaux.c,v 1.428 2008/12/16 17:20:38 shane Exp $
+** $Id: vdbeaux.c,v 1.430 2009/01/07 08:12:16 danielk1977 Exp $
 */
 
 
 
 /*
 ** When debugging the code generator in a symbolic debugger, one can
 ** set the sqlite3VdbeAddopTrace to 1 and all opcodes will be printed
 ** as they are added to the instruction stream.
@@ -45548,19 +46093,22 @@ static int vdbeCommit(sqlite3 *db, Vdbe 
     if( sqlite3BtreeIsInTrans(pBt) ){
       needXcommit = 1;
       if( i!=1 ) nTrans++;
     }
   }
 
   /* If there are any write-transactions at all, invoke the commit hook */
   if( needXcommit && db->xCommitCallback ){
+    assert( (db->flags & SQLITE_CommitBusy)==0 );
+    db->flags |= SQLITE_CommitBusy;
     (void)sqlite3SafetyOff(db);
     rc = db->xCommitCallback(db->pCommitArg);
     (void)sqlite3SafetyOn(db);
+    db->flags &= ~SQLITE_CommitBusy;
     if( rc ){
       return SQLITE_CONSTRAINT;
     }
   }
 
   /* The simple case - no more than one database file (not counting the
   ** TEMP database) has a transaction active.   There is no need for the
   ** master-journal.
@@ -45852,30 +46400,32 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe 
                    && p->usesStmtJournal ){
           xFunc = sqlite3BtreeRollbackStmt;
         }else{
           /* We are forced to roll back the active transaction. Before doing
           ** so, abort any other statements this handle currently has active.
           */
           invalidateCursorsOnModifiedBtrees(db);
           sqlite3RollbackAll(db);
+          sqlite3CloseSavepoints(db);
           db->autoCommit = 1;
         }
       }
     }
   
     /* If the auto-commit flag is set and this is the only active vdbe, then
     ** we do either a commit or rollback of the current transaction. 
     **
     ** Note: This block also runs if one of the special errors handled 
     ** above has occurred. 
     */
     if( !sqlite3VtabInSync(db) 
      && db->autoCommit 
      && db->writeVdbeCnt==(p->readOnly==0) 
+     && (db->flags & SQLITE_CommitBusy)==0
     ){
       if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){
         /* The auto-commit flag is true, and the vdbe program was 
         ** successful or hit an 'OR FAIL' constraint. This means a commit 
         ** is required.
         */
         int rc = vdbeCommit(db, p);
         if( rc==SQLITE_BUSY ){
@@ -45895,16 +46445,17 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe 
         if( p->openedStatement ){
           xFunc = sqlite3BtreeCommitStmt;
         } 
       }else if( p->errorAction==OE_Abort ){
         xFunc = sqlite3BtreeRollbackStmt;
       }else{
         invalidateCursorsOnModifiedBtrees(db);
         sqlite3RollbackAll(db);
+        sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
       }
     }
   
     /* If xFunc is not NULL, then it is one of sqlite3BtreeRollbackStmt or
     ** sqlite3BtreeCommitStmt. Call it once on each backend. If an error occurs
     ** and the return code is still SQLITE_OK, set the return code to the new
     ** error value.
@@ -48168,17 +48719,17 @@ SQLITE_API int sqlite3_stmt_status(sqlit
 ** a program instruction by instruction.
 **
 ** Various scripts scan this source file in order to generate HTML
 ** documentation, headers files, or other derived files.  The formatting
 ** of the code in this file is, therefore, important.  See other comments
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.803 2008/12/15 15:27:52 drh Exp $
+** $Id: vdbe.c,v 1.811 2009/01/14 00:55:10 drh Exp $
 */
 
 /*
 ** The following global variable is incremented every time a cursor
 ** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes.  The test
 ** procedures use this information to make sure that indices are
 ** working correctly.  This variable has no function other than to
 ** help verify the correct operation of the library.
@@ -48712,16 +49263,36 @@ static int fileExists(sqlite3 *db, const
   extern int sqlite3_io_error_pending;
   if( sqlite3_io_error_pending<=0 )
 #endif
     rc = sqlite3OsAccess(db->pVfs, zFile, SQLITE_ACCESS_EXISTS, &res);
   return (res && rc==SQLITE_OK);
 }
 #endif
 
+#ifndef NDEBUG
+/*
+** This function is only called from within an assert() expression. It
+** checks that the sqlite3.nTransaction variable is correctly set to
+** the number of non-transaction savepoints currently in the 
+** linked list starting at sqlite3.pSavepoint.
+** 
+** Usage:
+**
+**     assert( checkSavepointCount(db) );
+*/
+static int checkSavepointCount(sqlite3 *db){
+  int n = 0;
+  Savepoint *p;
+  for(p=db->pSavepoint; p; p=p->pNext) n++;
+  assert( n==(db->nSavepoint + db->isTransactionSavepoint) );
+  return 1;
+}
+#endif
+
 /*
 ** Execute as much of a VDBE program as we can then return.
 **
 ** sqlite3VdbeMakeReady() must be called before this routine in order to
 ** close the program with a final OP_Halt and to set up the callbacks
 ** and the error message pointer.
 **
 ** Whenever a row or result data is available, this routine will either
@@ -49002,21 +49573,18 @@ case OP_Return: {           /* in1 */
   pc = (int)pIn1->u.i;
   break;
 }
 
 /* Opcode:  Yield P1 * * * *
 **
 ** Swap the program counter with the value in register P1.
 */
-case OP_Yield: {
+case OP_Yield: {            /* in1 */
   int pcDest;
-  assert( pOp->p1>0 );
-  assert( pOp->p1<=p->nMem );
-  pIn1 = &p->aMem[pOp->p1];
   assert( (pIn1->flags & MEM_Dyn)==0 );
   pIn1->flags = MEM_Int;
   pcDest = (int)pIn1->u.i;
   pIn1->u.i = pc;
   REGISTER_TRACE(pOp->p1, pIn1);
   pc = pcDest;
   break;
 }
@@ -49226,20 +49794,17 @@ case OP_Move: {
 
 /* Opcode: Copy P1 P2 * * *
 **
 ** Make a copy of register P1 into register P2.
 **
 ** This instruction makes a deep copy of the value.  A duplicate
 ** is made of any string or blob constant.  See also OP_SCopy.
 */
-case OP_Copy: {
-  assert( pOp->p1>0 );
-  assert( pOp->p1<=p->nMem );
-  pIn1 = &p->aMem[pOp->p1];
+case OP_Copy: {             /* in1 */
   assert( pOp->p2>0 );
   assert( pOp->p2<=p->nMem );
   pOut = &p->aMem[pOp->p2];
   assert( pOut!=pIn1 );
   sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
   Deephemeralize(pOut);
   REGISTER_TRACE(pOp->p2, pOut);
   break;
@@ -49252,20 +49817,17 @@ case OP_Copy: {
 ** This instruction makes a shallow copy of the value.  If the value
 ** is a string or blob, then the copy is only a pointer to the
 ** original and hence if the original changes so will the copy.
 ** Worse, if the original is deallocated, the copy becomes invalid.
 ** Thus the program must guarantee that the original will not change
 ** during the lifetime of the copy.  Use OP_Copy to make a complete
 ** copy.
 */
-case OP_SCopy: {
-  assert( pOp->p1>0 );
-  assert( pOp->p1<=p->nMem );
-  pIn1 = &p->aMem[pOp->p1];
+case OP_SCopy: {            /* in1 */
   REGISTER_TRACE(pOp->p1, pIn1);
   assert( pOp->p2>0 );
   assert( pOp->p2<=p->nMem );
   pOut = &p->aMem[pOp->p2];
   assert( pOut!=pIn1 );
   sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
   REGISTER_TRACE(pOp->p2, pOut);
   break;
@@ -49705,17 +50267,17 @@ case OP_RealAffinity: {                 
 */
 case OP_ToText: {                  /* same as TK_TO_TEXT, in1 */
   if( pIn1->flags & MEM_Null ) break;
   assert( MEM_Str==(MEM_Blob>>3) );
   pIn1->flags |= (pIn1->flags&MEM_Blob)>>3;
   applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
   rc = ExpandBlob(pIn1);
   assert( pIn1->flags & MEM_Str || db->mallocFailed );
-  pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob);
+  pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero);
   UPDATE_MAX_BLOBSIZE(pIn1);
   break;
 }
 
 /* Opcode: ToBlob P1 * * * *
 **
 ** Force the value in register P1 to be a BLOB.
 ** If the value is numeric, convert it to a string first.
@@ -49724,18 +50286,20 @@ case OP_ToText: {                  /* sa
 **
 ** A NULL value is not changed by this routine.  It remains NULL.
 */
 case OP_ToBlob: {                  /* same as TK_TO_BLOB, in1 */
   if( pIn1->flags & MEM_Null ) break;
   if( (pIn1->flags & MEM_Blob)==0 ){
     applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
     assert( pIn1->flags & MEM_Str || db->mallocFailed );
-  }
-  MemSetTypeFlag(pIn1, MEM_Blob);
+    MemSetTypeFlag(pIn1, MEM_Blob);
+  }else{
+    pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob);
+  }
   UPDATE_MAX_BLOBSIZE(pIn1);
   break;
 }
 
 /* Opcode: ToNumeric P1 * * * *
 **
 ** Force the value in register P1 to be numeric (either an
 ** integer or a floating-point number.)
@@ -50569,16 +51133,150 @@ case OP_Statement: {
     if( !sqlite3BtreeIsInStmt(pBt) ){
       rc = sqlite3BtreeBeginStmt(pBt);
       p->openedStatement = 1;
     }
   }
   break;
 }
 
+/* Opcode: Savepoint P1 * * P4 *
+**
+** Open, release or rollback the savepoint named by parameter P4, depending
+** on the value of P1. To open a new savepoint, P1==0. To release (commit) an
+** existing savepoint, P1==1, or to rollback an existing savepoint P1==2.
+*/
+case OP_Savepoint: {
+  int p1 = pOp->p1;
+  char *zName = pOp->p4.z;         /* Name of savepoint */
+
+  /* Assert that the p1 parameter is valid. Also that if there is no open
+  ** transaction, then there cannot be any savepoints. 
+  */
+  assert( db->pSavepoint==0 || db->autoCommit==0 );
+  assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK );
+  assert( db->pSavepoint || db->isTransactionSavepoint==0 );
+  assert( checkSavepointCount(db) );
+
+  if( p1==SAVEPOINT_BEGIN ){
+    if( db->writeVdbeCnt>0 ){
+      /* A new savepoint cannot be created if there are active write 
+      ** statements (i.e. open read/write incremental blob handles).
+      */
+      sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - "
+        "SQL statements in progress");
+      rc = SQLITE_BUSY;
+    }else{
+      int nName = sqlite3Strlen30(zName);
+      Savepoint *pNew;
+
+      /* Create a new savepoint structure. */
+      pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1);
+      if( pNew ){
+        pNew->zName = (char *)&pNew[1];
+        memcpy(pNew->zName, zName, nName+1);
+    
+        /* If there is no open transaction, then mark this as a special
+        ** "transaction savepoint". */
+        if( db->autoCommit ){
+          db->autoCommit = 0;
+          db->isTransactionSavepoint = 1;
+        }else{
+          db->nSavepoint++;
+	}
+    
+        /* Link the new savepoint into the database handle's list. */
+        pNew->pNext = db->pSavepoint;
+        db->pSavepoint = pNew;
+      }
+    }
+  }else{
+    Savepoint *pSavepoint;
+    int iSavepoint = 0;
+
+    /* Find the named savepoint. If there is no such savepoint, then an
+    ** an error is returned to the user.  */
+    for(
+      pSavepoint=db->pSavepoint; 
+      pSavepoint && sqlite3StrICmp(pSavepoint->zName, zName);
+      pSavepoint=pSavepoint->pNext
+    ){
+      iSavepoint++;
+    }
+    if( !pSavepoint ){
+      sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName);
+      rc = SQLITE_ERROR;
+    }else if( 
+        db->writeVdbeCnt>0 || (p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) 
+    ){
+      /* It is not possible to release (commit) a savepoint if there are 
+      ** active write statements. It is not possible to rollback a savepoint
+      ** if there are any active statements at all.
+      */
+      sqlite3SetString(&p->zErrMsg, db, 
+        "cannot %s savepoint - SQL statements in progress",
+        (p1==SAVEPOINT_ROLLBACK ? "rollback": "release")
+      );
+      rc = SQLITE_BUSY;
+    }else{
+
+      /* Determine whether or not this is a transaction savepoint. If so,
+      ** and this is a RELEASE command, then the current transaction 
+      ** is committed. 
+      */
+      int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
+      if( isTransaction && p1==SAVEPOINT_RELEASE ){
+        db->autoCommit = 1;
+        if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
+          p->pc = pc;
+          db->autoCommit = 0;
+          p->rc = rc = SQLITE_BUSY;
+          goto vdbe_return;
+        }
+        db->isTransactionSavepoint = 0;
+        rc = p->rc;
+      }else{
+        int ii;
+        iSavepoint = db->nSavepoint - iSavepoint - 1;
+        for(ii=0; ii<db->nDb; ii++){
+          rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint);
+          if( rc!=SQLITE_OK ){
+            goto abort_due_to_error;
+	  }
+        }
+        if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){
+          sqlite3ExpirePreparedStatements(db);
+          sqlite3ResetInternalSchema(db, 0);
+        }
+      }
+  
+      /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all 
+      ** savepoints nested inside of the savepoint being operated on. */
+      while( db->pSavepoint!=pSavepoint ){
+        Savepoint *pTmp = db->pSavepoint;
+        db->pSavepoint = pTmp->pNext;
+        sqlite3DbFree(db, pTmp);
+        db->nSavepoint--;
+      }
+
+      /* If it is a RELEASE, then destroy the savepoint being operated on too */
+      if( p1==SAVEPOINT_RELEASE ){
+        assert( pSavepoint==db->pSavepoint );
+        db->pSavepoint = pSavepoint->pNext;
+        sqlite3DbFree(db, pSavepoint);
+        if( !isTransaction ){
+          db->nSavepoint--;
+        }
+      }
+    }
+  }
+
+  break;
+}
+
 /* Opcode: AutoCommit P1 P2 * * *
 **
 ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll
 ** back any currently active btree transactions. If there are any active
 ** VMs (apart from this one), then the COMMIT or ROLLBACK statement fails.
 **
 ** This instruction causes the VM to halt.
 */
@@ -50603,29 +51301,30 @@ case OP_AutoCommit: {
   }else if( turnOnAC && !rollback && db->writeVdbeCnt>1 ){
     /* If this instruction implements a COMMIT and other VMs are writing
     ** return an error indicating that the other VMs must complete first. 
     */
     sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - "
         "SQL statements in progress");
     rc = SQLITE_BUSY;
   }else if( desiredAutoCommit!=db->autoCommit ){
-    if( pOp->p2 ){
+    if( rollback ){
       assert( desiredAutoCommit==1 );
       sqlite3RollbackAll(db);
       db->autoCommit = 1;
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
       if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
         p->pc = pc;
         db->autoCommit = (u8)(1-desiredAutoCommit);
         p->rc = rc = SQLITE_BUSY;
         goto vdbe_return;
       }
     }
+    sqlite3CloseSavepoints(db);
     if( p->rc==SQLITE_OK ){
       rc = SQLITE_DONE;
     }else{
       rc = SQLITE_ERROR;
     }
     goto vdbe_return;
   }else{
     sqlite3SetString(&p->zErrMsg, db,
@@ -51973,16 +52672,17 @@ case OP_Last: {        /* jump */
   assert( i>=0 && i<p->nCursor );
   pC = p->apCsr[i];
   assert( pC!=0 );
   pCrsr = pC->pCursor;
   assert( pCrsr!=0 );
   rc = sqlite3BtreeLast(pCrsr, &res);
   pC->nullRow = (u8)res;
   pC->deferredMoveto = 0;
+  pC->rowidIsValid = 0;
   pC->cacheStatus = CACHE_STALE;
   if( res && pOp->p2>0 ){
     pc = pOp->p2 - 1;
   }
   break;
 }
 
 
@@ -52023,16 +52723,17 @@ case OP_Rewind: {        /* jump */
   assert( i>=0 && i<p->nCursor );
   pC = p->apCsr[i];
   assert( pC!=0 );
   if( (pCrsr = pC->pCursor)!=0 ){
     rc = sqlite3BtreeFirst(pCrsr, &res);
     pC->atFirst = res==0 ?1:0;
     pC->deferredMoveto = 0;
     pC->cacheStatus = CACHE_STALE;
+    pC->rowidIsValid = 0;
   }else{
     res = 1;
   }
   pC->nullRow = (u8)res;
   assert( pOp->p2>0 && pOp->p2<p->nOp );
   if( res ){
     pc = pOp->p2 - 1;
   }
@@ -52566,26 +53267,26 @@ case OP_RowSetAdd: {       /* in2 */
 ** unchanged and jump to instruction P2.
 */
 case OP_RowSetRead: {       /* jump, out3 */
   Mem *pIdx;
   i64 val;
   assert( pOp->p1>0 && pOp->p1<=p->nMem );
   CHECK_FOR_INTERRUPT;
   pIdx = &p->aMem[pOp->p1];
+  pOut = &p->aMem[pOp->p3];
   if( (pIdx->flags & MEM_RowSet)==0 
    || sqlite3RowSetNext(pIdx->u.pRowSet, &val)==0
   ){
     /* The boolean index is empty */
     sqlite3VdbeMemSetNull(pIdx);
     pc = pOp->p2 - 1;
   }else{
     /* A value was pulled from the index */
     assert( pOp->p3>0 && pOp->p3<=p->nMem );
-    pOut = &p->aMem[pOp->p3];
     sqlite3VdbeMemSetInt64(pOut, val);
   }
   break;
 }
 
 
 #ifndef SQLITE_OMIT_TRIGGER
 /* Opcode: ContextPush * * * 
@@ -54008,17 +54709,17 @@ SQLITE_PRIVATE int sqlite3JournalSize(sq
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 **
 ** This file contains code use to implement an in-memory rollback journal.
 ** The in-memory rollback journal is used to journal transactions for
 ** ":memory:" databases and when the journal_mode=MEMORY pragma is used.
 **
-** @(#) $Id: memjournal.c,v 1.7 2008/12/10 21:19:57 drh Exp $
+** @(#) $Id: memjournal.c,v 1.8 2008/12/20 02:14:40 drh Exp $
 */
 
 /* Forward references to internal structures */
 typedef struct MemJournal MemJournal;
 typedef struct FilePoint FilePoint;
 typedef struct FileChunk FileChunk;
 
 /* Space to hold the rollback journal is allocated in increments of
@@ -54233,17 +54934,17 @@ SQLITE_PRIVATE void sqlite3MemJournalOpe
 SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){
   return pJfd->pMethods==&MemJournalMethods;
 }
 
 /* 
 ** Return the number of bytes required to store a MemJournal that uses vfs
 ** pVfs to create the underlying on-disk files.
 */
-SQLITE_PRIVATE int sqlite3MemJournalSize(){
+SQLITE_PRIVATE int sqlite3MemJournalSize(void){
   return sizeof(MemJournal);
 }
 
 /************** End of memjournal.c ******************************************/
 /************** Begin file walker.c ******************************************/
 /*
 ** 2008 August 16
 **
@@ -55556,17 +56257,17 @@ SQLITE_PRIVATE void sqlite3ResolveSelect
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains routines used for analyzing expressions and
 ** for generating VDBE code that evaluates expressions in SQLite.
 **
-** $Id: expr.c,v 1.408 2008/12/15 15:27:52 drh Exp $
+** $Id: expr.c,v 1.409 2009/01/10 13:24:51 drh Exp $
 */
 
 /*
 ** Return the 'affinity' of the expression pExpr if any.
 **
 ** If pExpr is a column, a reference to a column via an 'AS' alias,
 ** or a sub-select with a column as the return value, then the 
 ** affinity of that column is returned. Otherwise, 0x00 is returned,
@@ -56462,28 +57163,26 @@ static int exprNodeIsConstant(Walker *pW
   switch( pExpr->op ){
     /* Consider functions to be constant if all their arguments are constant
     ** and pWalker->u.i==2 */
     case TK_FUNCTION:
       if( pWalker->u.i==2 ) return 0;
       /* Fall through */
     case TK_ID:
     case TK_COLUMN:
-    case TK_DOT:
     case TK_AGG_FUNCTION:
     case TK_AGG_COLUMN:
 #ifndef SQLITE_OMIT_SUBQUERY
     case TK_SELECT:
     case TK_EXISTS:
       testcase( pExpr->op==TK_SELECT );
       testcase( pExpr->op==TK_EXISTS );
 #endif
       testcase( pExpr->op==TK_ID );
       testcase( pExpr->op==TK_COLUMN );
-      testcase( pExpr->op==TK_DOT );
       testcase( pExpr->op==TK_AGG_FUNCTION );
       testcase( pExpr->op==TK_AGG_COLUMN );
       pWalker->u.i = 0;
       return WRC_Abort;
     default:
       return WRC_Continue;
   }
 }
@@ -56580,50 +57279,43 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(
 */
 SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
   if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
   if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
   if( sqlite3StrICmp(z, "OID")==0 ) return 1;
   return 0;
 }
 
-#ifdef SQLITE_TEST
-  int sqlite3_enable_in_opt = 1;
-#else
-  #define sqlite3_enable_in_opt 1
-#endif
-
 /*
 ** Return true if the IN operator optimization is enabled and
 ** the SELECT statement p exists and is of the
 ** simple form:
 **
 **     SELECT <column> FROM <table>
 **
 ** If this is the case, it may be possible to use an existing table
 ** or index instead of generating an epheremal table.
 */
 #ifndef SQLITE_OMIT_SUBQUERY
 static int isCandidateForInOpt(Select *p){
   SrcList *pSrc;
   ExprList *pEList;
   Table *pTab;
-  if( !sqlite3_enable_in_opt ) return 0; /* IN optimization must be enabled */
   if( p==0 ) return 0;                   /* right-hand side of IN is SELECT */
   if( p->pPrior ) return 0;              /* Not a compound SELECT */
   if( p->selFlags & (SF_Distinct|SF_Aggregate) ){
       return 0; /* No DISTINCT keyword and no aggregate functions */
   }
   if( p->pGroupBy ) return 0;            /* Has no GROUP BY clause */
   if( p->pLimit ) return 0;              /* Has no LIMIT clause */
   if( p->pOffset ) return 0;
   if( p->pWhere ) return 0;              /* Has no WHERE clause */
   pSrc = p->pSrc;
-  if( pSrc==0 ) return 0;                /* A single table in the FROM clause */
-  if( pSrc->nSrc!=1 ) return 0;
+  assert( pSrc!=0 );
+  if( pSrc->nSrc!=1 ) return 0;          /* Single term in FROM clause */
   if( pSrc->a[0].pSelect ) return 0;     /* FROM clause is not a subquery */
   pTab = pSrc->a[0].pTab;
   if( pTab==0 ) return 0;
   if( pTab->pSelect ) return 0;          /* FROM clause is not a view */
   if( IsVirtual(pTab) ) return 0;        /* FROM clause not a virtual table */
   pEList = p->pEList;
   if( pEList->nExpr!=1 ) return 0;       /* One column in the result set */
   if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */
@@ -57200,43 +57892,32 @@ static int usedAsColumnCache(Parse *pPar
   for(i=0; i<pParse->nColCache; i++){
     int r = pParse->aColCache[i].iReg;
     if( r>=iFrom && r<=iTo ) return 1;
   }
   return 0;
 }
 
 /*
-** Theres is a value in register iCurrent.  We ultimately want
-** the value to be in register iTarget.  It might be that
-** iCurrent and iTarget are the same register.
+** There is a value in register iReg.
 **
 ** We are going to modify the value, so we need to make sure it
-** is not a cached register.  If iCurrent is a cached register,
-** then try to move the value over to iTarget.  If iTarget is a
-** cached register, then clear the corresponding cache line.
-**
-** Return the register that the value ends up in.
-*/
-SQLITE_PRIVATE int sqlite3ExprWritableRegister(Parse *pParse, int iCurrent, int iTarget){
-  int i;
-  assert( pParse->pVdbe!=0 );
-  if( !usedAsColumnCache(pParse, iCurrent, iCurrent) ){
-    return iCurrent;
-  }
-  if( iCurrent!=iTarget ){
-    sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, iCurrent, iTarget);
-  }
-  for(i=0; i<pParse->nColCache; i++){
-    if( pParse->aColCache[i].iReg==iTarget ){
-      pParse->aColCache[i] = pParse->aColCache[--pParse->nColCache];
-      pParse->iColCache = pParse->nColCache;
-    }
-  }
-  return iTarget;
+** is not a cached register.  If iReg is a cached register,
+** then clear the corresponding cache line.
+*/
+SQLITE_PRIVATE void sqlite3ExprWritableRegister(Parse *pParse, int iReg){
+  int i;
+  if( usedAsColumnCache(pParse, iReg, iReg) ){
+    for(i=0; i<pParse->nColCache; i++){
+      if( pParse->aColCache[i].iReg==iReg ){
+        pParse->aColCache[i] = pParse->aColCache[--pParse->nColCache];
+        pParse->iColCache = pParse->nColCache;
+      }
+    }
+  }
 }
 
 /*
 ** If the last instruction coded is an ephemeral copy of any of
 ** the registers in the nReg registers beginning with iReg, then
 ** convert the last instruction from OP_SCopy to OP_Copy.
 */
 SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse *pParse, int iReg, int nReg){
@@ -58619,17 +59300,17 @@ SQLITE_PRIVATE void sqlite3ExprAnalyzeAg
 SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){
   if( pParse->nTempReg==0 ){
     return ++pParse->nMem;
   }
   return pParse->aTempReg[--pParse->nTempReg];
 }
 SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){
   if( iReg && pParse->nTempReg<ArraySize(pParse->aTempReg) ){
-    sqlite3ExprWritableRegister(pParse, iReg, iReg);
+    sqlite3ExprWritableRegister(pParse, iReg);
     pParse->aTempReg[pParse->nTempReg++] = iReg;
   }
 }
 
 /*
 ** Allocate or deallocate a block of nReg consecutive registers
 */
 SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){
@@ -60506,17 +61187,17 @@ SQLITE_PRIVATE void sqlite3AuthContextPo
 **     DROP TABLE
 **     CREATE INDEX
 **     DROP INDEX
 **     creating ID lists
 **     BEGIN TRANSACTION
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.508 2008/12/10 22:30:25 shane Exp $
+** $Id: build.c,v 1.511 2008/12/30 06:24:58 danielk1977 Exp $
 */
 
 /*
 ** This routine is called when a new SQL statement is beginning to
 ** be parsed.  Initialize the pParse structure as needed.
 */
 SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){
   pParse->explain = (u8)explainFlag;
@@ -61437,28 +62118,28 @@ SQLITE_PRIVATE void sqlite3AddColumn(Par
   sqlite3 *db = pParse->db;
   if( (p = pParse->pNewTable)==0 ) return;
 #if SQLITE_MAX_COLUMN
   if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){
     sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName);
     return;
   }
 #endif
-  z = sqlite3NameFromToken(pParse->db, pName);
+  z = sqlite3NameFromToken(db, pName);
   if( z==0 ) return;
   for(i=0; i<p->nCol; i++){
     if( STRICMP(z, p->aCol[i].zName) ){
       sqlite3ErrorMsg(pParse, "duplicate column name: %s", z);
       sqlite3DbFree(db, z);
       return;
     }
   }
   if( (p->nCol & 0x7)==0 ){
     Column *aNew;
-    aNew = sqlite3DbRealloc(pParse->db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0]));
+    aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0]));
     if( aNew==0 ){
       sqlite3DbFree(db, z);
       return;
     }
     p->aCol = aNew;
   }
   pCol = &p->aCol[p->nCol];
   memset(pCol, 0, sizeof(p->aCol[0]));
@@ -63794,16 +64475,36 @@ SQLITE_PRIVATE void sqlite3RollbackTrans
 
   v = sqlite3GetVdbe(pParse);
   if( v ){
     sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1);
   }
 }
 
 /*
+** This function is called by the parser when it parses a command to create,
+** release or rollback an SQL savepoint. 
+*/
+SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){
+  char *zName = sqlite3NameFromToken(pParse->db, pName);
+  if( zName ){
+    Vdbe *v = sqlite3GetVdbe(pParse);
+#ifndef SQLITE_OMIT_AUTHORIZATION
+    static const char *az[] = { "BEGIN", "RELEASE", "ROLLBACK" };
+    assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 );
+#endif
+    if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){
+      sqlite3DbFree(pParse->db, zName);
+      return;
+    }
+    sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC);
+  }
+}
+
+/*
 ** Make sure the TEMP database is open and available for use.  Return
 ** the number of errors.  Leave any error messages in the pParse structure.
 */
 SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){
   sqlite3 *db = pParse->db;
   if( db->aDb[1].pBt==0 && !pParse->explain ){
     int rc;
     static const int flags = 
@@ -64537,17 +65238,17 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains C code routines that are called by the parser
 ** in order to generate code for DELETE FROM statements.
 **
-** $Id: delete.c,v 1.190 2008/12/10 21:19:57 drh Exp $
+** $Id: delete.c,v 1.191 2008/12/23 23:56:22 drh Exp $
 */
 
 /*
 ** Look up every table that is named in pSrc.  If any table is not found,
 ** add an error message to pParse->zErrMsg and return NULL.  If all tables
 ** are found, return a pointer to the last table.
 */
 SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){
@@ -64916,31 +65617,25 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(
 #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */
   /* The usual case: There is a WHERE clause so we have to scan through
   ** the table and pick which records to delete.
   */
   {
     int iRowid = ++pParse->nMem;    /* Used for storing rowid values. */
     int iRowSet = ++pParse->nMem;   /* Register for rowset of rows to delete */
 
-    /* Begin the database scan
-    */
-    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0);
+    /* Collect rowids of every row to be deleted.
+    */
+    sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
+    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0,
+                               WHERE_FILL_ROWSET, iRowSet);
     if( pWInfo==0 ) goto delete_from_cleanup;
-
-    /* Remember the rowid of every item to be deleted.
-    */
-    sqlite3VdbeAddOp2(v, IsVirtual(pTab) ? OP_VRowid : OP_Rowid, iCur, iRowid);
-    sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iRowid);
     if( db->flags & SQLITE_CountRows ){
       sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
     }
-
-    /* End the database scan loop.
-    */
     sqlite3WhereEnd(pWInfo);
 
     /* Open the pseudo-table used to store OLD if there are triggers.
     */
     if( triggers_exist ){
       sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol);
       sqlite3VdbeAddOp1(v, OP_OpenPseudo, oldIdx);
     }
@@ -69447,17 +70142,17 @@ SQLITE_PRIVATE int sqlite3AutoLoadExtens
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains code used to implement the PRAGMA command.
 **
-** $Id: pragma.c,v 1.199 2008/12/10 23:04:13 drh Exp $
+** $Id: pragma.c,v 1.201 2009/01/13 20:14:16 drh Exp $
 */
 
 /* Ignore this whole file if pragmas are disabled
 */
 #if !defined(SQLITE_OMIT_PRAGMA) && !defined(SQLITE_OMIT_PARSER)
 
 /*
 ** Interpret the given string as a safety level.  Return 0 for OFF,
@@ -69630,17 +70325,18 @@ static int flagPragma(Parse *pParse, con
   };
   int i;
   const struct sPragmaType *p;
   for(i=0, p=aPragma; i<ArraySize(aPragma); i++, p++){
     if( sqlite3StrICmp(zLeft, p->zName)==0 ){
       sqlite3 *db = pParse->db;
       Vdbe *v;
       v = sqlite3GetVdbe(pParse);
-      if( v ){
+      assert( v!=0 );  /* Already allocated by sqlite3Pragma() */
+      if( ALWAYS(v) ){
         if( zRight==0 ){
           returnSingleInt(pParse, p->zName, (db->flags & p->mask)!=0 );
         }else{
           if( getBoolean(zRight) ){
             db->flags |= p->mask;
           }else{
             db->flags &= ~p->mask;
           }
@@ -69655,24 +70351,29 @@ static int flagPragma(Parse *pParse, con
 
       return 1;
     }
   }
   return 0;
 }
 #endif /* SQLITE_OMIT_FLAG_PRAGMAS */
 
+/*
+** Return a human-readable name for a constraint resolution action.
+*/
 static const char *actionName(u8 action){
+  const char *zName;
   switch( action ){
-    case OE_SetNull:  return "SET NULL";
-    case OE_SetDflt:  return "SET DEFAULT";
-    case OE_Restrict: return "RESTRICT";
-    case OE_Cascade:  return "CASCADE";
-  }
-  return "";
+    case OE_SetNull:  zName = "SET NULL";            break;
+    case OE_SetDflt:  zName = "SET DEFAULT";         break;
+    case OE_Cascade:  zName = "CASCADE";             break;
+    default:          zName = "RESTRICT";  
+                      assert( action==OE_Restrict ); break;
+  }
+  return zName;
 }
 
 /*
 ** Process a pragma statement.  
 **
 ** Pragmas are of this form:
 **
 **      PRAGMA [database.]id [= value]
@@ -69719,17 +70420,18 @@ SQLITE_PRIVATE void sqlite3Pragma(
   zLeft = sqlite3NameFromToken(db, pId);
   if( !zLeft ) return;
   if( minusFlag ){
     zRight = sqlite3MPrintf(db, "-%T", pValue);
   }else{
     zRight = sqlite3NameFromToken(db, pValue);
   }
 
-  zDb = ((pId2 && pId2->n>0)?pDb->zName:0);
+  assert( pId2 );
+  zDb = pId2->n>0 ? pDb->zName : 0;
   if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){
     goto pragma_out;
   }
  
 #ifndef SQLITE_OMIT_PAGER_PRAGMAS
   /*
   **  PRAGMA [database.]default_cache_size
   **  PRAGMA [database.]default_cache_size=N
@@ -69787,18 +70489,19 @@ SQLITE_PRIVATE void sqlite3Pragma(
   **
   ** The first form reports the current setting for the
   ** database page size in bytes.  The second form sets the
   ** database page size value.  The value can only be set if
   ** the database has not yet been created.
   */
   if( sqlite3StrICmp(zLeft,"page_size")==0 ){
     Btree *pBt = pDb->pBt;
+    assert( pBt!=0 );
     if( !zRight ){
-      int size = pBt ? sqlite3BtreeGetPageSize(pBt) : 0;
+      int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0;
       returnSingleInt(pParse, "page_size", size);
     }else{
       /* Malloc may fail when setting the page-size, as there is an internal
       ** buffer that the pager module resizes using sqlite3_realloc().
       */
       db->nextPagesize = atoi(zRight);
       if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1) ){
         db->mallocFailed = 1;
@@ -69813,33 +70516,34 @@ SQLITE_PRIVATE void sqlite3Pragma(
   ** The first form reports the current setting for the
   ** maximum number of pages in the database file.  The 
   ** second form attempts to change this setting.  Both
   ** forms return the current setting.
   */
   if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){
     Btree *pBt = pDb->pBt;
     int newMax = 0;
+    assert( pBt!=0 );
     if( zRight ){
       newMax = atoi(zRight);
     }
-    if( pBt ){
+    if( ALWAYS(pBt) ){
       newMax = sqlite3BtreeMaxPageCount(pBt, newMax);
     }
     returnSingleInt(pParse, "max_page_count", newMax);
   }else
 
   /*
   **  PRAGMA [database.]page_count
   **
   ** Return the number of pages in the specified database.
   */
   if( sqlite3StrICmp(zLeft,"page_count")==0 ){
     int iReg;
-    if( !v || sqlite3ReadSchema(pParse) ) goto pragma_out;
+    if( sqlite3ReadSchema(pParse) ) goto pragma_out;
     sqlite3CodeVerifySchema(pParse, iDb);
     iReg = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg);
     sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1);
     sqlite3VdbeSetNumCols(v, 1);
     sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC);
   }else
 
@@ -69950,17 +70654,17 @@ SQLITE_PRIVATE void sqlite3Pragma(
            azModeName[eMode], P4_STATIC);
     sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
   }else
 
   /*
   **  PRAGMA [database.]journal_size_limit
   **  PRAGMA [database.]journal_size_limit=N
   **
-  ** Get or set the (boolean) value of the database 'auto-vacuum' parameter.
+  ** Get or set the size limit on rollback journal files.
   */
   if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){
     Pager *pPager = sqlite3BtreePager(pDb->pBt);
     i64 iLimit = -2;
     if( zRight ){
       int iLimit32 = atoi(zRight);
       if( iLimit32<-1 ){
         iLimit32 = -1;
@@ -69972,32 +70676,39 @@ SQLITE_PRIVATE void sqlite3Pragma(
   }else
 
 #endif /* SQLITE_OMIT_PAGER_PRAGMAS */
 
   /*
   **  PRAGMA [database.]auto_vacuum
   **  PRAGMA [database.]auto_vacuum=N
   **
-  ** Get or set the (boolean) value of the database 'auto-vacuum' parameter.
+  ** Get or set the value of the database 'auto-vacuum' parameter.
+  ** The value is one of:  0 NONE 1 FULL 2 INCREMENTAL
   */
 #ifndef SQLITE_OMIT_AUTOVACUUM
   if( sqlite3StrICmp(zLeft,"auto_vacuum")==0 ){
     Btree *pBt = pDb->pBt;
+    assert( pBt!=0 );
     if( sqlite3ReadSchema(pParse) ){
       goto pragma_out;
     }
     if( !zRight ){
-      int auto_vacuum = 
-          pBt ? sqlite3BtreeGetAutoVacuum(pBt) : SQLITE_DEFAULT_AUTOVACUUM;
+      int auto_vacuum;
+      if( ALWAYS(pBt) ){
+         auto_vacuum = sqlite3BtreeGetAutoVacuum(pBt);
+      }else{
+         auto_vacuum = SQLITE_DEFAULT_AUTOVACUUM;
+      }
       returnSingleInt(pParse, "auto_vacuum", auto_vacuum);
     }else{
       int eAuto = getAutoVacuum(zRight);
+      assert( eAuto>=0 && eAuto<=2 );
       db->nextAutovac = (u8)eAuto;
-      if( eAuto>=0 ){
+      if( ALWAYS(eAuto>=0) ){
         /* Call SetAutoVacuum() to set initialize the internal auto and
         ** incr-vacuum flags. This is required in case this connection
         ** creates the database file. It is important that it is created
         ** as an auto-vacuum capable db.
         */
         int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto);
         if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){
           /* When setting the auto_vacuum mode to either "full" or 
@@ -70138,16 +70849,24 @@ SQLITE_PRIVATE void sqlite3Pragma(
         sqlite3_temp_directory = sqlite3DbStrDup(0, zRight);
       }else{
         sqlite3_temp_directory = 0;
       }
 #endif /* SQLITE_OMIT_WSD */
     }
   }else
 
+#if !defined(SQLITE_ENABLE_LOCKING_STYLE)
+#  if defined(__APPLE__)
+#    define SQLITE_ENABLE_LOCKING_STYLE 1
+#  else
+#    define SQLITE_ENABLE_LOCKING_STYLE 0
+#  endif
+#endif
+#if SQLITE_ENABLE_LOCKING_STYLE
   /*
    **   PRAGMA [database.]lock_proxy_file
    **   PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path"
    **
    ** Return or set the value of the lock_proxy_file flag.  Changing
    ** the value sets a specific file to be used for database access locks.
    **
    */
@@ -70178,17 +70897,17 @@ SQLITE_PRIVATE void sqlite3Pragma(
                                      NULL);
       }
       if( res!=SQLITE_OK ){
         sqlite3ErrorMsg(pParse, "failed to set lock proxy file");
         goto pragma_out;
       }
     }
   }else
-      
+#endif /* SQLITE_ENABLE_LOCKING_STYLE */      
     
   /*
   **   PRAGMA [database.]synchronous
   **   PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL
   **
   ** Return or set the local value of the synchronous flag.  Changing
   ** the local value does not make changes to the disk file and the
   ** default value will be restored the next time the database is
@@ -70252,17 +70971,19 @@ SQLITE_PRIVATE void sqlite3Pragma(
           nHidden++;
           continue;
         }
         sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1);
         sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0);
         sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0,
            pCol->zType ? pCol->zType : "", 0);
         sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4);
-        if( pCol->pDflt && (pDflt = &pCol->pDflt->span)->z ){
+        if( pCol->pDflt ){
+          pDflt = &pCol->pDflt->span;
+          assert( pDflt->z );
           sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pDflt->z, pDflt->n);
         }else{
           sqlite3VdbeAddOp2(v, OP_Null, 0, 5);
         }
         sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6);
         sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6);
       }
     }
@@ -70612,38 +71333,36 @@ SQLITE_PRIVATE void sqlite3Pragma(
   ** new database files created using this database handle. It is only
   ** useful if invoked immediately after the main database i
   */
   if( sqlite3StrICmp(zLeft, "encoding")==0 ){
     static const struct EncName {
       char *zName;
       u8 enc;
     } encnames[] = {
-      { "UTF-8",    SQLITE_UTF8        },
       { "UTF8",     SQLITE_UTF8        },
-      { "UTF-16le", SQLITE_UTF16LE     },
+      { "UTF-8",    SQLITE_UTF8        },  /* Must be element [1] */
+      { "UTF-16le", SQLITE_UTF16LE     },  /* Must be element [2] */
+      { "UTF-16be", SQLITE_UTF16BE     },  /* Must be element [3] */
       { "UTF16le",  SQLITE_UTF16LE     },
-      { "UTF-16be", SQLITE_UTF16BE     },
       { "UTF16be",  SQLITE_UTF16BE     },
       { "UTF-16",   0                  }, /* SQLITE_UTF16NATIVE */
       { "UTF16",    0                  }, /* SQLITE_UTF16NATIVE */
       { 0, 0 }
     };
     const struct EncName *pEnc;
     if( !zRight ){    /* "PRAGMA encoding" */
       if( sqlite3ReadSchema(pParse) ) goto pragma_out;
       sqlite3VdbeSetNumCols(v, 1);
       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC);
       sqlite3VdbeAddOp2(v, OP_String8, 0, 1);
-      for(pEnc=&encnames[0]; pEnc->zName; pEnc++){
-        if( pEnc->enc==ENC(pParse->db) ){
-          sqlite3VdbeChangeP4(v, -1, pEnc->zName, P4_STATIC);
-          break;
-        }
-      }
+      assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 );
+      assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE );
+      assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE );
+      sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC);
       sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
     }else{                        /* "PRAGMA encoding = XXX" */
       /* Only change the value of sqlite.enc if the database handle is not
       ** initialized. If the main database exists, the new sqlite.enc value
       ** will be overwritten when the schema is next loaded. If it does not
       ** already exists, it will be created to use the new encoding value.
       */
       if( 
@@ -70778,57 +71497,74 @@ SQLITE_PRIVATE void sqlite3Pragma(
   */
   if( sqlite3StrICmp(zLeft, "create_sqlite_statement_table")==0 ){
     extern int sqlite3CreateStatementsTable(Parse*);
     sqlite3CreateStatementsTable(pParse);
   }else
 #endif
 
 #if SQLITE_HAS_CODEC
-  if( sqlite3StrICmp(zLeft, "key")==0 ){
+  if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){
     sqlite3_key(db, zRight, sqlite3Strlen30(zRight));
   }else
+  if( sqlite3StrICmp(zLeft, "rekey")==0 && zRight ){
+    sqlite3_rekey(db, zRight, sqlite3Strlen30(zRight));
+  }else
+  if( zRight && (sqlite3StrICmp(zLeft, "hexkey")==0 ||
+                 sqlite3StrICmp(zLeft, "hexrekey")==0) ){
+    int i, h1, h2;
+    char zKey[40];
+    for(i=0; (h1 = zRight[i])!=0 && (h2 = zRight[i+1])!=0; i+=2){
+      h1 += 9*(1&(h1>>6));
+      h2 += 9*(1&(h2>>6));
+      zKey[i/2] = (h2 & 0x0f) | ((h1 & 0xf)<<4);
+    }
+    if( (zLeft[3] & 0xf)==0xb ){
+      sqlite3_key(db, zKey, i/2);
+    }else{
+      sqlite3_rekey(db, zKey, i/2);
+    }
+  }else
 #endif
 #if SQLITE_HAS_CODEC || defined(SQLITE_ENABLE_CEROD)
   if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){
 #if SQLITE_HAS_CODEC
     if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){
       extern void sqlite3_activate_see(const char*);
       sqlite3_activate_see(&zRight[4]);
     }
 #endif
 #ifdef SQLITE_ENABLE_CEROD
     if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){
       extern void sqlite3_activate_cerod(const char*);
       sqlite3_activate_cerod(&zRight[6]);
     }
 #endif
-  }
-#endif
-
-  {}
-
-  if( v ){
-    /* Code an OP_Expire at the end of each PRAGMA program to cause
-    ** the VDBE implementing the pragma to expire. Most (all?) pragmas
-    ** are only valid for a single execution.
-    */
-    sqlite3VdbeAddOp2(v, OP_Expire, 1, 0);
-
-    /*
-    ** Reset the safety level, in case the fullfsync flag or synchronous
-    ** setting changed.
-    */
+  }else
+#endif
+
+ 
+  {/* Empty ELSE clause */}
+
+  /* Code an OP_Expire at the end of each PRAGMA program to cause
+  ** the VDBE implementing the pragma to expire. Most (all?) pragmas
+  ** are only valid for a single execution.
+  */
+  sqlite3VdbeAddOp2(v, OP_Expire, 1, 0);
+
+  /*
+  ** Reset the safety level, in case the fullfsync flag or synchronous
+  ** setting changed.
+  */
 #ifndef SQLITE_OMIT_PAGER_PRAGMAS
-    if( db->autoCommit ){
-      sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
-                 (db->flags&SQLITE_FullFSync)!=0);
-    }
-#endif
-  }
+  if( db->autoCommit ){
+    sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
+               (db->flags&SQLITE_FullFSync)!=0);
+  }
+#endif
 pragma_out:
   sqlite3DbFree(db, zLeft);
   sqlite3DbFree(db, zRight);
 }
 
 #endif /* SQLITE_OMIT_PRAGMA || SQLITE_OMIT_PARSER */
 
 /************** End of pragma.c **********************************************/
@@ -70843,17 +71579,17 @@ pragma_out:
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains the implementation of the sqlite3_prepare()
 ** interface, and routines that contribute to loading the database schema
 ** from disk.
 **
-** $Id: prepare.c,v 1.103 2008/12/10 19:26:24 drh Exp $
+** $Id: prepare.c,v 1.104 2009/01/09 02:49:32 drh Exp $
 */
 
 /*
 ** Fill the InitData structure with an error message that indicates
 ** that the database is corrupt.
 */
 static void corruptSchema(
   InitData *pData,     /* Initialization context */
@@ -71329,28 +72065,28 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(
   int i = -1000000;
 
   /* If pSchema is NULL, then return -1000000. This happens when code in 
   ** expr.c is trying to resolve a reference to a transient table (i.e. one
   ** created by a sub-select). In this case the return value of this 
   ** function should never be used.
   **
   ** We return -1000000 instead of the more usual -1 simply because using
-  ** -1000000 as incorrectly using -1000000 index into db->aDb[] is much 
+  ** -1000000 as the incorrect index into db->aDb[] is much 
   ** more likely to cause a segfault than -1 (of course there are assert()
   ** statements too, but it never hurts to play the odds).
   */
   assert( sqlite3_mutex_held(db->mutex) );
   if( pSchema ){
-    for(i=0; i<db->nDb; i++){
+    for(i=0; ALWAYS(i<db->nDb); i++){
       if( db->aDb[i].pSchema==pSchema ){
         break;
       }
     }
-    assert( i>=0 &&i>=0 &&  i<db->nDb );
+    assert( i>=0 && i<db->nDb );
   }
   return i;
 }
 
 /*
 ** Compile the UTF-8 encoded SQL statement zSql into a statement handle.
 */
 static int sqlite3Prepare(
@@ -71654,17 +72390,17 @@ SQLITE_API int sqlite3_prepare16_v2(
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.494 2008/12/10 22:15:00 drh Exp $
+** $Id: select.c,v 1.498 2009/01/09 02:49:32 drh Exp $
 */
 
 
 /*
 ** Delete all the content of a Select structure but do not deallocate
 ** the select structure itself.
 */
 static void clearSelect(sqlite3 *db, Select *p){
@@ -71983,17 +72719,17 @@ static int sqliteProcessJoin(Parse *pPar
   pSrc = p->pSrc;
   pLeft = &pSrc->a[0];
   pRight = &pLeft[1];
   for(i=0; i<pSrc->nSrc-1; i++, pRight++, pLeft++){
     Table *pLeftTab = pLeft->pTab;
     Table *pRightTab = pRight->pTab;
     int isOuter;
 
-    if( pLeftTab==0 || pRightTab==0 ) continue;
+    if( NEVER(pLeftTab==0 || pRightTab==0) ) continue;
     isOuter = (pRight->jointype & JT_OUTER)!=0;
 
     /* When the NATURAL keyword is present, add WHERE clause terms for
     ** every column that the two tables have in common.
     */
     if( pRight->jointype & JT_NATURAL ){
       if( pRight->pOn || pRight->pUsing ){
         sqlite3ErrorMsg(pParse, "a NATURAL join may not have "
@@ -72183,17 +72919,18 @@ static void selectInnerLoop(
   Vdbe *v = pParse->pVdbe;
   int i;
   int hasDistinct;        /* True if the DISTINCT keyword is present */
   int regResult;              /* Start of memory holding result set */
   int eDest = pDest->eDest;   /* How to dispose of results */
   int iParm = pDest->iParm;   /* First argument to disposal method */
   int nResultCol;             /* Number of result columns */
 
-  if( v==0 ) return;
+  assert( v );
+  if( NEVER(v==0) ) return;
   assert( pEList!=0 );
   hasDistinct = distinct>=0;
   if( pOrderBy==0 && !hasDistinct ){
     codeOffset(v, p, iContinue);
   }
 
   /* Pull the requested columns.
   */
@@ -72201,21 +72938,18 @@ static void selectInnerLoop(
     nResultCol = nColumn;
   }else{
     nResultCol = pEList->nExpr;
   }
   if( pDest->iMem==0 ){
     pDest->iMem = pParse->nMem+1;
     pDest->nMem = nResultCol;
     pParse->nMem += nResultCol;
-  }else if( pDest->nMem!=nResultCol ){
-    /* This happens when two SELECTs of a compound SELECT have differing
-    ** numbers of result columns.  The error message will be generated by
-    ** a higher-level routine. */
-    return;
+  }else{ 
+    assert( pDest->nMem==nResultCol );
   }
   regResult = pDest->iMem;
   if( nColumn>0 ){
     for(i=0; i<nColumn; i++){
       sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
     }
   }else if( eDest!=SRT_Exists ){
     /* If the destination is an EXISTS(...) expression, the actual
@@ -72454,16 +73188,18 @@ static void generateSortTail(
   addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak);
   codeOffset(v, p, addrContinue);
   regRow = sqlite3GetTempReg(pParse);
   regRowid = sqlite3GetTempReg(pParse);
   sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow);
   switch( eDest ){
     case SRT_Table:
     case SRT_EphemTab: {
+      testcase( eDest==SRT_Table );
+      testcase( eDest==SRT_EphemTab );
       sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
       sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid);
       sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
       break;
     }
 #ifndef SQLITE_OMIT_SUBQUERY
     case SRT_Set: {
       assert( nColumn==1 );
@@ -72477,16 +73213,18 @@ static void generateSortTail(
       sqlite3ExprCodeMove(pParse, regRow, iParm, 1);
       /* The LIMIT clause will terminate the loop for us */
       break;
     }
 #endif
     case SRT_Output:
     case SRT_Coroutine: {
       int i;
+      testcase( eDest==SRT_Output );
+      testcase( eDest==SRT_Coroutine );
       sqlite3VdbeAddOp2(v, OP_Integer, 1, regRowid);
       sqlite3VdbeAddOp3(v, OP_Insert, pseudoTab, regRow, regRowid);
       for(i=0; i<nColumn; i++){
         assert( regRow!=pDest->iMem+i );
         sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i);
       }
       if( eDest==SRT_Output ){
         sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn);
@@ -72587,29 +73325,29 @@ static const char *columnType(
       }
 
       assert( pTab );
       if( pS ){
         /* The "table" is actually a sub-select or a view in the FROM clause
         ** of the SELECT statement. Return the declaration type and origin
         ** data for the result-set column of the sub-select.
         */
-        if( iCol>=0 && iCol<pS->pEList->nExpr ){
+        if( ALWAYS(iCol>=0 && iCol<pS->pEList->nExpr) ){
           /* If iCol is less than zero, then the expression requests the
           ** rowid of the sub-select or view. This expression is legal (see 
           ** test case misc2.2.2) - it always evaluates to NULL.
           */
           NameContext sNC;
           Expr *p = pS->pEList->a[iCol].pExpr;
           sNC.pSrcList = pS->pSrc;
           sNC.pNext = 0;
           sNC.pParse = pNC->pParse;
           zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); 
         }
-      }else if( pTab->pSchema ){
+      }else if( ALWAYS(pTab->pSchema) ){
         /* A real table */
         assert( !pS );
         if( iCol<0 ) iCol = pTab->iPKey;
         assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
         if( iCol<0 ){
           zType = "INTEGER";
           zOriginCol = "rowid";
         }else{
@@ -72708,52 +73446,50 @@ static void generateColumnNames(
 #ifndef SQLITE_OMIT_EXPLAIN
   /* If this is an EXPLAIN, skip this step */
   if( pParse->explain ){
     return;
   }
 #endif
 
   assert( v!=0 );
-  if( pParse->colNamesSet || v==0 || db->mallocFailed ) return;
+  if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return;
   pParse->colNamesSet = 1;
   fullNames = (db->flags & SQLITE_FullColNames)!=0;
   shortNames = (db->flags & SQLITE_ShortColNames)!=0;
   sqlite3VdbeSetNumCols(v, pEList->nExpr);
   for(i=0; i<pEList->nExpr; i++){
     Expr *p;
     p = pEList->a[i].pExpr;
     if( p==0 ) continue;
     if( pEList->a[i].zName ){
       char *zName = pEList->a[i].zName;
       sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT);
     }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){
       Table *pTab;
       char *zCol;
       int iCol = p->iColumn;
-      for(j=0; j<pTabList->nSrc && pTabList->a[j].iCursor!=p->iTable; j++){}
+      for(j=0; ALWAYS(j<pTabList->nSrc); j++){
+        if( pTabList->a[j].iCursor==p->iTable ) break;
+      }
       assert( j<pTabList->nSrc );
       pTab = pTabList->a[j].pTab;
       if( iCol<0 ) iCol = pTab->iPKey;
       assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
       if( iCol<0 ){
         zCol = "rowid";
       }else{
         zCol = pTab->aCol[iCol].zName;
       }
       if( !shortNames && !fullNames ){
         sqlite3VdbeSetColName(v, i, COLNAME_NAME, 
             sqlite3DbStrNDup(db, (char*)p->span.z, p->span.n), SQLITE_DYNAMIC);
-      }else if( fullNames || (!shortNames && pTabList->nSrc>1) ){
+      }else if( fullNames ){
         char *zName = 0;
-        char *zTab;
- 
-        zTab = pTabList->a[j].zAlias;
-        if( fullNames || zTab==0 ) zTab = pTab->zName;
-        zName = sqlite3MPrintf(db, "%s.%s", zTab, zCol);
+        zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol);
         sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC);
       }else{
         sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT);
       }
     }else{
       sqlite3VdbeSetColName(v, i, COLNAME_NAME, 
           sqlite3DbStrNDup(db, (char*)p->span.z, p->span.n), SQLITE_DYNAMIC);
     }
@@ -73186,20 +73922,24 @@ static int multiSelect(
       int unionTab;    /* Cursor number of the temporary table holding result */
       u8 op = 0;       /* One of the SRT_ operations to apply to self */
       int priorOp;     /* The SRT_ operation to apply to prior selects */
       Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */
       int addr;
       SelectDest uniondest;
 
       priorOp = SRT_Union;
-      if( dest.eDest==priorOp && !p->pLimit && !p->pOffset ){
+      if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){
         /* We can reuse a temporary table generated by a SELECT to our
         ** right.
         */
+        assert( p->pRightmost!=p );  /* Can only happen for leftward elements
+                                     ** of a 3-way or more compound */
+        assert( p->pLimit==0 );      /* Not allowed on leftward elements */
+        assert( p->pOffset==0 );     /* Not allowed on leftward elements */
         unionTab = dest.iParm;
       }else{
         /* We will need to create our own temporary table to hold the
         ** intermediate results.
         */
         unionTab = pParse->nTab++;
         assert( p->pOrderBy==0 );
         addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0);
@@ -74047,16 +74787,18 @@ static void substExpr(
       pExpr->iTable = pNew->iTable;
       pExpr->pTab = pNew->pTab;
       pExpr->iColumn = pNew->iColumn;
       pExpr->iAgg = pNew->iAgg;
       sqlite3TokenCopy(db, &pExpr->token, &pNew->token);
       sqlite3TokenCopy(db, &pExpr->span, &pNew->span);
       pExpr->pSelect = sqlite3SelectDup(db, pNew->pSelect);
       pExpr->flags = pNew->flags;
+      pExpr->pAggInfo = pNew->pAggInfo;
+      pNew->pAggInfo = 0;
     }
   }else{
     substExpr(db, pExpr->pLeft, iTable, pEList);
     substExpr(db, pExpr->pRight, iTable, pEList);
     substSelect(db, pExpr->pSelect, iTable, pEList);
     substExprList(db, pExpr->pList, iTable, pEList);
   }
 }
@@ -74084,17 +74826,18 @@ static void substSelect(
   if( !p ) return;
   substExprList(db, p->pEList, iTable, pEList);
   substExprList(db, p->pGroupBy, iTable, pEList);
   substExprList(db, p->pOrderBy, iTable, pEList);
   substExpr(db, p->pHaving, iTable, pEList);
   substExpr(db, p->pWhere, iTable, pEList);
   substSelect(db, p->pPrior, iTable, pEList);
   pSrc = p->pSrc;
-  if( pSrc ){
+  assert( pSrc );  /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */
+  if( ALWAYS(pSrc) ){
     for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
       substSelect(db, pItem->pSelect, iTable, pEList);
     }
   }
 }
 #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
 
 #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
@@ -74216,17 +74959,16 @@ static int flattenSubquery(
   int i;              /* Loop counter */
   Expr *pWhere;                    /* The WHERE clause */
   struct SrcList_item *pSubitem;   /* The subquery */
   sqlite3 *db = pParse->db;
 
   /* Check to see if flattening is permitted.  Return 0 if not.
   */
   assert( p!=0 );
-  if( p==0 ) return 0;
   assert( p->pPrior==0 );  /* Unable to flatten compound queries */
   pSrc = p->pSrc;
   assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc );
   pSubitem = &pSrc->a[iFrom];
   iParent = pSubitem->iCursor;
   pSub = pSubitem->pSelect;
   assert( pSub!=0 );
   if( isAgg && subqueryIsAgg ) return 0;                 /* Restriction (1)  */
@@ -74294,17 +75036,17 @@ static int flattenSubquery(
   }
 
   /* Restriction 17: If the sub-query is a compound SELECT, then it must
   ** use only the UNION ALL operator. And none of the simple select queries
   ** that make up the compound SELECT are allowed to be aggregate or distinct
   ** queries.
   */
   if( pSub->pPrior ){
-    if( p->pPrior || isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
+    if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
       return 0;
     }
     for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){
       if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0
        || (pSub1->pPrior && pSub1->op!=TK_ALL) 
        || !pSub1->pSrc || pSub1->pSrc->nSrc!=1
       ){
         return 0;
@@ -75346,17 +76088,17 @@ SQLITE_PRIVATE int sqlite3Select(
     distinct = -1;
   }
 
   /* Aggregate and non-aggregate queries are handled differently */
   if( !isAgg && pGroupBy==0 ){
     /* This case is for non-aggregate queries
     ** Begin the database scan
     */
-    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
+    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0, 0);
     if( pWInfo==0 ) goto select_end;
 
     /* If sorting index that was created by a prior OP_OpenEphemeral 
     ** instruction ended up not being needed, then change the OP_OpenEphemeral
     ** into an OP_Noop.
     */
     if( addrSortIndex>=0 && pOrderBy==0 ){
       sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
@@ -75467,17 +76209,17 @@ SQLITE_PRIVATE int sqlite3Select(
       VdbeComment((v, "indicate accumulator empty"));
 
       /* Begin a loop that will extract all source rows in GROUP BY order.
       ** This might involve two separate loops with an OP_Sort in between, or
       ** it might be a single loop that uses an index to extract information
       ** in the right order to begin with.
       */
       sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
-      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0);
+      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0);
       if( pWInfo==0 ) goto select_end;
       if( pGroupBy==0 ){
         /* The optimizer is able to deliver rows in group by order so
         ** we do not have to sort.  The OP_OpenEphemeral table will be
         ** cancelled later because we still need to use the pKeyInfo
         */
         pGroupBy = p->pGroupBy;
         groupBySort = 0;
@@ -75665,17 +76407,17 @@ SQLITE_PRIVATE int sqlite3Select(
         }
       }
 
       /* This case runs if the aggregate has no GROUP BY clause.  The
       ** processing is much simpler since there is only a single row
       ** of output.
       */
       resetAccumulator(pParse, &sAggInfo);
-      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag);
+      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag, 0);
       if( pWInfo==0 ){
         sqlite3ExprListDelete(db, pDel);
         goto select_end;
       }
       updateAccumulator(pParse, &sAggInfo);
       if( !pMinMax && flag ){
         sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak);
         VdbeComment((v, "%s() by index",(flag==WHERE_ORDERBY_MIN?"min":"max")));
@@ -76033,17 +76775,17 @@ SQLITE_API void sqlite3_free_table(
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 **
 **
-** $Id: trigger.c,v 1.132 2008/12/10 19:26:24 drh Exp $
+** $Id: trigger.c,v 1.133 2008/12/26 07:56:39 danielk1977 Exp $
 */
 
 #ifndef SQLITE_OMIT_TRIGGER
 /*
 ** Delete a linked list of TriggerStep structures.
 */
 SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){
   while( pTriggerStep ){
@@ -76694,16 +77436,17 @@ static int codeTriggerProgram(
   Vdbe *v = pParse->pVdbe;
   sqlite3 *db = pParse->db;
 
   assert( pTriggerStep!=0 );
   assert( v!=0 );
   sqlite3VdbeAddOp2(v, OP_ContextPush, 0, 0);
   VdbeComment((v, "begin trigger %s", pStepList->pTrig->name));
   while( pTriggerStep ){
+    sqlite3ExprClearColumnCache(pParse, -1);
     orconf = (orconfin == OE_Default)?pTriggerStep->orconf:orconfin;
     pParse->trigStack->orconf = orconf;
     switch( pTriggerStep->op ){
       case TK_SELECT: {
         Select *ss = sqlite3SelectDup(db, pTriggerStep->pSelect);
         if( ss ){
           SelectDest dest;
 
@@ -76891,17 +77634,17 @@ SQLITE_PRIVATE int sqlite3CodeRowTrigger
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** This file contains C code routines that are called by the parser
 ** to handle UPDATE statements.
 **
-** $Id: update.c,v 1.190 2008/12/10 22:15:00 drh Exp $
+** $Id: update.c,v 1.191 2008/12/23 23:56:22 drh Exp $
 */
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /* Forward declaration */
 static void updateVirtualTable(
   Parse *pParse,       /* The parsing context */
   SrcList *pSrc,       /* The virtual table to be modified */
   Table *pTab,         /* The virtual table */
@@ -77224,17 +77967,17 @@ SQLITE_PRIVATE void sqlite3Update(
   if( sqlite3ResolveExprNames(&sNC, pWhere) ){
     goto update_cleanup;
   }
 
   /* Begin the database scan
   */
   sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
   pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0,
-                             WHERE_ONEPASS_DESIRED);
+                             WHERE_ONEPASS_DESIRED, 0);
   if( pWInfo==0 ) goto update_cleanup;
   okOnePass = pWInfo->okOnePass;
 
   /* Remember the rowid of every item to be updated.
   */
   sqlite3VdbeAddOp2(v, IsVirtual(pTab)?OP_VRowid:OP_Rowid, iCur, regOldRowid);
   if( !okOnePass ){
     regRowSet = ++pParse->nMem;
@@ -78728,17 +79471,17 @@ SQLITE_PRIVATE void sqlite3VtabMakeWrita
 *************************************************************************
 ** This module contains C code that generates VDBE code used to process
 ** the WHERE clause of SQL statements.  This module is responsible for
 ** generating the code that loops through a table looking for applicable
 ** rows.  Indices are selected and used to speed the search when doing
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.337 2008/12/12 17:56:16 drh Exp $
+** $Id: where.c,v 1.364 2009/01/14 00:55:10 drh Exp $
 */
 
 /*
 ** Trace output macros
 */
 #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG)
 SQLITE_PRIVATE int sqlite3WhereTrace = 0;
 #endif
@@ -78746,17 +79489,20 @@ SQLITE_PRIVATE int sqlite3WhereTrace = 0
 # define WHERETRACE(X)  if(sqlite3WhereTrace) sqlite3DebugPrintf X
 #else
 # define WHERETRACE(X)
 #endif
 
 /* Forward reference
 */
 typedef struct WhereClause WhereClause;
-typedef struct ExprMaskSet ExprMaskSet;
+typedef struct WhereMaskSet WhereMaskSet;
+typedef struct WhereOrInfo WhereOrInfo;
+typedef struct WhereAndInfo WhereAndInfo;
+typedef struct WhereCost WhereCost;
 
 /*
 ** The query generator uses an array of instances of this structure to
 ** help it analyze the subexpressions of the WHERE clause.  Each WHERE
 ** clause subexpression is separated from the others by AND operators.
 ** (Note: the same data structure is also reused to hold a group of terms
 ** separated by OR operators.  But at the top-level, everything is AND
 ** separated.)
@@ -78766,176 +79512,251 @@ typedef struct ExprMaskSet ExprMaskSet;
 **
 **        WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm
 **
 ** When a term is of the form:
 **
 **              X <op> <expr>
 **
 ** where X is a column name and <op> is one of certain operators,
-** then WhereTerm.leftCursor and WhereTerm.leftColumn record the
-** cursor number and column number for X.  WhereTerm.operator records
+** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the
+** cursor number and column number for X.  WhereTerm.eOperator records
 ** the <op> using a bitmask encoding defined by WO_xxx below.  The
 ** use of a bitmask encoding for the operator allows us to search
 ** quickly for terms that match any of several different operators.
 **
-** prereqRight and prereqAll record sets of cursor numbers,
-** but they do so indirectly.  A single ExprMaskSet structure translates
+** A WhereTerm might also be two or more subterms connected by OR:
+**
+**         (t1.X <op> <expr>) OR (t1.Y <op> <expr>) OR ....
+**
+** In this second case, wtFlag as the TERM_ORINFO set and eOperator==WO_OR
+** and the WhereTerm.u.pOrInfo field points to auxiliary information that
+** is collected about the
+**
+** If a term in the WHERE clause does not match either of the two previous
+** categories, then eOperator==0.  The WhereTerm.pExpr field is still set
+** to the original subexpression content and wtFlags is set up appropriately
+** but no other fields in the WhereTerm object are meaningful.
+**
+** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers,
+** but they do so indirectly.  A single WhereMaskSet structure translates
 ** cursor number into bits and the translated bit is stored in the prereq
 ** fields.  The translation is used in order to maximize the number of
 ** bits that will fit in a Bitmask.  The VDBE cursor numbers might be
 ** spread out over the non-negative integers.  For example, the cursor
-** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45.  The ExprMaskSet
+** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45.  The WhereMaskSet
 ** translates these sparse cursor numbers into consecutive integers
 ** beginning with 0 in order to make the best possible use of the available
 ** bits in the Bitmask.  So, in the example above, the cursor numbers
 ** would be mapped into integers 0 through 7.
 **
 ** The number of terms in a join is limited by the number of bits
 ** in prereqRight and prereqAll.  The default is 64 bits, hence SQLite
 ** is only able to process joins with 64 or fewer tables.
 */
 typedef struct WhereTerm WhereTerm;
 struct WhereTerm {
   Expr *pExpr;            /* Pointer to the subexpression that is this term */
   int iParent;            /* Disable pWC->a[iParent] when this term disabled */
   int leftCursor;         /* Cursor number of X in "X <op> <expr>" */
-  int leftColumn;         /* Column number of X in "X <op> <expr>" */
+  union {
+    int leftColumn;         /* Column number of X in "X <op> <expr>" */
+    WhereOrInfo *pOrInfo;   /* Extra information if eOperator==WO_OR */
+    WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */
+  } u;
   u16 eOperator;          /* A WO_xx value describing <op> */
   u8 wtFlags;             /* TERM_xxx bit flags.  See below */
   u8 nChild;              /* Number of children that must disable us */
   WhereClause *pWC;       /* The clause this term is part of */
   Bitmask prereqRight;    /* Bitmask of tables used by pExpr->pRight */
   Bitmask prereqAll;      /* Bitmask of tables referenced by pExpr */
 };
 
 /*
 ** Allowed values of WhereTerm.wtFlags
 */
 #define TERM_DYNAMIC    0x01   /* Need to call sqlite3ExprDelete(db, pExpr) */
 #define TERM_VIRTUAL    0x02   /* Added by the optimizer.  Do not code */
 #define TERM_CODED      0x04   /* This term is already coded */
 #define TERM_COPIED     0x08   /* Has a child */
-#define TERM_OR_OK      0x10   /* Used during OR-clause processing */
+#define TERM_ORINFO     0x10   /* Need to free the WhereTerm.u.pOrInfo object */
+#define TERM_ANDINFO    0x20   /* Need to free the WhereTerm.u.pAndInfo obj */
+#define TERM_OR_OK      0x40   /* Used during OR-clause processing */
 
 /*
 ** An instance of the following structure holds all information about a
 ** WHERE clause.  Mostly this is a container for one or more WhereTerms.
 */
 struct WhereClause {
   Parse *pParse;           /* The parser context */
-  ExprMaskSet *pMaskSet;   /* Mapping of table indices to bitmasks */
+  WhereMaskSet *pMaskSet;  /* Mapping of table cursor numbers to bitmasks */
+  u8 op;                   /* Split operator.  TK_AND or TK_OR */
   int nTerm;               /* Number of terms */
   int nSlot;               /* Number of entries in a[] */
   WhereTerm *a;            /* Each a[] describes a term of the WHERE cluase */
   WhereTerm aStatic[4];    /* Initial static space for a[] */
 };
 
 /*
+** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to
+** a dynamically allocated instance of the following structure.
+*/
+struct WhereOrInfo {
+  WhereClause wc;          /* Decomposition into subterms */
+  Bitmask indexable;       /* Bitmask of all indexable tables in the clause */
+};
+
+/*
+** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to
+** a dynamically allocated instance of the following structure.
+*/
+struct WhereAndInfo {
+  WhereClause wc;          /* The subexpression broken out */
+};
+
+/*
 ** An instance of the following structure keeps track of a mapping
 ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm.
 **
 ** The VDBE cursor numbers are small integers contained in 
 ** SrcList_item.iCursor and Expr.iTable fields.  For any given WHERE 
 ** clause, the cursor numbers might not begin with 0 and they might
 ** contain gaps in the numbering sequence.  But we want to make maximum
 ** use of the bits in our bitmasks.  This structure provides a mapping
 ** from the sparse cursor numbers into consecutive integers beginning
 ** with 0.
 **
-** If ExprMaskSet.ix[A]==B it means that The A-th bit of a Bitmask
+** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask
 ** corresponds VDBE cursor number B.  The A-th bit of a bitmask is 1<<A.
 **
 ** For example, if the WHERE clause expression used these VDBE
-** cursors:  4, 5, 8, 29, 57, 73.  Then the  ExprMaskSet structure
+** cursors:  4, 5, 8, 29, 57, 73.  Then the  WhereMaskSet structure
 ** would map those cursor numbers into bits 0 through 5.
 **
 ** Note that the mapping is not necessarily ordered.  In the example
 ** above, the mapping might go like this:  4->3, 5->1, 8->2, 29->0,
 ** 57->5, 73->4.  Or one of 719 other combinations might be used. It
 ** does not really matter.  What is important is that sparse cursor
 ** numbers all get mapped into bit numbers that begin with 0 and contain
 ** no gaps.
 */
-struct ExprMaskSet {
+struct WhereMaskSet {
   int n;                        /* Number of assigned cursor values */
   int ix[BMS];                  /* Cursor assigned to each bit */
 };
 
+/*
+** A WhereCost object records a lookup strategy and the estimated
+** cost of pursuing that strategy.
+*/
+struct WhereCost {
+  WherePlan plan;    /* The lookup strategy */
+  double rCost;      /* Overall cost of pursuing this search strategy */
+  double nRow;       /* Estimated number of output rows */
+};
 
 /*
 ** Bitmasks for the operators that indices are able to exploit.  An
 ** OR-ed combination of these values can be used when searching for
 ** terms in the where clause.
 */
 #define WO_IN     0x001
 #define WO_EQ     0x002
 #define WO_LT     (WO_EQ<<(TK_LT-TK_EQ))
 #define WO_LE     (WO_EQ<<(TK_LE-TK_EQ))
 #define WO_GT     (WO_EQ<<(TK_GT-TK_EQ))
 #define WO_GE     (WO_EQ<<(TK_GE-TK_EQ))
 #define WO_MATCH  0x040
 #define WO_ISNULL 0x080
-#define WO_OR     0x100
+#define WO_OR     0x100       /* Two or more OR-connected terms */
+#define WO_AND    0x200       /* Two or more AND-connected terms */
 
 #define WO_ALL    0xfff       /* Mask of all possible WO_* values */
-
-/*
-** Value for wsFlags returned by bestIndex().  These flags determine which
-** search strategies are appropriate.
+#define WO_SINGLE 0x0ff       /* Mask of all non-compound WO_* values */
+
+/*
+** Value for wsFlags returned by bestIndex() and stored in
+** WhereLevel.wsFlags.  These flags determine which search
+** strategies are appropriate.
 **
 ** The least significant 12 bits is reserved as a mask for WO_ values above.
-** The WhereLevel.wtFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL.
-** But if the table is the right table of a left join, WhereLevel.wtFlags
-** is set to WO_IN|WO_EQ.  The WhereLevel.wtFlags field can then be used as
+** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL.
+** But if the table is the right table of a left join, WhereLevel.wsFlags
+** is set to WO_IN|WO_EQ.  The WhereLevel.wsFlags field can then be used as
 ** the "op" parameter to findTerm when we are resolving equality constraints.
 ** ISNULL constraints will then not be used on the right table of a left
 ** join.  Tickets #2177 and #2189.
 */
 #define WHERE_ROWID_EQ     0x00001000  /* rowid=EXPR or rowid IN (...) */
 #define WHERE_ROWID_RANGE  0x00002000  /* rowid<EXPR and/or rowid>EXPR */
 #define WHERE_COLUMN_EQ    0x00010000  /* x=EXPR or x IN (...) */
 #define WHERE_COLUMN_RANGE 0x00020000  /* x<EXPR and/or x>EXPR */
 #define WHERE_COLUMN_IN    0x00040000  /* x IN (...) */
+#define WHERE_INDEXED      0x00070000  /* Anything that uses an index */
+#define WHERE_IN_ABLE      0x00071000  /* Able to support an IN operator */
 #define WHERE_TOP_LIMIT    0x00100000  /* x<EXPR or x<=EXPR constraint */
 #define WHERE_BTM_LIMIT    0x00200000  /* x>EXPR or x>=EXPR constraint */
 #define WHERE_IDX_ONLY     0x00800000  /* Use index only - omit table */
 #define WHERE_ORDERBY      0x01000000  /* Output will appear in correct order */
 #define WHERE_REVERSE      0x02000000  /* Scan in reverse order */
 #define WHERE_UNIQUE       0x04000000  /* Selects no more than one row */
 #define WHERE_VIRTUALTABLE 0x08000000  /* Use virtual-table processing */
 #define WHERE_MULTI_OR     0x10000000  /* OR using multiple indices */
 
 /*
 ** Initialize a preallocated WhereClause structure.
 */
 static void whereClauseInit(
   WhereClause *pWC,        /* The WhereClause to be initialized */
   Parse *pParse,           /* The parsing context */
-  ExprMaskSet *pMaskSet    /* Mapping from table indices to bitmasks */
+  WhereMaskSet *pMaskSet   /* Mapping from table cursor numbers to bitmasks */
 ){
   pWC->pParse = pParse;
   pWC->pMaskSet = pMaskSet;
   pWC->nTerm = 0;
   pWC->nSlot = ArraySize(pWC->aStatic);
   pWC->a = pWC->aStatic;
 }
 
+/* Forward reference */
+static void whereClauseClear(WhereClause*);
+
+/*
+** Deallocate all memory associated with a WhereOrInfo object.
+*/
+static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){
+  whereClauseClear(&p->wc);
+  sqlite3DbFree(db, p);
+}
+
+/*
+** Deallocate all memory associated with a WhereAndInfo object.
+*/
+static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){
+  whereClauseClear(&p->wc);
+  sqlite3DbFree(db, p);
+}
+
 /*
 ** Deallocate a WhereClause structure.  The WhereClause structure
 ** itself is not freed.  This routine is the inverse of whereClauseInit().
 */
 static void whereClauseClear(WhereClause *pWC){
   int i;
   WhereTerm *a;
   sqlite3 *db = pWC->pParse->db;
   for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){
     if( a->wtFlags & TERM_DYNAMIC ){
       sqlite3ExprDelete(db, a->pExpr);
     }
+    if( a->wtFlags & TERM_ORINFO ){
+      whereOrInfoDelete(db, a->u.pOrInfo);
+    }else if( a->wtFlags & TERM_ANDINFO ){
+      whereAndInfoDelete(db, a->u.pAndInfo);
+    }
   }
   if( pWC->a!=pWC->aStatic ){
     sqlite3DbFree(db, pWC->a);
   }
 }
 
 /*
 ** Add a single new WhereTerm entry to the WhereClause object pWC.
@@ -78997,16 +79818,17 @@ static int whereClauseInsert(WhereClause
 ** The original WHERE clause in pExpr is unaltered.  All this routine
 ** does is make slot[] entries point to substructure within pExpr.
 **
 ** In the previous sentence and in the diagram, "slot[]" refers to
 ** the WhereClause.a[] array.  The slot[] array grows as needed to contain
 ** all terms of the WHERE clause.
 */
 static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){
+  pWC->op = (u8)op;
   if( pExpr==0 ) return;
   if( pExpr->op!=op ){
     whereClauseInsert(pWC, pExpr, 0);
   }else{
     whereSplit(pWC, pExpr->pLeft, op);
     whereSplit(pWC, pExpr->pRight, op);
   }
 }
@@ -79015,17 +79837,17 @@ static void whereSplit(WhereClause *pWC,
 ** Initialize an expression mask set
 */
 #define initMaskSet(P)  memset(P, 0, sizeof(*P))
 
 /*
 ** Return the bitmask for the given cursor number.  Return 0 if
 ** iCursor is not in the set.
 */
-static Bitmask getMask(ExprMaskSet *pMaskSet, int iCursor){
+static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){
   int i;
   for(i=0; i<pMaskSet->n; i++){
     if( pMaskSet->ix[i]==iCursor ){
       return ((Bitmask)1)<<i;
     }
   }
   return 0;
 }
@@ -79033,17 +79855,17 @@ static Bitmask getMask(ExprMaskSet *pMas
 /*
 ** Create a new mask for cursor iCursor.
 **
 ** There is one cursor per table in the FROM clause.  The number of
 ** tables in the FROM clause is limited by a test early in the
 ** sqlite3WhereBegin() routine.  So we know that the pMaskSet->ix[]
 ** array will never overflow.
 */
-static void createMask(ExprMaskSet *pMaskSet, int iCursor){
+static void createMask(WhereMaskSet *pMaskSet, int iCursor){
   assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
   pMaskSet->ix[pMaskSet->n++] = iCursor;
 }
 
 /*
 ** This routine walks (recursively) an expression tree and generates
 ** a bitmask indicating which tables are used in that expression
 ** tree.
@@ -79052,42 +79874,42 @@ static void createMask(ExprMaskSet *pMas
 ** previously invoked sqlite3ResolveExprNames() on the expression.  See
 ** the header comment on that routine for additional information.
 ** The sqlite3ResolveExprNames() routines looks for column names and
 ** sets their opcodes to TK_COLUMN and their Expr.iTable fields to
 ** the VDBE cursor number of the table.  This routine just has to
 ** translate the cursor numbers into bitmask values and OR all
 ** the bitmasks together.
 */
-static Bitmask exprListTableUsage(ExprMaskSet*, ExprList*);
-static Bitmask exprSelectTableUsage(ExprMaskSet*, Select*);
-static Bitmask exprTableUsage(ExprMaskSet *pMaskSet, Expr *p){
+static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*);
+static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*);
+static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){
   Bitmask mask = 0;
   if( p==0 ) return 0;
   if( p->op==TK_COLUMN ){
     mask = getMask(pMaskSet, p->iTable);
     return mask;
   }
   mask = exprTableUsage(pMaskSet, p->pRight);
   mask |= exprTableUsage(pMaskSet, p->pLeft);
   mask |= exprListTableUsage(pMaskSet, p->pList);
   mask |= exprSelectTableUsage(pMaskSet, p->pSelect);
   return mask;
 }
-static Bitmask exprListTableUsage(ExprMaskSet *pMaskSet, ExprList *pList){
+static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){
   int i;
   Bitmask mask = 0;
   if( pList ){
     for(i=0; i<pList->nExpr; i++){
       mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr);
     }
   }
   return mask;
 }
-static Bitmask exprSelectTableUsage(ExprMaskSet *pMaskSet, Select *pS){
+static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){
   Bitmask mask = 0;
   while( pS ){
     mask |= exprListTableUsage(pMaskSet, pS->pEList);
     mask |= exprListTableUsage(pMaskSet, pS->pGroupBy);
     mask |= exprListTableUsage(pMaskSet, pS->pOrderBy);
     mask |= exprTableUsage(pMaskSet, pS->pWhere);
     mask |= exprTableUsage(pMaskSet, pS->pHaving);
     pS = pS->pPrior;
@@ -79150,24 +79972,21 @@ static void exprCommute(Parse *pParse, E
 */
 static u16 operatorMask(int op){
   u16 c;
   assert( allowedOp(op) );
   if( op==TK_IN ){
     c = WO_IN;
   }else if( op==TK_ISNULL ){
     c = WO_ISNULL;
-  }else if( op==TK_OR ){
-    c = WO_OR;
   }else{
     assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff );
     c = (u16)(WO_EQ<<(op-TK_EQ));
   }
   assert( op!=TK_ISNULL || c==WO_ISNULL );
-  assert( op!=TK_OR || c==WO_OR );
   assert( op!=TK_IN || c==WO_IN );
   assert( op!=TK_EQ || c==WO_EQ );
   assert( op!=TK_LT || c==WO_LT );
   assert( op!=TK_LE || c==WO_LE );
   assert( op!=TK_GT || c==WO_GT );
   assert( op!=TK_GE || c==WO_GE );
   return c;
 }
@@ -79188,17 +80007,17 @@ static WhereTerm *findTerm(
 ){
   WhereTerm *pTerm;
   int k;
   assert( iCur>=0 );
   op &= WO_ALL;
   for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){
     if( pTerm->leftCursor==iCur
        && (pTerm->prereqRight & notReady)==0
-       && pTerm->leftColumn==iColumn
+       && pTerm->u.leftColumn==iColumn
        && (pTerm->eOperator & op)!=0
     ){
       if( pIdx && pTerm->eOperator!=WO_ISNULL ){
         Expr *pX = pTerm->pExpr;
         CollSeq *pColl;
         char idxaff;
         int j;
         Parse *pParse = pWC->pParse;
@@ -79207,24 +80026,22 @@ static WhereTerm *findTerm(
         if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue;
 
         /* Figure out the collation sequence required from an index for
         ** it to be useful for optimising expression pX. Store this
         ** value in variable pColl.
         */
         assert(pX->pLeft);
         pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-        if( !pColl ){
-          pColl = pParse->db->pDfltColl;
-        }
+        assert(pColl || pParse->nErr);
 
         for(j=0; pIdx->aiColumn[j]!=iColumn; j++){
           if( NEVER(j>=pIdx->nColumn) ) return 0;
         }
-        if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue;
+        if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue;
       }
       return pTerm;
     }
   }
   return 0;
 }
 
 /* Forward reference */
@@ -79256,34 +80073,34 @@ static void exprAnalyzeAll(
 */
 static int isLikeOrGlob(
   Parse *pParse,    /* Parsing and code generating context */
   Expr *pExpr,      /* Test this expression */
   int *pnPattern,   /* Number of non-wildcard prefix characters */
   int *pisComplete, /* True if the only wildcard is % in the last character */
   int *pnoCase      /* True if uppercase is equivalent to lowercase */
 ){
-  const char *z;
-  Expr *pRight, *pLeft;
-  ExprList *pList;
-  int c, cnt;
-  char wc[3];
-  CollSeq *pColl;
-  sqlite3 *db = pParse->db;
+  const char *z;             /* String on RHS of LIKE operator */
+  Expr *pRight, *pLeft;      /* Right and left size of LIKE operator */
+  ExprList *pList;           /* List of operands to the LIKE operator */
+  int c;                     /* One character in z[] */
+  int cnt;                   /* Number of non-wildcard prefix characters */
+  char wc[3];                /* Wildcard characters */
+  CollSeq *pColl;            /* Collating sequence for LHS */
+  sqlite3 *db = pParse->db;  /* Database connection */
 
   if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){
     return 0;
   }
 #ifdef SQLITE_EBCDIC
   if( *pnoCase ) return 0;
 #endif
   pList = pExpr->pList;
   pRight = pList->a[0].pExpr;
-  if( pRight->op!=TK_STRING
-   && (pRight->op!=TK_REGISTER || pRight->iColumn!=TK_STRING) ){
+  if( pRight->op!=TK_STRING ){
     return 0;
   }
   pLeft = pList->a[1].pExpr;
   if( pLeft->op!=TK_COLUMN ){
     return 0;
   }
   pColl = sqlite3ExprCollSeq(pParse, pLeft);
   assert( pColl!=0 || pLeft->iColumn==-1 );
@@ -79296,17 +80113,17 @@ static int isLikeOrGlob(
     return 0;
   }
   sqlite3DequoteExpr(db, pRight);
   z = (char *)pRight->token.z;
   cnt = 0;
   if( z ){
     while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; }
   }
-  if( cnt==0 || 255==(u8)z[cnt] ){
+  if( cnt==0 || 255==(u8)z[cnt-1] ){
     return 0;
   }
   *pisComplete = z[cnt]==wc[0] && z[cnt+1]==0;
   *pnPattern = cnt;
   return 1;
 }
 #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */
 
@@ -79348,130 +80165,320 @@ static int isMatchOfColumn(
 */
 static void transferJoinMarkings(Expr *pDerived, Expr *pBase){
   pDerived->flags |= pBase->flags & EP_FromJoin;
   pDerived->iRightJoinTable = pBase->iRightJoinTable;
 }
 
 #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY)
 /*
-** Return TRUE if the given term of an OR clause can be converted
-** into an IN clause.  The iCursor and iColumn define the left-hand
-** side of the IN clause.
-**
-** The context is that we have multiple OR-connected equality terms
-** like this:
-**
-**           a=<expr1> OR  a=<expr2> OR b=<expr3>  OR ...
-**
-** The pOrTerm input to this routine corresponds to a single term of
-** this OR clause.  In order for the term to be a candidate for
-** conversion to an IN operator, the following must be true:
-**
-**     *  The left-hand side of the term must be the column which
-**        is identified by iCursor and iColumn.
-**
-**     *  If the right-hand side is also a column, then the affinities
-**        of both right and left sides must be such that no type
-**        conversions are required on the right.  (Ticket #2249)
-**
-** If both of these conditions are true, then return true.  Otherwise
-** return false.
-*/
-static int orTermIsOptCandidate(WhereTerm *pOrTerm, int iCursor, int iColumn){
-  int affLeft, affRight;
-  assert( pOrTerm->eOperator==WO_EQ );
-  if( pOrTerm->leftCursor!=iCursor ){
-    return 0;
-  }
-  if( pOrTerm->leftColumn!=iColumn ){
-    return 0;
-  }
-  affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight);
-  if( affRight==0 ){
-    return 1;
-  }
-  affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft);
-  if( affRight!=affLeft ){
-    return 0;
-  }
-  return 1;
-}
-
-/*
-** Return true if the given term of an OR clause can be ignored during
-** a check to make sure all OR terms are candidates for optimization.
-** In other words, return true if a call to the orTermIsOptCandidate()
-** above returned false but it is not necessary to disqualify the
-** optimization.
-**
-** Suppose the original OR phrase was this:
-**
-**           a=4  OR  a=11  OR  a=b
-**
-** During analysis, the third term gets flipped around and duplicate
-** so that we are left with this:
-**
-**           a=4  OR  a=11  OR  a=b  OR  b=a
-**
-** Since the last two terms are duplicates, only one of them
-** has to qualify in order for the whole phrase to qualify.  When
-** this routine is called, we know that pOrTerm did not qualify.
-** This routine merely checks to see if pOrTerm has a duplicate that
-** might qualify.  If there is a duplicate that has not yet been
-** disqualified, then return true.  If there are no duplicates, or
-** the duplicate has also been disqualified, return false.
-*/
-static int orTermHasOkDuplicate(WhereClause *pOr, WhereTerm *pOrTerm){
-  if( pOrTerm->wtFlags & TERM_COPIED ){
-    /* This is the original term.  The duplicate is to the left had
-    ** has not yet been analyzed and thus has not yet been disqualified. */
-    return 1;
-  }
-  if( (pOrTerm->wtFlags & TERM_VIRTUAL)!=0
-     && (pOr->a[pOrTerm->iParent].wtFlags & TERM_OR_OK)!=0 ){
-    /* This is a duplicate term.  The original qualified so this one
-    ** does not have to. */
-    return 1;
-  }
-  /* This is either a singleton term or else it is a duplicate for
-  ** which the original did not qualify.  Either way we are done for. */
-  return 0;
+** Analyze a term that consists of two or more OR-connected
+** subterms.  So in:
+**
+**     ... WHERE  (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13)
+**                          ^^^^^^^^^^^^^^^^^^^^
+**
+** This routine analyzes terms such as the middle term in the above example.
+** A WhereOrTerm object is computed and attached to the term under
+** analysis, regardless of the outcome of the analysis.  Hence:
+**
+**     WhereTerm.wtFlags   |=  TERM_ORINFO
+**     WhereTerm.u.pOrInfo  =  a dynamically allocated WhereOrTerm object
+**
+** The term being analyzed must have two or more of OR-connected subterms.
+** A single subterm might be a set of AND-connected sub-subterms.
+** Examples of terms under analysis:
+**
+**     (A)     t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5
+**     (B)     x=expr1 OR expr2=x OR x=expr3
+**     (C)     t1.x=t2.y OR (t1.x=t2.z AND t1.y=15)
+**     (D)     x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*')
+**     (E)     (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6)
+**
+** CASE 1:
+**
+** If all subterms are of the form T.C=expr for some single column of C
+** a single table T (as shown in example B above) then create a new virtual
+** term that is an equivalent IN expression.  In other words, if the term
+** being analyzed is:
+**
+**      x = expr1  OR  expr2 = x  OR  x = expr3
+**
+** then create a new virtual term like this:
+**
+**      x IN (expr1,expr2,expr3)
+**
+** CASE 2:
+**
+** If all subterms are indexable by a single table T, then set
+**
+**     WhereTerm.eOperator              =  WO_OR
+**     WhereTerm.u.pOrInfo->indexable  |=  the cursor number for table T
+**
+** A subterm is "indexable" if it is of the form
+** "T.C <op> <expr>" where C is any column of table T and 
+** <op> is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN".
+** A subterm is also indexable if it is an AND of two or more
+** subsubterms at least one of which is indexable.  Indexable AND 
+** subterms have their eOperator set to WO_AND and they have
+** u.pAndInfo set to a dynamically allocated WhereAndTerm object.
+**
+** From another point of view, "indexable" means that the subterm could
+** potentially be used with an index if an appropriate index exists.
+** This analysis does not consider whether or not the index exists; that
+** is something the bestIndex() routine will determine.  This analysis
+** only looks at whether subterms appropriate for indexing exist.
+**
+** All examples A through E above all satisfy case 2.  But if a term
+** also statisfies case 1 (such as B) we know that the optimizer will
+** always prefer case 1, so in that case we pretend that case 2 is not
+** satisfied.
+**
+** It might be the case that multiple tables are indexable.  For example,
+** (E) above is indexable on tables P, Q, and R.
+**
+** Terms that satisfy case 2 are candidates for lookup by using
+** separate indices to find rowids for each subterm and composing
+** the union of all rowids using a RowSet object.  This is similar
+** to "bitmap indices" in other database engines.
+**
+** OTHERWISE:
+**
+** If neither case 1 nor case 2 apply, then leave the eOperator set to
+** zero.  This term is not useful for search.
+*/
+static void exprAnalyzeOrTerm(
+  SrcList *pSrc,            /* the FROM clause */
+  WhereClause *pWC,         /* the complete WHERE clause */
+  int idxTerm               /* Index of the OR-term to be analyzed */
+){
+  Parse *pParse = pWC->pParse;            /* Parser context */
+  sqlite3 *db = pParse->db;               /* Database connection */
+  WhereTerm *pTerm = &pWC->a[idxTerm];    /* The term to be analyzed */
+  Expr *pExpr = pTerm->pExpr;             /* The expression of the term */
+  WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */
+  int i;                                  /* Loop counters */
+  WhereClause *pOrWc;       /* Breakup of pTerm into subterms */
+  WhereTerm *pOrTerm;       /* A Sub-term within the pOrWc */
+  WhereOrInfo *pOrInfo;     /* Additional information associated with pTerm */
+  Bitmask chngToIN;         /* Tables that might satisfy case 1 */
+  Bitmask indexable;        /* Tables that are indexable, satisfying case 2 */
+
+  /*
+  ** Break the OR clause into its separate subterms.  The subterms are
+  ** stored in a WhereClause structure containing within the WhereOrInfo
+  ** object that is attached to the original OR clause term.
+  */
+  assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 );
+  assert( pExpr->op==TK_OR );
+  pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo));
+  if( pOrInfo==0 ) return;
+  pTerm->wtFlags |= TERM_ORINFO;
+  pOrWc = &pOrInfo->wc;
+  whereClauseInit(pOrWc, pWC->pParse, pMaskSet);
+  whereSplit(pOrWc, pExpr, TK_OR);
+  exprAnalyzeAll(pSrc, pOrWc);
+  if( db->mallocFailed ) return;
+  assert( pOrWc->nTerm>=2 );
+
+  /*
+  ** Compute the set of tables that might satisfy cases 1 or 2.
+  */
+  indexable = chngToIN = ~(Bitmask)0;
+  for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){
+    if( (pOrTerm->eOperator & WO_SINGLE)==0 ){
+      WhereAndInfo *pAndInfo;
+      assert( pOrTerm->eOperator==0 );
+      assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 );
+      chngToIN = 0;
+      pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo));
+      if( pAndInfo ){
+        WhereClause *pAndWC;
+        WhereTerm *pAndTerm;
+        int j;
+        Bitmask b = 0;
+        pOrTerm->u.pAndInfo = pAndInfo;
+        pOrTerm->wtFlags |= TERM_ANDINFO;
+        pOrTerm->eOperator = WO_AND;
+        pAndWC = &pAndInfo->wc;
+        whereClauseInit(pAndWC, pWC->pParse, pMaskSet);
+        whereSplit(pAndWC, pOrTerm->pExpr, TK_AND);
+        exprAnalyzeAll(pSrc, pAndWC);
+        testcase( db->mallocFailed );
+        if( !db->mallocFailed ){
+          for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
+            assert( pAndTerm->pExpr );
+            if( allowedOp(pAndTerm->pExpr->op) ){
+              b |= getMask(pMaskSet, pAndTerm->leftCursor);
+            }
+          }
+        }
+        indexable &= b;
+      }
+    }else if( pOrTerm->wtFlags & TERM_COPIED ){
+      /* Skip this term for now.  We revisit it when we process the
+      ** corresponding TERM_VIRTUAL term */
+    }else{
+      Bitmask b;
+      b = getMask(pMaskSet, pOrTerm->leftCursor);
+      if( pOrTerm->wtFlags & TERM_VIRTUAL ){
+        WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent];
+        b |= getMask(pMaskSet, pOther->leftCursor);
+      }
+      indexable &= b;
+      if( pOrTerm->eOperator!=WO_EQ ){
+        chngToIN = 0;
+      }else{
+        chngToIN &= b;
+      }
+    }
+  }
+
+  /*
+  ** Record the set of tables that satisfy case 2.  The set might be
+  ** empty.
+  */
+  pOrInfo->indexable = indexable;
+  pTerm->eOperator = indexable==0 ? 0 : WO_OR;
+
+  /*
+  ** chngToIN holds a set of tables that *might* satisfy case 1.  But
+  ** we have to do some additional checking to see if case 1 really
+  ** is satisfied.
+  */
+  if( chngToIN ){
+    int okToChngToIN = 0;     /* True if the conversion to IN is valid */
+    int iColumn = -1;         /* Column index on lhs of IN operator */
+    int iCursor;              /* Table cursor common to all terms */
+    int j = 0;                /* Loop counter */
+
+    /* Search for a table and column that appears on one side or the
+    ** other of the == operator in every subterm.  That table and column
+    ** will be recorded in iCursor and iColumn.  There might not be any
+    ** such table and column.  Set okToChngToIN if an appropriate table
+    ** and column is found but leave okToChngToIN false if not found.
+    */
+    for(j=0; j<2 && !okToChngToIN; j++){
+      pOrTerm = pOrWc->a;
+      for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){
+        assert( pOrTerm->eOperator==WO_EQ );
+        pOrTerm->wtFlags &= ~TERM_OR_OK;
+        if( pOrTerm->leftCursor==iColumn ) continue;
+        if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ) continue;
+        iColumn = pOrTerm->u.leftColumn;
+        iCursor = pOrTerm->leftCursor;
+        break;
+      }
+      if( i<0 ){
+        assert( j==1 );
+        assert( (chngToIN&(chngToIN-1))==0 );
+        assert( chngToIN==getMask(pMaskSet, iColumn) );
+        break;
+      }
+      okToChngToIN = 1;
+      for(; i>=0 && okToChngToIN; i--, pOrTerm++){
+        assert( pOrTerm->eOperator==WO_EQ );
+        if( pOrTerm->leftCursor!=iCursor ){
+          pOrTerm->wtFlags &= ~TERM_OR_OK;
+        }else if( pOrTerm->u.leftColumn!=iColumn ){
+          okToChngToIN = 0;
+        }else{
+          int affLeft, affRight;
+          /* If the right-hand side is also a column, then the affinities
+          ** of both right and left sides must be such that no type
+          ** conversions are required on the right.  (Ticket #2249)
+          */
+          affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight);
+          affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft);
+          if( affRight!=0 && affRight!=affLeft ){
+            okToChngToIN = 0;
+          }else{
+            pOrTerm->wtFlags |= TERM_OR_OK;
+          }
+        }
+      }
+    }
+
+    /* At this point, okToChngToIN is true if original pTerm satisfies
+    ** case 1.  In that case, construct a new virtual term that is 
+    ** pTerm converted into an IN operator.
+    */
+    if( okToChngToIN ){
+      Expr *pDup;            /* A transient duplicate expression */
+      ExprList *pList = 0;   /* The RHS of the IN operator */
+      Expr *pLeft = 0;       /* The LHS of the IN operator */
+      Expr *pNew;            /* The complete IN operator */
+
+      for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){
+        if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue;
+        assert( pOrTerm->eOperator==WO_EQ );
+        assert( pOrTerm->leftCursor==iCursor );
+        assert( pOrTerm->u.leftColumn==iColumn );
+        pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight);
+        pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup, 0);
+        pLeft = pOrTerm->pExpr->pLeft;
+      }
+      assert( pLeft!=0 );
+      pDup = sqlite3ExprDup(db, pLeft);
+      pNew = sqlite3Expr(db, TK_IN, pDup, 0, 0);
+      if( pNew ){
+        int idxNew;
+        transferJoinMarkings(pNew, pExpr);
+        pNew->pList = pList;
+        idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC);
+        testcase( idxNew==0 );
+        exprAnalyze(pSrc, pWC, idxNew);
+        pTerm = &pWC->a[idxTerm];
+        pWC->a[idxNew].iParent = idxTerm;
+        pTerm->nChild = 1;
+      }else{
+        sqlite3ExprListDelete(db, pList);
+      }
+      pTerm->eOperator = 0;  /* case 1 trumps case 2 */
+    }
+  }
 }
 #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */
 
+
 /*
 ** The input to this routine is an WhereTerm structure with only the
 ** "pExpr" field filled in.  The job of this routine is to analyze the
 ** subexpression and populate all the other fields of the WhereTerm
 ** structure.
 **
 ** If the expression is of the form "<expr> <op> X" it gets commuted
-** to the standard form of "X <op> <expr>".  If the expression is of
-** the form "X <op> Y" where both X and Y are columns, then the original
-** expression is unchanged and a new virtual expression of the form
-** "Y <op> X" is added to the WHERE clause and analyzed separately.
+** to the standard form of "X <op> <expr>".
+**
+** If the expression is of the form "X <op> Y" where both X and Y are
+** columns, then the original expression is unchanged and a new virtual
+** term of the form "Y <op> X" is added to the WHERE clause and
+** analyzed separately.  The original term is marked with TERM_COPIED
+** and the new term is marked with TERM_DYNAMIC (because it's pExpr
+** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it
+** is a commuted copy of a prior term.)  The original term has nChild=1
+** and the copy has idxParent set to the index of the original term.
 */
 static void exprAnalyze(
   SrcList *pSrc,            /* the FROM clause */
   WhereClause *pWC,         /* the WHERE clause */
   int idxTerm               /* Index of the term to be analyzed */
 ){
-  WhereTerm *pTerm;
-  ExprMaskSet *pMaskSet;
-  Expr *pExpr;
-  Bitmask prereqLeft;
-  Bitmask prereqAll;
+  WhereTerm *pTerm;                /* The term to be analyzed */
+  WhereMaskSet *pMaskSet;          /* Set of table index masks */
+  Expr *pExpr;                     /* The expression to be analyzed */
+  Bitmask prereqLeft;              /* Prerequesites of the pExpr->pLeft */
+  Bitmask prereqAll;               /* Prerequesites of pExpr */
   Bitmask extraRight = 0;
   int nPattern;
   int isComplete;
   int noCase;
-  int op;
-  Parse *pParse = pWC->pParse;
-  sqlite3 *db = pParse->db;
+  int op;                          /* Top-level operator.  pExpr->op */
+  Parse *pParse = pWC->pParse;     /* Parsing context */
+  sqlite3 *db = pParse->db;        /* Database connection */
 
   if( db->mallocFailed ){
     return;
   }
   pTerm = &pWC->a[idxTerm];
   pMaskSet = pWC->pMaskSet;
   pExpr = pTerm->pExpr;
   prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft);
@@ -79496,17 +80503,17 @@ static void exprAnalyze(
   pTerm->leftCursor = -1;
   pTerm->iParent = -1;
   pTerm->eOperator = 0;
   if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){
     Expr *pLeft = pExpr->pLeft;
     Expr *pRight = pExpr->pRight;
     if( pLeft->op==TK_COLUMN ){
       pTerm->leftCursor = pLeft->iTable;
-      pTerm->leftColumn = pLeft->iColumn;
+      pTerm->u.leftColumn = pLeft->iColumn;
       pTerm->eOperator = operatorMask(op);
     }
     if( pRight && pRight->op==TK_COLUMN ){
       WhereTerm *pNew;
       Expr *pDup;
       if( pTerm->leftCursor>=0 ){
         int idxNew;
         pDup = sqlite3ExprDup(db, pExpr);
@@ -79523,28 +80530,40 @@ static void exprAnalyze(
         pTerm->wtFlags |= TERM_COPIED;
       }else{
         pDup = pExpr;
         pNew = pTerm;
       }
       exprCommute(pParse, pDup);
       pLeft = pDup->pLeft;
       pNew->leftCursor = pLeft->iTable;
-      pNew->leftColumn = pLeft->iColumn;
+      pNew->u.leftColumn = pLeft->iColumn;
       pNew->prereqRight = prereqLeft;
       pNew->prereqAll = prereqAll;
       pNew->eOperator = operatorMask(pDup->op);
     }
   }
 
 #ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION
   /* If a term is the BETWEEN operator, create two new virtual terms
-  ** that define the range that the BETWEEN implements.
-  */
-  else if( pExpr->op==TK_BETWEEN ){
+  ** that define the range that the BETWEEN implements.  For example:
+  **
+  **      a BETWEEN b AND c
+  **
+  ** is converted into:
+  **
+  **      (a BETWEEN b AND c) AND (a>=b) AND (a<=c)
+  **
+  ** The two new terms are added onto the end of the WhereClause object.
+  ** The new terms are "dynamic" and are children of the original BETWEEN
+  ** term.  That means that if the BETWEEN term is coded, the children are
+  ** skipped.  Or, if the children are satisfied by an index, the original
+  ** BETWEEN term is skipped.
+  */
+  else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){
     ExprList *pList = pExpr->pList;
     int i;
     static const u8 ops[] = {TK_GE, TK_LE};
     assert( pList!=0 );
     assert( pList->nExpr==2 );
     for(i=0; i<2; i++){
       Expr *pNewExpr;
       int idxNew;
@@ -79556,104 +80575,38 @@ static void exprAnalyze(
       pTerm = &pWC->a[idxTerm];
       pWC->a[idxNew].iParent = idxTerm;
     }
     pTerm->nChild = 2;
   }
 #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */
 
 #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY)
-  /* Attempt to convert OR-connected terms into an IN operator so that
-  ** they can make use of indices.  Example:
-  **
-  **      x = expr1  OR  expr2 = x  OR  x = expr3
-  **
-  ** is converted into
-  **
-  **      x IN (expr1,expr2,expr3)
-  **
-  ** This optimization must be omitted if OMIT_SUBQUERY is defined because
-  ** the compiler for the the IN operator is part of sub-queries.
+  /* Analyze a term that is composed of two or more subterms connected by
+  ** an OR operator.
   */
   else if( pExpr->op==TK_OR ){
-    int ok;
-    int i, j;
-    int iColumn, iCursor;
-    WhereClause sOr;
-    WhereTerm *pOrTerm;
-
-    assert( (pTerm->wtFlags & TERM_DYNAMIC)==0 );
-    whereClauseInit(&sOr, pWC->pParse, pMaskSet);
-    whereSplit(&sOr, pExpr, TK_OR);
-    exprAnalyzeAll(pSrc, &sOr);
-    assert( sOr.nTerm>=2 );
-    j = 0;
-    if( db->mallocFailed ) goto or_not_possible;
-    do{
-      assert( j<sOr.nTerm );
-      iColumn = sOr.a[j].leftColumn;
-      iCursor = sOr.a[j].leftCursor;
-      ok = iCursor>=0;
-      for(i=sOr.nTerm-1, pOrTerm=sOr.a; i>=0 && ok; i--, pOrTerm++){
-        if( pOrTerm->eOperator!=WO_EQ ){
-          goto or_not_possible;
-        }
-        if( orTermIsOptCandidate(pOrTerm, iCursor, iColumn) ){
-          pOrTerm->wtFlags |= TERM_OR_OK;
-        }else if( orTermHasOkDuplicate(&sOr, pOrTerm) ){
-          pOrTerm->wtFlags &= ~TERM_OR_OK;
-        }else{
-          ok = 0;
-        }
-      }
-    }while( !ok && (sOr.a[j++].wtFlags & TERM_COPIED)!=0 && j<2 );
-    if( ok ){
-      ExprList *pList = 0;
-      Expr *pNew, *pDup;
-      Expr *pLeft = 0;
-      for(i=sOr.nTerm-1, pOrTerm=sOr.a; i>=0; i--, pOrTerm++){
-        if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue;
-        pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight);
-        pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup, 0);
-        pLeft = pOrTerm->pExpr->pLeft;
-      }
-      assert( pLeft!=0 );
-      pDup = sqlite3ExprDup(db, pLeft);
-      pNew = sqlite3Expr(db, TK_IN, pDup, 0, 0);
-      if( pNew ){
-        int idxNew;
-        transferJoinMarkings(pNew, pExpr);
-        pNew->pList = pList;
-        idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC);
-        testcase( idxNew==0 );
-        exprAnalyze(pSrc, pWC, idxNew);
-        pTerm = &pWC->a[idxTerm];
-        pWC->a[idxNew].iParent = idxTerm;
-        pTerm->nChild = 1;
-      }else{
-        sqlite3ExprListDelete(db, pList);
-      }
-    }
-or_not_possible:
-    whereClauseClear(&sOr);
+    assert( pWC->op==TK_AND );
+    exprAnalyzeOrTerm(pSrc, pWC, idxTerm);
   }
 #endif /* SQLITE_OMIT_OR_OPTIMIZATION */
 
 #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION
   /* Add constraints to reduce the search space on a LIKE or GLOB
   ** operator.
   **
   ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints
   **
   **          x>='abc' AND x<'abd' AND x LIKE 'abc%'
   **
   ** The last character of the prefix "abc" is incremented to form the
   ** termination condition "abd".
   */
-  if( isLikeOrGlob(pParse, pExpr, &nPattern, &isComplete, &noCase) ){
+  if( isLikeOrGlob(pParse, pExpr, &nPattern, &isComplete, &noCase)
+         && pWC->op==TK_AND ){
     Expr *pLeft, *pRight;
     Expr *pStr1, *pStr2;
     Expr *pNewExpr1, *pNewExpr2;
     int idxNew1, idxNew2;
 
     pLeft = pExpr->pList->a[1].pExpr;
     pRight = pExpr->pList->a[0].pExpr;
     pStr1 = sqlite3PExpr(pParse, TK_STRING, 0, 0, 0);
@@ -79711,17 +80664,17 @@ or_not_possible:
     if( (prereqExpr & prereqColumn)==0 ){
       Expr *pNewExpr;
       pNewExpr = sqlite3Expr(db, TK_MATCH, 0, sqlite3ExprDup(db, pRight), 0);
       idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
       testcase( idxNew==0 );
       pNewTerm = &pWC->a[idxNew];
       pNewTerm->prereqRight = prereqExpr;
       pNewTerm->leftCursor = pLeft->iTable;
-      pNewTerm->leftColumn = pLeft->iColumn;
+      pNewTerm->u.leftColumn = pLeft->iColumn;
       pNewTerm->eOperator = WO_MATCH;
       pNewTerm->iParent = idxTerm;
       pTerm = &pWC->a[idxTerm];
       pTerm->nChild = 1;
       pTerm->wtFlags |= TERM_COPIED;
       pNewTerm->prereqAll = pTerm->prereqAll;
     }
   }
@@ -79734,17 +80687,17 @@ or_not_possible:
 }
 
 /*
 ** Return TRUE if any of the expressions in pList->a[iFirst...] contain
 ** a reference to any table other than the iBase table.
 */
 static int referencesOtherTables(
   ExprList *pList,          /* Search expressions in ths list */
-  ExprMaskSet *pMaskSet,    /* Mapping from tables to bitmaps */
+  WhereMaskSet *pMaskSet,   /* Mapping from tables to bitmaps */
   int iFirst,               /* Be searching with the iFirst-th expression */
   int iBase                 /* Ignore references to this table */
 ){
   Bitmask allowed = ~getMask(pMaskSet, iBase);
   while( iFirst<pList->nExpr ){
     if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){
       return 1;
     }
@@ -79769,17 +80722,17 @@ static int referencesOtherTables(
 ** All terms of the ORDER BY that match against the index must be either
 ** ASC or DESC.  (Terms of the ORDER BY clause past the end of a UNIQUE
 ** index do not need to satisfy this constraint.)  The *pbRev value is
 ** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if
 ** the ORDER BY clause is all ASC.
 */
 static int isSortingIndex(
   Parse *pParse,          /* Parsing context */
-  ExprMaskSet *pMaskSet,  /* Mapping from table indices to bitmaps */
+  WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */
   Index *pIdx,            /* The index we are testing */
   int base,               /* Cursor number for the table to be sorted */
   ExprList *pOrderBy,     /* The ORDER BY clause */
   int nEqCol,             /* Number of index columns with == constraints */
   int *pbRev              /* Set to 1 if ORDER BY is DESC */
 ){
   int i, j;                       /* Loop counters */
   int sortOrder = 0;              /* XOR of index and ORDER BY sort direction */
@@ -79892,17 +80845,17 @@ static int isSortingIndex(
 /*
 ** Check table to see if the ORDER BY clause in pOrderBy can be satisfied
 ** by sorting in order of ROWID.  Return true if so and set *pbRev to be
 ** true for reverse ROWID and false for forward ROWID order.
 */
 static int sortableByRowid(
   int base,               /* Cursor number for table to be sorted */
   ExprList *pOrderBy,     /* The ORDER BY clause */
-  ExprMaskSet *pMaskSet,  /* Mapping from tables to bitmaps */
+  WhereMaskSet *pMaskSet, /* Mapping from table cursors to bitmaps */
   int *pbRev              /* Set to 1 if ORDER BY is DESC */
 ){
   Expr *p;
 
   assert( pOrderBy!=0 );
   assert( pOrderBy->nExpr>0 );
   p = pOrderBy->a[0].pExpr;
   if( p->op==TK_COLUMN && p->iTable==base && p->iColumn==-1
@@ -80073,17 +81026,17 @@ static double bestVirtualIndex(
                                                                      pUsage;
 
     for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
       if( pTerm->leftCursor != pSrc->iCursor ) continue;
       assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 );
       testcase( pTerm->eOperator==WO_IN );
       testcase( pTerm->eOperator==WO_ISNULL );
       if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue;
-      pIdxCons[j].iColumn = pTerm->leftColumn;
+      pIdxCons[j].iColumn = pTerm->u.leftColumn;
       pIdxCons[j].iTermOffset = i;
       pIdxCons[j].op = (u8)pTerm->eOperator;
       /* The direct assignment in the previous line is possible only because
       ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
       ** following asserts verify this fact. */
       assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
       assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
       assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
@@ -80190,160 +81143,215 @@ static double bestVirtualIndex(
   }
 
   *(int*)&pIdxInfo->nOrderBy = nOrderBy;
   return pIdxInfo->estimatedCost;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
 /*
-** Find the best index for accessing a particular table.  Return a pointer
-** to the index, flags that describe how the index should be used, the
-** number of equality constraints, and the "cost" for this index.
-**
-** The lowest cost index wins.  The cost is an estimate of the amount of
-** CPU and disk I/O need to process the request using the selected index.
+** Find the query plan for accessing a particular table.  Write the
+** best query plan and its cost into the WhereCost object supplied as the
+** last parameter.
+**
+** The lowest cost plan wins.  The cost is an estimate of the amount of
+** CPU and disk I/O need to process the request using the selected plan.
 ** Factors that influence cost include:
 **
 **    *  The estimated number of rows that will be retrieved.  (The
 **       fewer the better.)
 **
 **    *  Whether or not sorting must occur.
 **
 **    *  Whether or not there must be separate lookups in the
 **       index and in the main table.
 **
 ** If there was an INDEXED BY clause attached to the table in the SELECT
-** statement, then this function only considers strategies using the 
+** statement, then this function only considers plans using the 
 ** named index. If one cannot be found, then the returned cost is
-** SQLITE_BIG_DBL. If a strategy can be found that uses the named index, 
+** SQLITE_BIG_DBL. If a plan can be found that uses the named index, 
 ** then the cost is calculated in the usual way.
 **
 ** If a NOT INDEXED clause was attached to the table in the SELECT 
 ** statement, then no indexes are considered. However, the selected 
-** stategy may still take advantage of the tables built-in rowid
+** plan may still take advantage of the tables built-in rowid
 ** index.
 */
-static double bestIndex(
+static void bestIndex(
   Parse *pParse,              /* The parsing context */
   WhereClause *pWC,           /* The WHERE clause */
   struct SrcList_item *pSrc,  /* The FROM clause term to search */
   Bitmask notReady,           /* Mask of cursors that are not available */
-  ExprList *pOrderBy,         /* The order by clause */
-  Index **ppIndex,            /* Make *ppIndex point to the best index */
-  int *pWsFlags,              /* Put wsFlags describing scan strategy here */
-  int *pnEq                   /* Put the number of == or IN constraints here */
-){
-  WhereTerm *pTerm;
-  Index *bestIdx = 0;         /* Index that gives the lowest cost */
-  double lowestCost;          /* The cost of using bestIdx */
-  int bestWsFlags = 0;        /* Flags associated with bestIdx */
-  int bestNEq = 0;            /* Best value for nEq */
+  ExprList *pOrderBy,         /* The ORDER BY clause */
+  WhereCost *pCost            /* Lowest cost query plan */
+){
+  WhereTerm *pTerm;           /* A single term of the WHERE clause */
   int iCur = pSrc->iCursor;   /* The cursor of the table to be accessed */
   Index *pProbe;              /* An index we are evaluating */
   int rev;                    /* True to scan in reverse order */
   int wsFlags;                /* Flags associated with pProbe */
   int nEq;                    /* Number of == or IN constraints */
   int eqTermMask;             /* Mask of valid equality operators */
   double cost;                /* Cost of using pProbe */
+  double nRow;                /* Estimated number of rows in result set */
+  int i;                      /* Loop counter */
+  Bitmask maskSrc;            /* Bitmask for the pSrc table */
 
   WHERETRACE(("bestIndex: tbl=%s notReady=%llx\n", pSrc->pTab->zName,notReady));
-  lowestCost = SQLITE_BIG_DBL;
   pProbe = pSrc->pTab->pIndex;
   if( pSrc->notIndexed ){
     pProbe = 0;
   }
 
   /* If the table has no indices and there are no terms in the where
   ** clause that refer to the ROWID, then we will never be able to do
   ** anything other than a full table scan on this table.  We might as
   ** well put it first in the join order.  That way, perhaps it can be
   ** referenced by other tables in the join.
   */
+  memset(pCost, 0, sizeof(*pCost));
   if( pProbe==0 &&
      findTerm(pWC, iCur, -1, 0, WO_EQ|WO_IN|WO_LT|WO_LE|WO_GT|WO_GE,0)==0 &&
      (pOrderBy==0 || !sortableByRowid(iCur, pOrderBy, pWC->pMaskSet, &rev)) ){
-    *pWsFlags = 0;
-    *ppIndex = 0;
-    *pnEq = 0;
-    return 0.0;
-  }
+    return;
+  }
+  pCost->rCost = SQLITE_BIG_DBL;
 
   /* Check for a rowid=EXPR or rowid IN (...) constraints. If there was
   ** an INDEXED BY clause attached to this table, skip this step.
   */
   if( !pSrc->pIndex ){
     pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0);
     if( pTerm ){
       Expr *pExpr;
-      *ppIndex = 0;
-      bestWsFlags = WHERE_ROWID_EQ;
+      pCost->plan.wsFlags = WHERE_ROWID_EQ;
       if( pTerm->eOperator & WO_EQ ){
         /* Rowid== is always the best pick.  Look no further.  Because only
         ** a single row is generated, output is always in sorted order */
-        *pWsFlags = WHERE_ROWID_EQ | WHERE_UNIQUE;
-        *pnEq = 1;
+        pCost->plan.wsFlags = WHERE_ROWID_EQ | WHERE_UNIQUE;
+        pCost->plan.nEq = 1;
         WHERETRACE(("... best is rowid\n"));
-        return 0.0;
+        pCost->rCost = 0;
+        pCost->nRow = 1;
+        return;
       }else if( (pExpr = pTerm->pExpr)->pList!=0 ){
         /* Rowid IN (LIST): cost is NlogN where N is the number of list
         ** elements.  */
-        lowestCost = pExpr->pList->nExpr;
-        lowestCost *= estLog(lowestCost);
+        pCost->rCost = pCost->nRow = pExpr->pList->nExpr;
+        pCost->rCost *= estLog(pCost->rCost);
       }else{
         /* Rowid IN (SELECT): cost is NlogN where N is the number of rows
         ** in the result of the inner select.  We have no way to estimate
         ** that value so make a wild guess. */
-        lowestCost = 200;
-      }
-      WHERETRACE(("... rowid IN cost: %.9g\n", lowestCost));
+        pCost->nRow = 100;
+        pCost->rCost = 200;
+      }
+      WHERETRACE(("... rowid IN cost: %.9g\n", pCost->rCost));
     }
   
     /* Estimate the cost of a table scan.  If we do not know how many
     ** entries are in the table, use 1 million as a guess.
     */
     cost = pProbe ? pProbe->aiRowEst[0] : 1000000;
     WHERETRACE(("... table scan base cost: %.9g\n", cost));
     wsFlags = WHERE_ROWID_RANGE;
   
     /* Check for constraints on a range of rowids in a table scan.
     */
     pTerm = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE|WO_GT|WO_GE, 0);
     if( pTerm ){
       if( findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0) ){
         wsFlags |= WHERE_TOP_LIMIT;
-        cost /= 3;  /* Guess that rowid<EXPR eliminates two-thirds or rows */
+        cost /= 3;  /* Guess that rowid<EXPR eliminates two-thirds of rows */
       }
       if( findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0) ){
         wsFlags |= WHERE_BTM_LIMIT;
         cost /= 3;  /* Guess that rowid>EXPR eliminates two-thirds of rows */
       }
       WHERETRACE(("... rowid range reduces cost to %.9g\n", cost));
     }else{
       wsFlags = 0;
     }
+    nRow = cost;
   
     /* If the table scan does not satisfy the ORDER BY clause, increase
     ** the cost by NlogN to cover the expense of sorting. */
     if( pOrderBy ){
       if( sortableByRowid(iCur, pOrderBy, pWC->pMaskSet, &rev) ){
         wsFlags |= WHERE_ORDERBY|WHERE_ROWID_RANGE;
         if( rev ){
           wsFlags |= WHERE_REVERSE;
         }
       }else{
         cost += cost*estLog(cost);
         WHERETRACE(("... sorting increases cost to %.9g\n", cost));
       }
     }
-    if( cost<lowestCost ){
-      lowestCost = cost;
-      bestWsFlags = wsFlags;
-    }
-  }
+    if( cost<pCost->rCost ){
+      pCost->rCost = cost;
+      pCost->nRow = nRow;
+      pCost->plan.wsFlags = wsFlags;
+    }
+  }
+
+#ifndef SQLITE_OMIT_OR_OPTIMIZATION
+  /* Search for an OR-clause that can be used to look up the table.
+  */
+  maskSrc = getMask(pWC->pMaskSet, iCur);
+  for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
+    WhereClause tempWC;
+    tempWC = *pWC;
+    if( pTerm->eOperator==WO_OR 
+        && ((pTerm->prereqAll & ~maskSrc) & notReady)==0
+        && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){
+      WhereClause *pOrWC = &pTerm->u.pOrInfo->wc;
+      WhereTerm *pOrTerm;
+      int j;
+      int sortable = 0;
+      double rTotal = 0;
+      nRow = 0;
+      for(j=0, pOrTerm=pOrWC->a; j<pOrWC->nTerm; j++, pOrTerm++){
+        WhereCost sTermCost;
+        WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", j,i));
+        if( pOrTerm->eOperator==WO_AND ){
+          WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
+          bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
+        }else if( pOrTerm->leftCursor==iCur ){
+          tempWC.a = pOrTerm;
+          tempWC.nTerm = 1;
+          bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
+        }else{
+          continue;
+        }
+        rTotal += sTermCost.rCost;
+        nRow += sTermCost.nRow;
+        if( rTotal>=pCost->rCost ) break;
+      }
+      if( pOrderBy!=0 ){
+        if( sortableByRowid(iCur, pOrderBy, pWC->pMaskSet, &rev) && !rev ){
+          sortable = 1;
+        }else{
+          rTotal += nRow*estLog(nRow);
+          WHERETRACE(("... sorting increases OR cost to %.9g\n", rTotal));
+        }
+      }
+      WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n",
+                  rTotal, nRow));
+      if( rTotal<pCost->rCost ){
+        pCost->rCost = rTotal;
+        pCost->nRow = nRow;
+        pCost->plan.wsFlags = WHERE_MULTI_OR;
+        pCost->plan.u.pTerm = pTerm;
+        if( sortable ){
+          pCost->plan.wsFlags = WHERE_ORDERBY|WHERE_MULTI_OR;
+        }
+      }
+    }
+  }
+#endif /* SQLITE_OMIT_OR_OPTIMIZATION */
 
   /* If the pSrc table is the right table of a LEFT JOIN then we may not
   ** use an index to satisfy IS NULL constraints on that table.  This is
   ** because columns might end up being NULL if the table does not match -
   ** a circumstance which the index cannot help us discover.  Ticket #2177.
   */
   if( (pSrc->jointype & JT_LEFT)!=0 ){
     eqTermMask = WO_EQ|WO_IN;
@@ -80352,17 +81360,16 @@ static double bestIndex(
   }
 
   /* Look at each index.
   */
   if( pSrc->pIndex ){
     pProbe = pSrc->pIndex;
   }
   for(; pProbe; pProbe=(pSrc->pIndex ? 0 : pProbe->pNext)){
-    int i;                       /* Loop counter */
     double inMultiplier = 1;
 
     WHERETRACE(("... index %s:\n", pProbe->zName));
 
     /* Count the number of columns in the index that are satisfied
     ** by x=EXPR constraints or x IN (...) constraints.
     */
     wsFlags = 0;
@@ -80376,17 +81383,18 @@ static double bestIndex(
         wsFlags |= WHERE_COLUMN_IN;
         if( pExpr->pSelect!=0 ){
           inMultiplier *= 25;
         }else if( ALWAYS(pExpr->pList) ){
           inMultiplier *= pExpr->pList->nExpr + 1;
         }
       }
     }
-    cost = pProbe->aiRowEst[i] * inMultiplier * estLog(inMultiplier);
+    nRow = pProbe->aiRowEst[i] * inMultiplier;
+    cost = nRow * estLog(inMultiplier);
     nEq = i;
     if( pProbe->onError!=OE_None && (wsFlags & WHERE_COLUMN_IN)==0
          && nEq==pProbe->nColumn ){
       wsFlags |= WHERE_UNIQUE;
     }
     WHERETRACE(("...... nEq=%d inMult=%.9g cost=%.9g\n",nEq,inMultiplier,cost));
 
     /* Look for range constraints
@@ -80394,20 +81402,22 @@ static double bestIndex(
     if( nEq<pProbe->nColumn ){
       int j = pProbe->aiColumn[nEq];
       pTerm = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pProbe);
       if( pTerm ){
         wsFlags |= WHERE_COLUMN_RANGE;
         if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pProbe) ){
           wsFlags |= WHERE_TOP_LIMIT;
           cost /= 3;
+          nRow /= 3;
         }
         if( findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pProbe) ){
           wsFlags |= WHERE_BTM_LIMIT;
           cost /= 3;
+          nRow /= 3;
         }
         WHERETRACE(("...... range reduces cost to %.9g\n", cost));
       }
     }
 
     /* Add the additional cost of sorting if that is a factor.
     */
     if( pOrderBy ){
@@ -80443,32 +81453,33 @@ static double bestIndex(
         wsFlags |= WHERE_IDX_ONLY;
         cost /= 2;
         WHERETRACE(("...... idx-only reduces cost to %.9g\n", cost));
       }
     }
 
     /* If this index has achieved the lowest cost so far, then use it.
     */
-    if( wsFlags && cost < lowestCost ){
-      bestIdx = pProbe;
-      lowestCost = cost;
-      bestWsFlags = wsFlags;
-      bestNEq = nEq;
+    if( wsFlags!=0 && cost < pCost->rCost ){
+      pCost->rCost = cost;
+      pCost->nRow = nRow;
+      pCost->plan.wsFlags = wsFlags;
+      pCost->plan.nEq = nEq;
+      assert( pCost->plan.wsFlags & WHERE_INDEXED );
+      pCost->plan.u.pIdx = pProbe;
     }
   }
 
   /* Report the best result
   */
-  *ppIndex = bestIdx;
-  WHERETRACE(("best index is %s, cost=%.9g, wsFlags=%x, nEq=%d\n",
-        bestIdx ? bestIdx->zName : "(none)", lowestCost, bestWsFlags, bestNEq));
-  *pWsFlags = bestWsFlags | eqTermMask;
-  *pnEq = bestNEq;
-  return lowestCost;
+  pCost->plan.wsFlags |= eqTermMask;
+  WHERETRACE(("best index is %s, cost=%.9g, nrow=%.9g, wsFlags=%x, nEq=%d\n",
+        (pCost->plan.wsFlags & WHERE_INDEXED)!=0 ?
+             pCost->plan.u.pIdx->zName : "(none)", pCost->nRow,
+        pCost->rCost, pCost->plan.wsFlags, pCost->plan.nEq));
 }
 
 
 /*
 ** Disable a term in the WHERE clause.  Except, do not disable the term
 ** if it controls a LEFT OUTER JOIN and it did not originate in the ON
 ** or USING clause of that join.
 **
@@ -80554,110 +81565,713 @@ static int codeEqualityTerm(
     struct InLoop *pIn;
 
     assert( pX->op==TK_IN );
     iReg = iTarget;
     eType = sqlite3FindInIndex(pParse, pX, 0);
     iTab = pX->iTable;
     sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
     VdbeComment((v, "%.*s", pX->span.n, pX->span.z));
-    if( pLevel->nIn==0 ){
+    assert( pLevel->plan.wsFlags & WHERE_IN_ABLE );
+    if( pLevel->u.in.nIn==0 ){
       pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
     }
-    pLevel->nIn++;
-    pLevel->aInLoop = sqlite3DbReallocOrFree(pParse->db, pLevel->aInLoop,
-                                    sizeof(pLevel->aInLoop[0])*pLevel->nIn);
-    pIn = pLevel->aInLoop;
+    pLevel->u.in.nIn++;
+    pLevel->u.in.aInLoop =
+       sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop,
+                              sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
+    pIn = pLevel->u.in.aInLoop;
     if( pIn ){
-      pIn += pLevel->nIn - 1;
+      pIn += pLevel->u.in.nIn - 1;
       pIn->iCur = iTab;
       if( eType==IN_INDEX_ROWID ){
         pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg);
       }else{
         pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg);
       }
       sqlite3VdbeAddOp1(v, OP_IsNull, iReg);
     }else{
-      pLevel->nIn = 0;
+      pLevel->u.in.nIn = 0;
     }
 #endif
   }
   disableTerm(pLevel, pTerm);
   return iReg;
 }
 
 /*
 ** Generate code that will evaluate all == and IN constraints for an
 ** index.  The values for all constraints are left on the stack.
 **
 ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
 ** Suppose the WHERE clause is this:  a==5 AND b IN (1,2,3) AND c>5 AND c<10
 ** The index has as many as three equality constraints, but in this
 ** example, the third "c" value is an inequality.  So only two 
 ** constraints are coded.  This routine will generate code to evaluate
-** a==5 and b IN (1,2,3).  The current values for a and b will be left
-** on the stack - a is the deepest and b the shallowest.
+** a==5 and b IN (1,2,3).  The current values for a and b will be stored
+** in consecutive registers and the index of the first register is returned.
 **
 ** In the example above nEq==2.  But this subroutine works for any value
 ** of nEq including 0.  If nEq==0, this routine is nearly a no-op.
 ** The only thing it does is allocate the pLevel->iMem memory cell.
 **
-** This routine always allocates at least one memory cell and puts
-** the address of that memory cell in pLevel->iMem.  The code that
-** calls this routine will use pLevel->iMem to store the termination
+** This routine always allocates at least one memory cell and returns
+** the index of that memory cell. The code that
+** calls this routine will use that memory cell to store the termination
 ** key value of the loop.  If one or more IN operators appear, then
 ** this routine allocates an additional nEq memory cells for internal
 ** use.
 */
 static int codeAllEqualityTerms(
   Parse *pParse,        /* Parsing context */
   WhereLevel *pLevel,   /* Which nested loop of the FROM we are coding */
   WhereClause *pWC,     /* The WHERE clause */
   Bitmask notReady,     /* Which parts of FROM have not yet been coded */
   int nExtraReg         /* Number of extra registers to allocate */
 ){
-  int nEq = pLevel->nEq;        /* The number of == or IN constraints to code */
-  Vdbe *v = pParse->pVdbe;      /* The virtual machine under construction */
-  Index *pIdx = pLevel->pIdx;   /* The index being used for this loop */
+  int nEq = pLevel->plan.nEq;   /* The number of == or IN constraints to code */
+  Vdbe *v = pParse->pVdbe;      /* The vm under construction */
+  Index *pIdx;                  /* The index being used for this loop */
   int iCur = pLevel->iTabCur;   /* The cursor of the table */
   WhereTerm *pTerm;             /* A single constraint term */
   int j;                        /* Loop counter */
   int regBase;                  /* Base register */
+  int nReg;                     /* Number of registers to allocate */
+
+  /* This module is only called on query plans that use an index. */
+  assert( pLevel->plan.wsFlags & WHERE_INDEXED );
+  pIdx = pLevel->plan.u.pIdx;
 
   /* Figure out how many memory cells we will need then allocate them.
-  ** We always need at least one used to store the loop terminator
-  ** value.  If there are IN operators we'll need one for each == or
-  ** IN constraint.
-  */
-  pLevel->iMem = pParse->nMem + 1;
-  regBase = pParse->nMem + 2;
-  pParse->nMem += pLevel->nEq + 2 + nExtraReg;
+  */
+  regBase = pParse->nMem + 1;
+  nReg = pLevel->plan.nEq + nExtraReg;
+  pParse->nMem += nReg;
 
   /* Evaluate the equality constraints
   */
   assert( pIdx->nColumn>=nEq );
   for(j=0; j<nEq; j++){
     int r1;
     int k = pIdx->aiColumn[j];
-    pTerm = findTerm(pWC, iCur, k, notReady, pLevel->wsFlags, pIdx);
+    pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx);
     if( NEVER(pTerm==0) ) break;
     assert( (pTerm->wtFlags & TERM_CODED)==0 );
     r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j);
     if( r1!=regBase+j ){
-      sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j);
+      if( nReg==1 ){
+        sqlite3ReleaseTempReg(pParse, regBase);
+        regBase = r1;
+      }else{
+        sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j);
+      }
     }
     testcase( pTerm->eOperator & WO_ISNULL );
     testcase( pTerm->eOperator & WO_IN );
     if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){
       sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
     }
   }
   return regBase;
 }
 
+/*
+** Return TRUE if the WhereClause pWC contains no terms that
+** are not virtual and which have not been coded.
+**
+** To put it another way, return TRUE if no additional WHERE clauses
+** tests are required in order to establish that the current row
+** should go to output and return FALSE if there are some terms of
+** the WHERE clause that need to be validated before outputing the row.
+*/
+static int whereRowReadyForOutput(WhereClause *pWC){
+  WhereTerm *pTerm;
+  int j;
+ 
+  for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
+    if( (pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED))==0 ) return 0;
+  }
+  return 1;
+}
+
+/*
+** Generate code for the start of the iLevel-th loop in the WHERE clause
+** implementation described by pWInfo.
+*/
+static Bitmask codeOneLoopStart(
+  WhereInfo *pWInfo,   /* Complete information about the WHERE clause */
+  int iLevel,          /* Which level of pWInfo->a[] should be coded */
+  u8 wctrlFlags,       /* One of the WHERE_* flags defined in sqliteInt.h */
+  Bitmask notReady     /* Which tables are currently available */
+){
+  int j, k;            /* Loop counters */
+  int iCur;            /* The VDBE cursor for the table */
+  int addrNxt;         /* Where to jump to continue with the next IN case */
+  int omitTable;       /* True if we use the index only */
+  int bRev;            /* True if we need to scan in reverse order */
+  WhereLevel *pLevel;  /* The where level to be coded */
+  WhereClause *pWC;    /* Decomposition of the entire WHERE clause */
+  WhereTerm *pTerm;               /* A WHERE clause term */
+  Parse *pParse;                  /* Parsing context */
+  Vdbe *v;                        /* The prepared stmt under constructions */
+  struct SrcList_item *pTabItem;  /* FROM clause term being coded */
+  int addrBrk;                    /* Jump here to break out of the loop */
+  int addrCont;                   /* Jump here to continue with next cycle */
+  int regRowSet;       /* Write rowids to this RowSet if non-negative */
+  int codeRowSetEarly; /* True if index fully constrains the search */
+  
+
+  pParse = pWInfo->pParse;
+  v = pParse->pVdbe;
+  pWC = pWInfo->pWC;
+  pLevel = &pWInfo->a[iLevel];
+  pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
+  iCur = pTabItem->iCursor;
+  bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;
+  omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0;
+  regRowSet = pWInfo->regRowSet;
+  codeRowSetEarly = 0;
+
+  /* Create labels for the "break" and "continue" instructions
+  ** for the current loop.  Jump to addrBrk to break out of a loop.
+  ** Jump to cont to go immediately to the next iteration of the
+  ** loop.
+  **
+  ** When there is an IN operator, we also have a "addrNxt" label that
+  ** means to continue with the next IN value combination.  When
+  ** there are no IN operators in the constraints, the "addrNxt" label
+  ** is the same as "addrBrk".
+  */
+  addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
+  addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v);
+
+  /* If this is the right table of a LEFT OUTER JOIN, allocate and
+  ** initialize a memory cell that records if this table matches any
+  ** row of the left table of the join.
+  */
+  if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){
+    pLevel->iLeftJoin = ++pParse->nMem;
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin);
+    VdbeComment((v, "init LEFT JOIN no-match flag"));
+  }
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if(  (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
+    /* Case 0:  The table is a virtual-table.  Use the VFilter and VNext
+    **          to access the data.
+    */
+    int iReg;   /* P3 Value for OP_VFilter */
+    sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
+    int nConstraint = pVtabIdx->nConstraint;
+    struct sqlite3_index_constraint_usage *aUsage =
+                                                pVtabIdx->aConstraintUsage;
+    const struct sqlite3_index_constraint *aConstraint =
+                                                pVtabIdx->aConstraint;
+
+    iReg = sqlite3GetTempRange(pParse, nConstraint+2);
+    pParse->disableColCache++;
+    for(j=1; j<=nConstraint; j++){
+      for(k=0; k<nConstraint; k++){
+        if( aUsage[k].argvIndex==j ){
+          int iTerm = aConstraint[k].iTermOffset;
+          assert( pParse->disableColCache );
+          sqlite3ExprCode(pParse, pWC->a[iTerm].pExpr->pRight, iReg+j+1);
+          break;
+        }
+      }
+      if( k==nConstraint ) break;
+    }
+    assert( pParse->disableColCache );
+    pParse->disableColCache--;
+    sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg);
+    sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1);
+    sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr,
+                      pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC);
+    pVtabIdx->needToFreeIdxStr = 0;
+    for(j=0; j<nConstraint; j++){
+      if( aUsage[j].omit ){
+        int iTerm = aConstraint[j].iTermOffset;
+        disableTerm(pLevel, &pWC->a[iTerm]);
+      }
+    }
+    pLevel->op = OP_VNext;
+    pLevel->p1 = iCur;
+    pLevel->p2 = sqlite3VdbeCurrentAddr(v);
+    codeRowSetEarly = regRowSet>=0 ? whereRowReadyForOutput(pWC) : 0;
+    if( codeRowSetEarly ){
+      sqlite3VdbeAddOp2(v, OP_VRowid, iCur, iReg);
+      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, iReg);
+    }
+    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+  }else
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+  if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){
+    /* Case 1:  We can directly reference a single row using an
+    **          equality comparison against the ROWID field.  Or
+    **          we reference multiple rows using a "rowid IN (...)"
+    **          construct.
+    */
+    int r1;
+    int rtmp = sqlite3GetTempReg(pParse);
+    pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0);
+    assert( pTerm!=0 );
+    assert( pTerm->pExpr!=0 );
+    assert( pTerm->leftCursor==iCur );
+    assert( omitTable==0 );
+    r1 = codeEqualityTerm(pParse, pTerm, pLevel, rtmp);
+    addrNxt = pLevel->addrNxt;
+    sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, addrNxt);
+    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, r1);
+    codeRowSetEarly = (pWC->nTerm==1 && regRowSet>=0) ?1:0;
+    if( codeRowSetEarly ){
+      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
+    }
+    sqlite3ReleaseTempReg(pParse, rtmp);
+    VdbeComment((v, "pk"));
+    pLevel->op = OP_Noop;
+  }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){
+    /* Case 2:  We have an inequality comparison against the ROWID field.
+    */
+    int testOp = OP_Noop;
+    int start;
+    int memEndValue = 0;
+    WhereTerm *pStart, *pEnd;
+
+    assert( omitTable==0 );
+    pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0);
+    pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0);
+    if( bRev ){
+      pTerm = pStart;
+      pStart = pEnd;
+      pEnd = pTerm;
+    }
+    if( pStart ){
+      Expr *pX;             /* The expression that defines the start bound */
+      int r1, rTemp;        /* Registers for holding the start boundary */
+
+      /* The following constant maps TK_xx codes into corresponding 
+      ** seek opcodes.  It depends on a particular ordering of TK_xx
+      */
+      const u8 aMoveOp[] = {
+           /* TK_GT */  OP_SeekGt,
+           /* TK_LE */  OP_SeekLe,
+           /* TK_LT */  OP_SeekLt,
+           /* TK_GE */  OP_SeekGe
+      };
+      assert( TK_LE==TK_GT+1 );      /* Make sure the ordering.. */
+      assert( TK_LT==TK_GT+2 );      /*  ... of the TK_xx values... */
+      assert( TK_GE==TK_GT+3 );      /*  ... is correcct. */
+
+      pX = pStart->pExpr;
+      assert( pX!=0 );
+      assert( pStart->leftCursor==iCur );
+      r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
+      sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1);
+      VdbeComment((v, "pk"));
+      sqlite3ExprCacheAffinityChange(pParse, r1, 1);
+      sqlite3ReleaseTempReg(pParse, rTemp);
+      disableTerm(pLevel, pStart);
+    }else{
+      sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk);
+    }
+    if( pEnd ){
+      Expr *pX;
+      pX = pEnd->pExpr;
+      assert( pX!=0 );
+      assert( pEnd->leftCursor==iCur );
+      memEndValue = ++pParse->nMem;
+      sqlite3ExprCode(pParse, pX->pRight, memEndValue);
+      if( pX->op==TK_LT || pX->op==TK_GT ){
+        testOp = bRev ? OP_Le : OP_Ge;
+      }else{
+        testOp = bRev ? OP_Lt : OP_Gt;
+      }
+      disableTerm(pLevel, pEnd);
+    }
+    start = sqlite3VdbeCurrentAddr(v);
+    pLevel->op = bRev ? OP_Prev : OP_Next;
+    pLevel->p1 = iCur;
+    pLevel->p2 = start;
+    pLevel->p5 = (pStart==0 && pEnd==0) ?1:0;
+    codeRowSetEarly = regRowSet>=0 ? whereRowReadyForOutput(pWC) : 0;
+    if( codeRowSetEarly || testOp!=OP_Noop ){
+      int r1 = sqlite3GetTempReg(pParse);
+      sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
+      if( testOp!=OP_Noop ){
+        sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, r1);
+        sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL);
+      }
+      if( codeRowSetEarly ){
+        sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
+      }
+      sqlite3ReleaseTempReg(pParse, r1);