diff options
Diffstat (limited to 'sqlite3.c')
| -rw-r--r-- | sqlite3.c | 43675 |
1 files changed, 35678 insertions, 7997 deletions
diff --git a/sqlite3.c b/sqlite3.c index 1344938873b4..3b9d703f7691 100644 --- a/sqlite3.c +++ b/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.8.11.1. By combining all the individual C code files into this +** version 3.12.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -41,6 +41,14 @@ #define _SQLITEINT_H_ /* +** Make sure that rand_s() is available on Windows systems with MSVC 2005 +** or higher. +*/ +#if defined(_MSC_VER) && _MSC_VER>=1400 +# define _CRT_RAND_S +#endif + +/* ** Include the header file used to customize the compiler options for MSVC. ** This should be done first so that it can successfully prevent spurious ** compiler warnings due to subsequent content in this file and other files @@ -121,6 +129,9 @@ #else /* This is not VxWorks. */ #define OS_VXWORKS 0 +#define HAVE_FCHOWN 1 +#define HAVE_READLINK 1 +#define HAVE_LSTAT 1 #endif /* defined(_WRS_KERNEL) */ /************** End of vxworks.h *********************************************/ @@ -325,9 +336,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.11.1" -#define SQLITE_VERSION_NUMBER 3008011 -#define SQLITE_SOURCE_ID "2015-07-29 20:00:57 cf538e2783e468bbc25e7cb2a9ee64d3e0e80b2f" +#define SQLITE_VERSION "3.12.1" +#define SQLITE_VERSION_NUMBER 3012001 +#define SQLITE_SOURCE_ID "2016-04-08 15:09:49 fe7d3b75fe1bde41511b323925af8ae1b910bc4d" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -338,7 +349,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** <blockquote><pre> @@ -561,7 +572,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** from [sqlite3_malloc()] and passed back through the 5th parameter. ** To avoid memory leaks, the application should invoke [sqlite3_free()] ** on error message strings returned through the 5th parameter of -** of sqlite3_exec() after the error message string is no longer needed. +** sqlite3_exec() after the error message string is no longer needed. ** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors ** occur, then sqlite3_exec() sets the pointer in its 5th parameter to ** NULL before returning. @@ -588,7 +599,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** Restrictions: ** ** <ul> -** <li> The application must insure that the 1st parameter to sqlite3_exec() +** <li> The application must ensure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. ** <li> The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. @@ -691,6 +702,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) +#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) @@ -1006,8 +1019,13 @@ struct sqlite3_io_methods { ** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database -** connection. See the [sqlite3_file_control()] documentation for -** additional information. +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +** <li>[[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. @@ -1094,6 +1112,15 @@ struct sqlite3_io_methods { ** pointer in case this file-control is not implemented. This file-control ** is intended for diagnostic use only. ** +** <li>[[SQLITE_FCNTL_VFS_POINTER]] +** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level +** [VFSes] currently in use. ^(The argument X in +** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be +** of type "[sqlite3_vfs] **". This opcodes will set *X +** to a pointer to the top-level VFS.)^ +** ^When there are multiple VFS shims in the stack, this opcode finds the +** upper-most shim only. +** ** <li>[[SQLITE_FCNTL_PRAGMA]] ** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] ** file control is sent to the open [sqlite3_file] object corresponding @@ -1212,6 +1239,8 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_WAL_BLOCK 24 #define SQLITE_FCNTL_ZIPVFS 25 #define SQLITE_FCNTL_RBU 26 +#define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1424,7 +1453,7 @@ struct sqlite3_vfs { const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); /* ** The methods above are in versions 1 through 3 of the sqlite_vfs object. - ** New fields may be appended in figure versions. The iVersion + ** New fields may be appended in future versions. The iVersion ** value will increment whenever this happens. */ }; @@ -1580,9 +1609,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); ** applications and so this routine is usually not necessary. It is ** provided to support rare applications with unusual needs. ** -** The sqlite3_config() interface is not threadsafe. The application -** must insure that no other SQLite interfaces are invoked by other -** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** <b>The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running.</b> +** +** The sqlite3_config() interface ** may only be invoked prior to library initialization using ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before @@ -1809,29 +1840,34 @@ struct sqlite3_mem_methods { ** </dd> ** ** [[SQLITE_CONFIG_PAGECACHE]] <dt>SQLITE_CONFIG_PAGECACHE</dt> -** <dd> ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** <dd> ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool ** that SQLite can use for the database page cache with the default page ** cache implementation. -** This configuration should not be used if an application-define page -** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] -** configuration option. +** This configuration option is a no-op if an application-define page +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to -** 8-byte aligned -** memory, the size of each page buffer (sz), and the number of pages (N). +** 8-byte aligned memory (pMem), the size of each page cache line (sz), +** and the number of cache lines (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 65536) plus some extra bytes for each ** page header. ^The number of extra bytes needed by the page header -** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option -** to [sqlite3_config()]. +** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ]. ** ^It is harmless, apart from the wasted memory, -** for the sz parameter to be larger than necessary. The first -** argument should pointer to an 8-byte aligned block of memory that -** is at least sz*N bytes of memory, otherwise subsequent behavior is -** undefined. -** ^SQLite will use the memory provided by the first argument to satisfy its -** memory needs for the first N pages that it adds to cache. ^If additional -** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space.</dd> +** for the sz parameter to be larger than necessary. The pMem +** argument must be either a NULL pointer or a pointer to an 8-byte +** aligned block of memory of at least sz*N bytes, otherwise +** subsequent behavior is undefined. +** ^When pMem is not NULL, SQLite will strive to use the memory provided +** to satisfy page cache needs, falling back to [sqlite3_malloc()] if +** a page cache line is larger than sz bytes or if all of the pMem buffer +** is exhausted. +** ^If pMem is NULL and N is non-zero, then each database connection +** does an initial bulk allocation for page cache memory +** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or +** of -1024*N bytes if N is negative, . ^If additional +** page cache memory is needed beyond what is provided by the initial +** allocation, then SQLite goes to [sqlite3_malloc()] separately for each +** additional cache line. </dd> ** ** [[SQLITE_CONFIG_HEAP]] <dt>SQLITE_CONFIG_HEAP</dt> ** <dd> ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer @@ -2009,6 +2045,20 @@ struct sqlite3_mem_methods { ** is enabled (using the [PRAGMA threads] command) and the amount of content ** to be sorted exceeds the page size times the minimum of the ** [PRAGMA cache_size] setting and this value. +** +** [[SQLITE_CONFIG_STMTJRNL_SPILL]] +** <dt>SQLITE_CONFIG_STMTJRNL_SPILL +** <dd>^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which +** becomes the [statement journal] spill-to-disk threshold. +** [Statement journals] are held in memory until their size (in bytes) +** exceeds this threshold, at which point they are written to disk. +** Or if the threshold is -1, statement journals are always held +** exclusively in memory. +** Since many statement journals never become large, setting the spill +** threshold to a value such as 64KiB can greatly reduce the amount of +** I/O required to support statement rollback. +** The default value for this setting is controlled by the +** [SQLITE_STMTJRNL_SPILL] compile-time option. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2036,6 +2086,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ +#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2093,11 +2144,25 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back. </dd> ** +** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt> +** <dd> ^This option is used to enable or disable the two-argument +** version of the [fts3_tokenizer()] function which is part of the +** [FTS3] full-text search engine extension. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable fts3_tokenizer() or +** positive to enable fts3_tokenizer() or negative to leave the setting +** unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the new setting is not reported back. </dd> +** ** </dl> */ -#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ -#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ -#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ /* @@ -3587,7 +3652,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the ** [prepared statement] S has been stepped at least once using -** [sqlite3_step(S)] but has not run to completion and/or has not +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor ** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) ** interface returns false if S is a NULL pointer. If S is not a ** NULL pointer and is not a pointer to a valid [prepared statement] @@ -3840,7 +3906,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*, ** ** See also: [sqlite3_bind_blob|sqlite3_bind()], ** [sqlite3_bind_parameter_count()], and -** [sqlite3_bind_parameter_index()]. +** [sqlite3_bind_parameter_name()]. */ SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); @@ -4570,6 +4636,22 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value*); SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); /* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** SQLite makes no use of subtype itself. It merely passes the subtype +** from the result of one [application-defined SQL function] into the +** input of another. +*/ +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value*); + +/* ** CAPI3REF: Copy And Free SQL Values ** METHOD: sqlite3_value ** @@ -4583,8 +4665,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); ** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer ** then sqlite3_value_free(V) is a harmless no-op. */ -SQLITE_API SQLITE_EXPERIMENTAL sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value*); -SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context @@ -4868,6 +4950,21 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context*, sqlite3_va SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context*, int n); SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned int); + /* ** CAPI3REF: Define New Collating Sequences ** METHOD: sqlite3 @@ -5788,6 +5885,17 @@ struct sqlite3_module { ** ^Information about the ORDER BY clause is stored in aOrderBy[]. ** ^Each term of aOrderBy records a column of the ORDER BY clause. ** +** The colUsed field indicates which columns of the virtual table may be +** required by the current scan. Virtual table columns are numbered from +** zero in the order in which they appear within the CREATE TABLE statement +** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62), +** the corresponding bit is set within the colUsed mask if the column may be +** required by SQLite. If the table has at least 64 columns and any column +** to the right of the first 63 is required, then bit 63 of colUsed is also +** set. In other words, column iCol may be required if the expression +** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to +** non-zero. +** ** The [xBestIndex] method must fill aConstraintUsage[] with information ** about what parameters to pass to xFilter. ^If argvIndex>0 then ** the right-hand side of the corresponding aConstraint[] is evaluated @@ -5813,19 +5921,37 @@ struct sqlite3_module { ** ^The estimatedRows value is an estimate of the number of rows that ** will be returned by the strategy. ** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info ** structure for SQLite version 3.8.2. If a virtual table extension is ** used with an SQLite version earlier than 3.8.2, the results of attempting ** to read or write the estimatedRows field are undefined (but are likely ** to included crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a -** value greater than or equal to 3008002. +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for version 3.9.0. It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. */ struct sqlite3_index_info { /* Inputs */ int nConstraint; /* Number of entries in aConstraint */ struct sqlite3_index_constraint { - int iColumn; /* Column on left-hand side of constraint */ + int iColumn; /* Column constrained. -1 for ROWID */ unsigned char op; /* Constraint operator */ unsigned char usable; /* True if this constraint is usable */ int iTermOffset; /* Used internally - xBestIndex should ignore */ @@ -5847,9 +5973,18 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ /* Fields below are only available in SQLite 3.8.2 and later */ sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ + /* Fields below are only available in SQLite 3.10.0 and later */ + sqlite3_uint64 colUsed; /* Input: Mask of columns used by statement */ }; /* +** CAPI3REF: Virtual Table Scan Flags +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + +/* ** CAPI3REF: Virtual Table Constraint Operator Codes ** ** These macros defined the allowed values for the @@ -5857,12 +5992,15 @@ struct sqlite3_index_info { ** an operator that is part of a constraint term in the wHERE clause of ** a query that uses a [virtual table]. */ -#define SQLITE_INDEX_CONSTRAINT_EQ 2 -#define SQLITE_INDEX_CONSTRAINT_GT 4 -#define SQLITE_INDEX_CONSTRAINT_LE 8 -#define SQLITE_INDEX_CONSTRAINT_LT 16 -#define SQLITE_INDEX_CONSTRAINT_GE 32 -#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -6306,6 +6444,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); ** <li> SQLITE_MUTEX_STATIC_APP1 ** <li> SQLITE_MUTEX_STATIC_APP2 ** <li> SQLITE_MUTEX_STATIC_APP3 +** <li> SQLITE_MUTEX_STATIC_VFS1 +** <li> SQLITE_MUTEX_STATIC_VFS2 +** <li> SQLITE_MUTEX_STATIC_VFS3 ** </ul> ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6723,7 +6864,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( ** The value written into the *pCurrent parameter is undefined.</dd>)^ ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(<dt>SQLITE_STATUS_PARSER_STACK</dt> -** <dd>This parameter records the deepest parser stack. It is only +** <dd>The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].</dd>)^ ** </dl> ** @@ -7509,19 +7651,44 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing * -** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches -** the glob pattern P, and it returns non-zero if string X does not match -** the glob pattern P. ^The definition of glob pattern matching used in +** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if +** string X matches the [GLOB] pattern P. +** ^The definition of [GLOB] pattern matching used in ** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the -** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case -** sensitive. +** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function +** is case sensitive. ** ** Note that this routine returns zero on a match and non-zero if the strings ** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strlike()]. */ SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zStr); /* +** CAPI3REF: String LIKE Matching +* +** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if +** string X matches the [LIKE] pattern P with escape character E. +** ^The definition of [LIKE] pattern matching used in +** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" +** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without +** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. +** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case +** insensitive - equivalent upper and lower case ASCII characters match +** one another. +** +** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though +** only ASCII characters are case folded. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strglob()]. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); + +/* ** CAPI3REF: Error Logging Interface ** ** ^The [sqlite3_log()] interface writes a message into the [error log] @@ -7576,7 +7743,7 @@ SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...) ** previously registered write-ahead log callback. ^Note that the ** [sqlite3_wal_autocheckpoint()] interface and the ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** those overwrite any prior [sqlite3_wal_hook()] settings. +** overwrite any prior [sqlite3_wal_hook()] settings. */ SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( sqlite3*, @@ -7941,6 +8108,145 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( */ SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +/* +** CAPI3REF: Flush caches to disk mid-transaction +** +** ^If a write-transaction is open on [database connection] D when the +** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] +** interface flushes caches for all schemas - "main", "temp", and +** any [attached] databases. +** +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. ^If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. ^If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** ^If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite [error code] is returned to the caller immediately. +** +** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. +** +** ^This function does not set the database handle error code or message +** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3*); + +/* +** CAPI3REF: Low-level system error code +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most reason I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_system_errno(sqlite3*); + +/* +** CAPI3REF: Database Snapshot +** KEYWORDS: {snapshot} +** EXPERIMENTAL +** +** An instance of the snapshot object records the state of a [WAL mode] +** database for some specific point in history. +** +** In [WAL mode], multiple [database connections] that are open on the +** same database file can each be reading a different historical version +** of the database file. When a [database connection] begins a read +** transaction, that connection sees an unchanging copy of the database +** as it existed for the point in time when the transaction first started. +** Subsequent changes to the database from other connections are not seen +** by the reader until a new read transaction is started. +** +** The sqlite3_snapshot object records state information about an historical +** version of the database file so that it is possible to later open a new read +** transaction that sees that historical version of the database rather than +** the most recent version. +** +** The constructor for this object is [sqlite3_snapshot_get()]. The +** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer +** to an historical snapshot (if possible). The destructor for +** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. +*/ +typedef struct sqlite3_snapshot sqlite3_snapshot; + +/* +** CAPI3REF: Record A Database Snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a +** new [sqlite3_snapshot] object that records the current state of +** schema S in database connection D. ^On success, the +** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly +** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. +** ^If schema S of [database connection] D is not a [WAL mode] database +** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)] +** leaves the *P value unchanged and returns an appropriate [error code]. +** +** The [sqlite3_snapshot] object returned from a successful call to +** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] +** to avoid a memory leak. +** +** The [sqlite3_snapshot_get()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_get( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot **ppSnapshot +); + +/* +** CAPI3REF: Start a read transaction on an historical snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface attempts to move the +** read transaction that is currently open on schema S of +** [database connection] D so that it refers to historical [snapshot] P. +** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success +** or an appropriate [error code] if it fails. +** +** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be +** the first operation, apart from other sqlite3_snapshot_open() calls, +** following the [BEGIN] that starts a new read transaction. +** ^A [snapshot] will fail to open if it has been overwritten by a +** [checkpoint]. +** ^A [snapshot] will fail to open if the database connection D has not +** previously completed at least one read operation against the database +** file. (Hint: Run "[PRAGMA application_id]" against a newly opened +** database connection in order to make it ready to use snapshots.) +** +** The [sqlite3_snapshot_open()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_open( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot *pSnapshot +); + +/* +** CAPI3REF: Destroy a snapshot +** EXPERIMENTAL +** +** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. +** The application must eventually free every [sqlite3_snapshot] object +** using this routine to avoid a memory leak. +** +** The [sqlite3_snapshot_free()] interface is only available when the +** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_snapshot_free(sqlite3_snapshot*); /* ** Undo the hack that converts floating point types to integer for @@ -8072,6 +8378,584 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. The exception is if the table was created +** with the offsets=0 option specified. In this case *piOff is always +** set to -1. +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. For each row visited, the callback function +** passed as the fourth argument is invoked. The context and API objects +** passed to the callback function may be used to access the properties of +** each matched row. Invoking Api.xUserData() returns a copy of the pointer +** passed as the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and inititalize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +** </ul> +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +** <ol><li> By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +** <li> By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** <codeblock> +** ... MATCH 'first place'</codeblock> +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** <codeblock> +** ... MATCH '(first OR 1st) place'</codeblock> +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +** <li> By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +** </ol> +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** <codeblock> +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +**</codeblock> +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** <codeblock> +** ... MATCH '1s*'</codeblock> +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -8189,13 +9073,13 @@ struct sqlite3_rtree_query_info { ** The suggested maximum number of in-memory pages to use for ** the main database table and for temporary tables. ** -** IMPLEMENTATION-OF: R-31093-59126 The default suggested cache size -** is 2000 pages. +** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000, +** which means the cache size is limited to 2048000 bytes of memory. ** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be ** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options. */ #ifndef SQLITE_DEFAULT_CACHE_SIZE -# define SQLITE_DEFAULT_CACHE_SIZE 2000 +# define SQLITE_DEFAULT_CACHE_SIZE -2000 #endif /* @@ -8208,8 +9092,9 @@ struct sqlite3_rtree_query_info { /* ** The maximum number of attached databases. This must be between 0 -** and 62. The upper bound on 62 is because a 64-bit integer bitmap -** is used internally to track attached databases. +** and 125. The upper bound of 125 is because the attached databases are +** counted using a signed 8-bit integer which has a maximum value of 127 +** and we have to allow 2 extra counts for the "main" and "temp" databases. */ #ifndef SQLITE_MAX_ATTACHED # define SQLITE_MAX_ATTACHED 10 @@ -8244,7 +9129,7 @@ struct sqlite3_rtree_query_info { ** The default size of a database page. */ #ifndef SQLITE_DEFAULT_PAGE_SIZE -# define SQLITE_DEFAULT_PAGE_SIZE 1024 +# define SQLITE_DEFAULT_PAGE_SIZE 4096 #endif #if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE # undef SQLITE_DEFAULT_PAGE_SIZE @@ -8325,7 +9210,7 @@ struct sqlite3_rtree_query_info { ** to the next, so we have developed the following set of #if statements ** to generate appropriate macros for a wide range of compilers. ** -** The correct "ANSI" way to do this is to use the intptr_t type. +** The correct "ANSI" way to do this is to use the intptr_t type. ** Unfortunately, that typedef is not available on all compilers, or ** if it is available, it requires an #include of specific headers ** that vary from one machine to the next. @@ -8363,15 +9248,19 @@ struct sqlite3_rtree_query_info { /* ** Make sure that the compiler intrinsics we desire are enabled when -** compiling with an appropriate version of MSVC. -*/ -#if defined(_MSC_VER) && _MSC_VER>=1300 -# if !defined(_WIN32_WCE) -# include <intrin.h> -# pragma intrinsic(_byteswap_ushort) -# pragma intrinsic(_byteswap_ulong) -# else -# include <cmnintrin.h> +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1300 +# if !defined(_WIN32_WCE) +# include <intrin.h> +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_ReadWriteBarrier) +# else +# include <cmnintrin.h> +# endif # endif #endif @@ -8473,7 +9362,7 @@ struct sqlite3_rtree_query_info { ** is set. Thus NDEBUG becomes an opt-in rather than an opt-out ** feature. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif #if defined(NDEBUG) && defined(SQLITE_DEBUG) @@ -8488,7 +9377,7 @@ struct sqlite3_rtree_query_info { #endif /* -** The testcase() macro is used to aid in coverage testing. When +** The testcase() macro is used to aid in coverage testing. When ** doing coverage testing, the condition inside the argument to ** testcase() must be evaluated both true and false in order to ** get full branch coverage. The testcase() macro is inserted @@ -8534,7 +9423,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* -** The ALWAYS and NEVER macros surround boolean expressions which +** The ALWAYS and NEVER macros surround boolean expressions which ** are intended to always be true or false, respectively. Such ** expressions could be omitted from the code completely. But they ** are included in a few cases in order to enhance the resilience @@ -8560,6 +9449,21 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* +** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is +** defined. We need to defend against those failures when testing with +** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches +** during a normal build. The following macro can be used to disable tests +** that are always false except when SQLITE_TEST_REALLOC_STRESS is set. +*/ +#if defined(SQLITE_TEST_REALLOC_STRESS) +# define ONLY_IF_REALLOC_STRESS(X) (X) +#elif !defined(NDEBUG) +# define ONLY_IF_REALLOC_STRESS(X) ((X)?(assert(0),1):0) +#else +# define ONLY_IF_REALLOC_STRESS(X) (0) +#endif + +/* ** Declarations used for tracing the operating system interfaces. */ #if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ @@ -8586,6 +9490,13 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* +** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN +*/ +#ifdef SQLITE_OMIT_EXPLAIN +# undef SQLITE_ENABLE_EXPLAIN_COMMENTS +#endif + +/* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. @@ -8854,16 +9765,25 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_TO_REAL 147 #define TK_ISNOT 148 #define TK_END_OF_FILE 149 -#define TK_ILLEGAL 150 -#define TK_SPACE 151 -#define TK_UNCLOSED_STRING 152 -#define TK_FUNCTION 153 -#define TK_COLUMN 154 -#define TK_AGG_FUNCTION 155 -#define TK_AGG_COLUMN 156 -#define TK_UMINUS 157 -#define TK_UPLUS 158 -#define TK_REGISTER 159 +#define TK_UNCLOSED_STRING 150 +#define TK_FUNCTION 151 +#define TK_COLUMN 152 +#define TK_AGG_FUNCTION 153 +#define TK_AGG_COLUMN 154 +#define TK_UMINUS 155 +#define TK_UPLUS 156 +#define TK_REGISTER 157 +#define TK_ASTERISK 158 +#define TK_SPAN 159 +#define TK_SPACE 160 +#define TK_ILLEGAL 161 + +/* The token codes above must all fit in 8 bits */ +#define TKFLG_MASK 0xff + +/* Flags that can be added to a token code when it is not +** being stored in a u8: */ +#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -8895,7 +9815,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0 -** afterward. Having this macro allows us to cause the C compiler +** afterward. Having this macro allows us to cause the C compiler ** to omit code used by TEMP tables without messy #ifndef statements. */ #ifdef SQLITE_OMIT_TEMPDB @@ -8934,7 +9854,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if -** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it ** to zero. */ #if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 @@ -8962,7 +9882,6 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define SQLITE_DEFAULT_PCACHE_INITSZ 100 #endif - /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -8974,8 +9893,12 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** Macros to compute minimum and maximum of two numbers. */ -#define MIN(A,B) ((A)<(B)?(A):(B)) -#define MAX(A,B) ((A)>(B)?(A):(B)) +#ifndef MIN +# define MIN(A,B) ((A)<(B)?(A):(B)) +#endif +#ifndef MAX +# define MAX(A,B) ((A)>(B)?(A):(B)) +#endif /* ** Swap two objects of type TYPE. @@ -9083,7 +10006,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** 4 -> 20 1000 -> 99 1048576 -> 200 ** 10 -> 33 1024 -> 100 4294967296 -> 320 ** -** The LogEst can be negative to indicate fractional values. +** The LogEst can be negative to indicate fractional values. ** Examples: ** ** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 @@ -9104,6 +10027,27 @@ typedef INT16_TYPE LogEst; # endif #endif +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; +#endif + +/* +** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to +** something between S (inclusive) and E (exclusive). +** +** In other words, S is a buffer and E is a pointer to the first byte after +** the end of buffer S. This macro returns true if P points to something +** contained within the buffer S. +*/ +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) + + /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. @@ -9113,11 +10057,6 @@ typedef INT16_TYPE LogEst; ** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined ** at run-time. */ -#ifdef SQLITE_AMALGAMATION -SQLITE_PRIVATE const int sqlite3one = 1; -#else -SQLITE_PRIVATE const int sqlite3one; -#endif #if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ @@ -9135,6 +10074,11 @@ SQLITE_PRIVATE const int sqlite3one; # define SQLITE_UTF16NATIVE SQLITE_UTF16BE #endif #if !defined(SQLITE_BYTEORDER) +# ifdef SQLITE_AMALGAMATION + const int sqlite3one = 1; +# else + extern const int sqlite3one; +# endif # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) @@ -9149,7 +10093,7 @@ SQLITE_PRIVATE const int sqlite3one; #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -/* +/* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ @@ -9188,10 +10132,6 @@ SQLITE_PRIVATE const int sqlite3one; */ #ifdef __APPLE__ # include <TargetConditionals.h> -# if TARGET_OS_IPHONE -# undef SQLITE_MAX_MMAP_SIZE -# define SQLITE_MAX_MMAP_SIZE 0 -# endif #endif #ifndef SQLITE_MAX_MMAP_SIZE # if defined(__linux__) \ @@ -9247,7 +10187,7 @@ SQLITE_PRIVATE const int sqlite3one; /* ** An instance of the following structure is used to store the busy-handler -** callback for a given sqlite handle. +** callback for a given sqlite handle. ** ** The sqlite.busyHandler member of the sqlite struct contains the busy ** callback for the database handle. Each pager opened via the sqlite @@ -9292,9 +10232,9 @@ struct BusyHandler { /* ** The following value as a destructor means to use sqlite3DbFree(). -** The sqlite3DbFree() routine requires two parameters instead of the -** one parameter that destructors normally want. So we have to introduce -** this magic value that the code knows to handle differently. Any +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ @@ -9321,16 +10261,16 @@ struct BusyHandler { SQLITE_API int SQLITE_STDCALL sqlite3_wsd_init(int N, int J); SQLITE_API void *SQLITE_STDCALL sqlite3_wsd_find(void *K, int L); #else - #define SQLITE_WSD + #define SQLITE_WSD #define GLOBAL(t,v) v #define sqlite3GlobalConfig sqlite3Config #endif /* ** The following macros are used to suppress compiler warnings and to -** make it clear to human readers when a function parameter is deliberately +** make it clear to human readers when a function parameter is deliberately ** left unused within the body of a function. This usually happens when -** a function is called via a function pointer. For example the +** a function is called via a function pointer. For example the ** implementation of an SQL aggregate step callback may not use the ** parameter indicating the number of arguments passed to the aggregate, ** if it knows that this is enforced elsewhere. @@ -9396,7 +10336,7 @@ typedef struct WhereInfo WhereInfo; typedef struct With With; /* -** Defer sourcing vdbe.h and btree.h until after the "u8" and +** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ @@ -9467,11 +10407,11 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); #if SQLITE_MAX_MMAP_SIZE>0 SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); #endif SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); -SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); @@ -9554,8 +10494,37 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* -** Values that may be OR'd together to form the second argument of an -** sqlite3BtreeCursorHints() call. +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extentions that use +** the SQLite parser and code generator but substitute their own storage +** engine. +*/ +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): ** ** The BTREE_BULKLOAD flag is set on index cursors when the index is going ** to be filled with content that is already in sorted order. @@ -9569,6 +10538,32 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ #define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just +** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and then delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +** +** The BTREE_FORDELETE flag is an optimization hint. It is not used by +** by this, the native b-tree engine of SQLite, but it is available to +** alternative storage engines that might be substituted in place of this +** b-tree system. For alternative storage engines in which a delete of +** the main table row automatically deletes corresponding index rows, +** the FORDELETE flag hint allows those alternative storage engines to +** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK +** and DELETE operations as no-ops, and any READ operation against a +** FORDELETE cursor may return a null row: 0x01 0x00. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ + SQLITE_PRIVATE int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ int iTable, /* Index of root page */ @@ -9578,6 +10573,10 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( ); SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( @@ -9589,7 +10588,12 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ); SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); + +/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ +#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ +#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ + SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, int nZero, int bias, int seekResult); @@ -9612,10 +10616,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); -SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); -#ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); -#endif SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); @@ -9644,15 +10645,17 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); #ifndef SQLITE_OMIT_SHARED_CACHE SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); #else # define sqlite3BtreeEnter(X) # define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeEnterCursor(X) #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); #ifndef NDEBUG @@ -9663,9 +10666,7 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif #else -# define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeLeave(X) -# define sqlite3BtreeEnterCursor(X) # define sqlite3BtreeLeaveCursor(X) # define sqlite3BtreeLeaveAll(X) @@ -9743,6 +10744,9 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Expr *pExpr; /* Used when p4type is P4_EXPR */ +#endif int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -9793,6 +10797,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ #define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-7) /* P4 is a pointer to an Expr tree */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ #define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ @@ -9845,39 +10850,40 @@ typedef struct VdbeOpList VdbeOpList; /************** Include opcodes.h in the middle of vdbe.h ********************/ /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ -/* See the mkopcodeh.awk script for details */ -#define OP_Savepoint 1 -#define OP_AutoCommit 2 -#define OP_Transaction 3 -#define OP_SorterNext 4 -#define OP_PrevIfOpen 5 -#define OP_NextIfOpen 6 -#define OP_Prev 7 -#define OP_Next 8 -#define OP_Checkpoint 9 -#define OP_JournalMode 10 -#define OP_Vacuum 11 -#define OP_VFilter 12 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 13 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 14 -#define OP_Gosub 15 -#define OP_Return 16 -#define OP_InitCoroutine 17 -#define OP_EndCoroutine 18 +/* See the tool/mkopcodeh.tcl script for details */ +#define OP_Savepoint 0 +#define OP_AutoCommit 1 +#define OP_Transaction 2 +#define OP_SorterNext 3 +#define OP_PrevIfOpen 4 +#define OP_NextIfOpen 5 +#define OP_Prev 6 +#define OP_Next 7 +#define OP_Checkpoint 8 +#define OP_JournalMode 9 +#define OP_Vacuum 10 +#define OP_VFilter 11 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 13 +#define OP_Gosub 14 +#define OP_Return 15 +#define OP_InitCoroutine 16 +#define OP_EndCoroutine 17 +#define OP_Yield 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Yield 20 -#define OP_HaltIfNull 21 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 22 -#define OP_Integer 23 /* synopsis: r[P2]=P1 */ -#define OP_Int64 24 /* synopsis: r[P2]=P4 */ -#define OP_String 25 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 26 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 27 /* synopsis: r[P1]=NULL */ -#define OP_Blob 28 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 29 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 30 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 31 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 32 /* synopsis: r[P2]=r[P1] */ +#define OP_HaltIfNull 20 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 21 +#define OP_Integer 22 /* synopsis: r[P2]=P1 */ +#define OP_Int64 23 /* synopsis: r[P2]=P4 */ +#define OP_String 24 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 25 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 26 /* synopsis: r[P1]=NULL */ +#define OP_Blob 27 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 28 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 29 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 30 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 31 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 32 /* synopsis: r[P2]=r[P1] */ #define OP_ResultRow 33 /* synopsis: output=r[P1@P2] */ #define OP_CollSeq 34 #define OP_Function0 35 /* synopsis: r[P3]=func(r[P2@P5]) */ @@ -9912,15 +10918,15 @@ typedef struct VdbeOpList VdbeOpList; #define OP_SeekLE 64 /* synopsis: key=r[P3@P4] */ #define OP_SeekGE 65 /* synopsis: key=r[P3@P4] */ #define OP_SeekGT 66 /* synopsis: key=r[P3@P4] */ -#define OP_Seek 67 /* synopsis: intkey=r[P2] */ -#define OP_NoConflict 68 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 69 /* synopsis: key=r[P3@P4] */ -#define OP_Found 70 /* synopsis: key=r[P3@P4] */ +#define OP_NoConflict 67 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 68 /* synopsis: key=r[P3@P4] */ +#define OP_Found 69 /* synopsis: key=r[P3@P4] */ +#define OP_NotExists 70 /* synopsis: intkey=r[P3] */ #define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_NotExists 73 /* synopsis: intkey=r[P3] */ -#define OP_Sequence 74 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 75 /* synopsis: r[P2]=rowid */ +#define OP_Sequence 73 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 74 /* synopsis: r[P2]=rowid */ +#define OP_Insert 75 /* synopsis: intkey=r[P3] data=r[P2] */ #define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ @@ -9929,7 +10935,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ #define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]<r[P3] goto P2 */ #define OP_Ge 83 /* same as TK_GE, synopsis: if r[P1]>=r[P3] goto P2 */ -#define OP_Insert 84 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_InsertInt 84 /* synopsis: intkey=P3 data=r[P2] */ #define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ @@ -9940,24 +10946,24 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ #define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ #define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_InsertInt 95 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Delete 95 #define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ #define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_Delete 98 -#define OP_ResetCount 99 -#define OP_SorterCompare 100 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 101 /* synopsis: r[P2]=data */ -#define OP_RowKey 102 /* synopsis: r[P2]=key */ -#define OP_RowData 103 /* synopsis: r[P2]=data */ -#define OP_Rowid 104 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 105 -#define OP_Last 106 -#define OP_SorterSort 107 -#define OP_Sort 108 -#define OP_Rewind 109 -#define OP_SorterInsert 110 -#define OP_IdxInsert 111 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 112 /* synopsis: key=r[P2@P3] */ +#define OP_ResetCount 98 +#define OP_SorterCompare 99 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 100 /* synopsis: r[P2]=data */ +#define OP_RowKey 101 /* synopsis: r[P2]=key */ +#define OP_RowData 102 /* synopsis: r[P2]=data */ +#define OP_Rowid 103 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 104 +#define OP_Last 105 +#define OP_SorterSort 106 +#define OP_Sort 107 +#define OP_Rewind 108 +#define OP_SorterInsert 109 +#define OP_IdxInsert 110 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 111 /* synopsis: key=r[P2@P3] */ +#define OP_Seek 112 /* synopsis: Move P3 to P1.rowid */ #define OP_IdxRowid 113 /* synopsis: r[P2]=rowid */ #define OP_IdxLE 114 /* synopsis: key=r[P3@P4] */ #define OP_IdxGT 115 /* synopsis: key=r[P3@P4] */ @@ -9983,9 +10989,9 @@ typedef struct VdbeOpList VdbeOpList; #define OP_FkCounter 135 /* synopsis: fkctr[P1]+=P2 */ #define OP_FkIfZero 136 /* synopsis: if fkctr[P1]==0 goto P2 */ #define OP_MemMax 137 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_IfPos 138 /* synopsis: if r[P1]>0 goto P2 */ -#define OP_IfNeg 139 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ -#define OP_IfNotZero 140 /* synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 */ +#define OP_IfPos 138 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_OffsetLimit 139 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_IfNotZero 140 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ #define OP_DecrJumpZero 141 /* synopsis: if (--r[P1])==0 goto P2 */ #define OP_JumpZeroIncr 142 /* synopsis: if (r[P1]++)==0 ) goto P2 */ #define OP_AggStep0 143 /* synopsis: accum=r[P3] step(r[P2@P5]) */ @@ -10004,42 +11010,42 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Pagecount 156 #define OP_MaxPgcnt 157 #define OP_Init 158 /* synopsis: Start at P2 */ -#define OP_Noop 159 -#define OP_Explain 160 - +#define OP_CursorHint 159 +#define OP_Noop 160 +#define OP_Explain 161 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c ** are encoded into bitvectors as follows: */ -#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ -#define OPFLG_IN1 0x0002 /* in1: P1 is an input */ -#define OPFLG_IN2 0x0004 /* in2: P2 is an input */ -#define OPFLG_IN3 0x0008 /* in3: P3 is an input */ -#define OPFLG_OUT2 0x0010 /* out2: P2 is an output */ -#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */ +#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x02 /* in1: P1 is an input */ +#define OPFLG_IN2 0x04 /* in2: P2 is an input */ +#define OPFLG_IN3 0x08 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,\ -/* 8 */ 0x01, 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01,\ -/* 16 */ 0x02, 0x01, 0x02, 0x12, 0x03, 0x08, 0x00, 0x10,\ -/* 24 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ +/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x02,\ +/* 16 */ 0x01, 0x02, 0x03, 0x12, 0x08, 0x00, 0x10, 0x10,\ +/* 24 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ /* 32 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x02,\ /* 40 */ 0x02, 0x00, 0x00, 0x01, 0x01, 0x03, 0x03, 0x00,\ -/* 48 */ 0x00, 0x00, 0x10, 0x10, 0x08, 0x00, 0x00, 0x00,\ +/* 48 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,\ /* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09,\ -/* 64 */ 0x09, 0x09, 0x09, 0x04, 0x09, 0x09, 0x09, 0x26,\ -/* 72 */ 0x26, 0x09, 0x10, 0x10, 0x03, 0x03, 0x0b, 0x0b,\ +/* 64 */ 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x26,\ +/* 72 */ 0x26, 0x10, 0x10, 0x00, 0x03, 0x03, 0x0b, 0x0b,\ /* 80 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x00, 0x26, 0x26, 0x26,\ /* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ -/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 104 */ 0x10, 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04,\ +/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ +/* 104 */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x00,\ /* 112 */ 0x00, 0x10, 0x01, 0x01, 0x01, 0x01, 0x10, 0x00,\ /* 120 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ /* 128 */ 0x00, 0x06, 0x23, 0x0b, 0x01, 0x10, 0x10, 0x00,\ -/* 136 */ 0x01, 0x04, 0x03, 0x03, 0x03, 0x03, 0x03, 0x00,\ +/* 136 */ 0x01, 0x04, 0x03, 0x1a, 0x03, 0x03, 0x03, 0x00,\ /* 144 */ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,\ /* 152 */ 0x00, 0x00, 0x01, 0x00, 0x10, 0x10, 0x01, 0x00,\ -/* 160 */ 0x00,} +/* 160 */ 0x00, 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -10052,18 +11058,28 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +#else +# define sqlite3VdbeVerifyNoMallocRequired(A,B) +#endif +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); @@ -10071,6 +11087,7 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); @@ -10262,7 +11279,7 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* -** Flags that make up the mask passed to sqlite3PagerAcquire(). +** Flags that make up the mask passed to sqlite3PagerGet(). */ #define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ #define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ @@ -10273,11 +11290,12 @@ typedef struct PgHdr DbPage; #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ #define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ -#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ -#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ -#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ -#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ -#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ +#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ +#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ +#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ /* ** The remainder of this file contains the declarations of the functions @@ -10301,8 +11319,12 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); @@ -10312,10 +11334,10 @@ SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +SQLITE_PRIVATE int sqlite3PagerFlush(Pager*); /* Functions used to obtain and release page references. */ -SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); -#define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) +SQLITE_PRIVATE int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); @@ -10347,6 +11369,10 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +# ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); +# endif #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -10361,10 +11387,10 @@ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); #endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); -SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); -SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); @@ -10458,6 +11484,8 @@ struct PgHdr { #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ #define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_WAL_APPEND 0x080 /* Appended to wal file */ + /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); SQLITE_PRIVATE void sqlite3PcacheShutdown(void); @@ -10549,6 +11577,13 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); #endif +/* Set or get the suggested spill-size for the specified pager-cache. +** +** The spill-size is the minimum number of pages in cache before the cache +** will attempt to spill dirty pages by calling xStress. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *, int); + /* Free up as much memory as possible from the page cache */ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); @@ -10571,7 +11606,6 @@ SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ - /************** Include os.h in the middle of sqliteInt.h ********************/ /************** Begin file os.h **********************************************/ /* @@ -10834,6 +11868,7 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* @@ -10923,6 +11958,36 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); /************** End of mutex.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* The SQLITE_EXTRA_DURABLE compile-time option used to set the default +** synchronous setting to EXTRA. It is no longer supported. +*/ +#ifdef SQLITE_EXTRA_DURABLE +# warning Use SQLITE_DEFAULT_SYNCHRONOUS=3 instead of SQLITE_EXTRA_DURABLE +# define SQLITE_DEFAULT_SYNCHRONOUS 3 +#endif + +/* +** Default synchronous levels. +** +** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. +** +** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS +** OFF 1 0 +** NORMAL 2 1 +** FULL 3 2 +** EXTRA 4 3 +** +** The "PRAGMA synchronous" statement also uses the zero-based numbers. +** In other words, the zero-based numbers are used for all external interfaces +** and the one-based values are used internally. +*/ +#ifndef SQLITE_DEFAULT_SYNCHRONOUS +# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1) +#endif +#ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS +# define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS +#endif /* ** Each database file to be accessed by the system is an instance @@ -10935,6 +12000,7 @@ struct Db { char *zName; /* Name of this database */ Btree *pBt; /* The B*Tree structure for this database file */ u8 safety_level; /* How aggressive at syncing data to disk */ + u8 bSyncSet; /* True if "PRAGMA synchronous=N" has been run */ Schema *pSchema; /* Pointer to database schema (possibly shared) */ }; @@ -10945,7 +12011,7 @@ struct Db { ** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. -** +** ** Schema objects are automatically deallocated when the last Btree that ** references them is destroyed. The TEMP Schema is manually freed by ** sqlite3_close(). @@ -10970,7 +12036,7 @@ struct Schema { }; /* -** These macros can be used to test, set, or clear bits in the +** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ #define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) @@ -11019,8 +12085,8 @@ struct Schema { ** lookaside allocations are not used to construct the schema objects. */ struct Lookaside { + u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ - u8 bEnabled; /* False to disable new lookaside allocations */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ int nOut; /* Number of buffers currently checked out */ int mxOut; /* Highwater mark for nOut */ @@ -11034,13 +12100,15 @@ struct LookasideSlot { }; /* -** A hash table for function definitions. +** A hash table for built-in function definitions. (Application-defined +** functions use a regular table table from hash.h.) ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.pHash chain. +** Collisions are on the FuncDef.u.pHash chain. */ +#define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { - FuncDef *a[23]; /* Hash table for functions */ + FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ }; #ifdef SQLITE_USER_AUTHENTICATION @@ -11098,11 +12166,13 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + int iSysErrno; /* Errno value from last system error */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ + u8 bBenignMalloc; /* Do not require OOMs if true */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ @@ -11132,9 +12202,9 @@ struct sqlite3 { void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ - void *pCommitArg; /* Argument to xCommitCallback() */ + void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ - void *pRollbackArg; /* Argument to xRollbackCallback() */ + void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); @@ -11167,7 +12237,7 @@ struct sqlite3 { VTable **aVTrans; /* Virtual tables with open transactions */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif - FuncDefHash aFunc; /* Hash table of connection functions */ + Hash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ @@ -11179,8 +12249,8 @@ struct sqlite3 { i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - /* The following variables are all protected by the STATIC_MASTER - ** mutex, not by sqlite3.mutex. They are used by code in notify.c. + /* The following variables are all protected by the STATIC_MASTER + ** mutex, not by sqlite3.mutex. They are used by code in notify.c. ** ** When X.pUnlockConnection==Y, that means that X is waiting for Y to ** unlock so that it can proceed. @@ -11211,10 +12281,10 @@ struct sqlite3 { */ #define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ #define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ -#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ -#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ -#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ #define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ @@ -11241,6 +12311,7 @@ struct sqlite3 { #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ #define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ #define SQLITE_CellSizeCk 0x10000000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x20000000 /* Enable fts3_tokenizer(2) */ /* @@ -11260,6 +12331,7 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -11293,28 +12365,33 @@ struct sqlite3 { /* ** Each SQL function is defined by an instance of the following -** structure. A pointer to this structure is stored in the sqlite.aFunc -** hash table. When multiple functions have the same name, the hash table -** points to a linked list of these structures. +** structure. For global built-in functions (ex: substr(), max(), count()) +** a pointer to this structure is held in the sqlite3BuiltinFunctions object. +** For per-connection application-defined functions, a pointer to this +** structure is held in the db->aHash hash table. +** +** The u.pHash field is used by the global built-ins. The u.pDestructor +** field is used by per-connection app-def functions. */ struct FuncDef { - i16 nArg; /* Number of arguments. -1 means unlimited */ + i8 nArg; /* Number of arguments. -1 means unlimited */ u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ - void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ - void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ - char *zName; /* SQL name of the function. */ - FuncDef *pHash; /* Next with a different name but the same hash */ - FuncDestructor *pDestructor; /* Reference counted destructor function */ + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ + void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + const char *zName; /* SQL name of the function. */ + union { + FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ + } u; }; /* ** This structure encapsulates a user-function destructor callback (as ** configured using create_function_v2()) and a reference counter. When ** create_function_v2() is called to create a function with a destructor, -** a single object of this type is allocated. FuncDestructor.nRef is set to +** a single object of this type is allocated. FuncDestructor.nRef is set to ** the number of FuncDef objects created (either 1 or 3, depending on whether ** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor ** member of each of the new FuncDef objects is set to point to the allocated @@ -11332,36 +12409,45 @@ struct FuncDestructor { /* ** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF -** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. And +** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC. There ** are assert() statements in the code to verify this. */ -#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ -#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are ** used to create the initializers for the FuncDef structures. ** ** FUNCTION(zName, nArg, iArg, bNC, xFunc) -** Used to create a scalar function definition of a function zName +** Used to create a scalar function definition of a function zName ** implemented by C function xFunc that accepts nArg arguments. The ** value passed as iArg is cast to a (void*) and made available -** as the user-data (sqlite3_user_data()) for the function. If +** as the user-data (sqlite3_user_data()) for the function. If ** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. ** ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -11369,8 +12455,8 @@ struct FuncDestructor { ** FUNCTION(). ** ** LIKEFUNC(zName, nArg, pArg, flags) -** Used to create a scalar function definition of a function zName -** that accepts nArg arguments and is implemented by a call to C +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C ** function likeFunc. Argument pArg is cast to a (void *) and made ** available as the function user-data (sqlite3_user_data()). The ** FuncDef.flags variable is set to the value passed as the flags @@ -11378,25 +12464,28 @@ struct FuncDestructor { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, #zName, 0, 0} + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, #zName, } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} + (void *)arg, 0, likeFunc, 0, #zName, {0} } #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} /* ** All current savepoints are stored in a linked list starting at @@ -11430,6 +12519,7 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -11437,14 +12527,12 @@ struct Module { ** of this structure. */ struct Column { - char *zName; /* Name of this column */ + char *zName; /* Name of this column, \000, then the type */ Expr *pDflt; /* Default value of this column */ - char *zDflt; /* Original text of the default value */ - char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ - u8 szEst; /* Estimated size of this column. INT==1 */ + u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; @@ -11452,6 +12540,7 @@ struct Column { */ #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -11475,13 +12564,14 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. ** ** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve -** the speed a little by numbering the values consecutively. +** the speed a little by numbering the values consecutively. ** ** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and @@ -11500,7 +12590,7 @@ struct CollSeq { /* ** The SQLITE_AFF_MASK values masks off the significant bits of an -** affinity value. +** affinity value. */ #define SQLITE_AFF_MASK 0x47 @@ -11520,20 +12610,20 @@ struct CollSeq { /* ** An object of this type is created for each virtual table present in -** the database schema. +** the database schema. ** ** If the database schema is shared, then there is one instance of this ** structure for each database connection (sqlite3*) that uses the shared ** schema. This is because each database connection requires its own unique -** instance of the sqlite3_vtab* handle used to access the virtual table -** implementation. sqlite3_vtab* handles can not be shared between -** database connections, even when the rest of the in-memory database +** instance of the sqlite3_vtab* handle used to access the virtual table +** implementation. sqlite3_vtab* handles can not be shared between +** database connections, even when the rest of the in-memory database ** schema is shared, as the implementation often stores the database ** connection handle passed to it via the xConnect() or xCreate() method ** during initialization internally. This database connection handle may -** then be used by the virtual table implementation to access real tables -** within the database. So that they appear as part of the callers -** transaction, these accesses need to be made via the same database +** then be used by the virtual table implementation to access real tables +** within the database. So that they appear as part of the callers +** transaction, these accesses need to be made via the same database ** connection as that used to execute SQL operations on the virtual table. ** ** All VTable objects that correspond to a single table in a shared @@ -11545,19 +12635,19 @@ struct CollSeq { ** sqlite3_vtab* handle in the compiled query. ** ** When an in-memory Table object is deleted (for example when the -** schema is being reloaded for some reason), the VTable objects are not -** deleted and the sqlite3_vtab* handles are not xDisconnect()ed +** schema is being reloaded for some reason), the VTable objects are not +** deleted and the sqlite3_vtab* handles are not xDisconnect()ed ** immediately. Instead, they are moved from the Table.pVTable list to ** another linked list headed by the sqlite3.pDisconnect member of the -** corresponding sqlite3 structure. They are then deleted/xDisconnected +** corresponding sqlite3 structure. They are then deleted/xDisconnected ** next time a statement is prepared using said sqlite3*. This is done ** to avoid deadlock issues involving multiple sqlite3.mutex mutexes. ** Refer to comments above function sqlite3VtabUnlockList() for an ** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect ** list without holding the corresponding sqlite3.mutex mutex. ** -** The memory for objects of this type is always allocated by -** sqlite3DbMalloc(), using the connection handle stored in VTable.db as +** The memory for objects of this type is always allocated by +** sqlite3DbMalloc(), using the connection handle stored in VTable.db as ** the first argument. */ struct VTable { @@ -11581,9 +12671,8 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ -#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ -#endif + /* ... also used as column name list in a VIEW */ int tnum; /* Root BTree page for this table */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ @@ -11600,7 +12689,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* Text of all module args. [0] is module name */ + char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -11611,7 +12700,7 @@ struct Table { /* ** Allowed values for Table.tabFlags. ** -** TF_OOOHidden applies to virtual tables that have hidden columns that are +** TF_OOOHidden applies to tables or view that have hidden columns that are ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, ** the TF_OOOHidden attribute would apply in this case. Such tables require @@ -11634,12 +12723,28 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 -# define IsHiddenColumn(X) 0 #endif +/* +** Macros to determine if a column is hidden. IsOrdinaryHiddenColumn() +** only works for non-virtual tables (ordinary tables and views) and is +** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined. The +** IsHiddenColumn() macro is general purpose. +*/ +#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +#elif !defined(SQLITE_OMIT_VIRTUALTABLE) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) 0 +#else +# define IsHiddenColumn(X) 0 +# define IsOrdinaryHiddenColumn(X) 0 +#endif + + /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) @@ -11710,7 +12815,7 @@ struct FKey { ** key is set to NULL. CASCADE means that a DELETE or UPDATE of the ** referenced table row is propagated into the row that holds the ** foreign key. -** +** ** The following symbolic values are used to record which type ** of action to take. */ @@ -11731,7 +12836,7 @@ struct FKey { /* ** An instance of the following structure is passed as the first -** argument to sqlite3VdbeKeyCompare and is used to control the +** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** ** Note that aSortOrder[] and aColl[] have nField+1 slots. There @@ -11749,9 +12854,8 @@ struct KeyInfo { }; /* -** An instance of the following structure holds information about a -** single index record that has already been parsed out into individual -** values. +** This object holds a record which has been parsed out into individual +** fields, for the purposes of doing a comparison. ** ** A record is an object that contains one or more fields of data. ** Records are used to store the content of a table row and to store @@ -11759,20 +12863,40 @@ struct KeyInfo { ** the OP_MakeRecord opcode of the VDBE and is disassembled by the ** OP_Column opcode. ** -** This structure holds a record that has already been disassembled -** into its constituent fields. -** -** The r1 and r2 member variables are only used by the optimized comparison -** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). +** An instance of this object serves as a "key" for doing a search on +** an index b+tree. The goal of the search is to find the entry that +** is closed to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by +** pKeyInfo->nField. +** +** The r1 and r2 fields are the values to return if this key is less than +** or greater than a key in the btree, respectively. These are normally +** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree +** is in DESC order. +** +** The key comparison functions actually return default_rc when they find +** an equals comparison. default_rc can be -1, 0, or +1. If there are +** multiple entries in the b-tree with the same key (when only looking +** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** cause the search to find the last match, or +1 to cause the search to +** find the first match. +** +** The key comparison functions will set eqSeen to true if they ever +** get and equal results when comparing this structure to a b-tree record. +** When default_rc!=0, the search might end up on the record immediately +** before the first match or immediately after the last match. The +** eqSeen field will indicate whether or not an exact match exists in the +** b-tree. */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ + Mem *aMem; /* Values */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ - Mem *aMem; /* Values */ - int r1; /* Value to return if (lhs > rhs) */ - int r2; /* Value to return if (rhs < lhs) */ + i8 r1; /* Value to return if (lhs > rhs) */ + i8 r2; /* Value to return if (rhs < lhs) */ + u8 eqSeen; /* True if an equality comparison has been seen */ }; @@ -11790,7 +12914,7 @@ struct UnpackedRecord { ** In the Table structure describing Ex1, nCol==3 because there are ** three columns in the table. In the Index structure describing ** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed. -** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the +** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the ** first column to be indexed (c3) has an index of 2 in Ex1.aCol[]. ** The second column to be indexed (c1) has an index of 0 in ** Ex1.aCol[], hence Ex2.aiColumn[1]==0. @@ -11798,7 +12922,7 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution +** and the value of Index.onError indicate the which conflict resolution ** algorithm to employ whenever an attempt is made to insert a non-unique ** element. ** @@ -11819,8 +12943,9 @@ struct Index { Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ + const char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -11855,8 +12980,14 @@ struct Index { /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -11951,9 +13082,9 @@ typedef int ynVar; ** to represent the greater-than-or-equal-to operator in the expression ** tree. ** -** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, +** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, ** or TK_STRING), then Expr.token contains the text of the SQL literal. If -** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** the expression is a variable (TK_VARIABLE), then Expr.token contains the ** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), ** then Expr.token contains the name of the function. ** @@ -11964,7 +13095,7 @@ typedef int ynVar; ** a CASE expression or an IN expression of the form "<lhs> IN (<y>, <z>...)". ** Expr.x.pSelect is used if the expression is a sub-select or an expression of ** the form "<lhs> IN (SELECT ...)". If the EP_xIsSelect bit is set in the -** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is +** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is ** valid. ** ** An expression of the form ID or ID.ID refers to a column in a table. @@ -11975,8 +13106,8 @@ typedef int ynVar; ** value is also stored in the Expr.iAgg column in the aggregate so that ** it can be accessed after all aggregates are computed. ** -** If the expression is an unbound variable marker (a question mark -** character '?' in the original SQL) then the Expr.iTable holds the index +** If the expression is an unbound variable marker (a question mark +** character '?' in the original SQL) then the Expr.iTable holds the index ** number for that variable. ** ** If the expression is a subquery then Expr.iColumn holds an integer @@ -12015,7 +13146,7 @@ struct Expr { /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no ** space is allocated for the fields below this point. An attempt to - ** access them will result in a segfault or malfunction. + ** access them will result in a segfault or malfunction. *********************************************************************/ Expr *pLeft; /* Left subnode */ @@ -12070,9 +13201,10 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ /* ** Combinations of two or more EP_* flags @@ -12080,7 +13212,7 @@ struct Expr { #define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ /* -** These macros can be used to test, set, or clear bits in the +** These macros can be used to test, set, or clear bits in the ** Expr.flags field. */ #define ExprHasProperty(E,P) (((E)->flags&(P))!=0) @@ -12099,8 +13231,8 @@ struct Expr { #endif /* -** Macros to determine the number of bytes required by a normal Expr -** struct, an Expr struct with the EP_Reduced flag set in Expr.flags +** Macros to determine the number of bytes required by a normal Expr +** struct, an Expr struct with the EP_Reduced flag set in Expr.flags ** and an Expr struct with the EP_TokenOnly flag set. */ #define EXPR_FULLSIZE sizeof(Expr) /* Full size */ @@ -12108,7 +13240,7 @@ struct Expr { #define EXPR_TOKENONLYSIZE offsetof(Expr,pLeft) /* Fewer features */ /* -** Flags passed to the sqlite3ExprDup() function. See the header comment +** Flags passed to the sqlite3ExprDup() function. See the header comment ** above sqlite3ExprDup() for details. */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ @@ -12190,7 +13322,11 @@ struct IdList { ** tables in a join to 32 instead of 64. But it also reduces the size ** of the library by 738 bytes on ix86. */ -typedef u64 Bitmask; +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif /* ** The number of bits in a Bitmask. "BMS" means "BitMask Size". @@ -12202,6 +13338,7 @@ typedef u64 Bitmask; */ #define MASKBIT(n) (((Bitmask)1)<<(n)) #define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) /* ** The following structure describes the FROM clause of a SELECT statement. @@ -12235,11 +13372,15 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ + struct { + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + } fg; #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -12247,8 +13388,11 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<<N) set if column N of pTab is used */ - char *zIndexedBy; /* Identifier from "INDEXED BY <zIndex>" clause */ - Index *pIndex; /* Index structure corresponding to zIndex, if any */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY <zIndex>" clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ } a[1]; /* One entry for each identifier on the list */ }; @@ -12282,6 +13426,8 @@ struct SrcList { #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ +#define WHERE_USE_LIMIT 0x4000 /* There is a constant LIMIT clause */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -12299,12 +13445,12 @@ struct SrcList { ** pEList corresponds to the result set of a SELECT and is NULL for ** other statements. ** -** NameContexts can be nested. When resolving names, the inner-most +** NameContexts can be nested. When resolving names, the inner-most ** context is searched first. If no match is found, the next outer ** context is checked. If there is still no match, the next context ** is checked. This process continues until either a match is found ** or all contexts are check. When a match is found, the nRef member of -** the context containing the match is incremented. +** the context containing the match is incremented. ** ** Each subquery gets a new NameContext. The pNext field points to the ** NameContext in the parent query. Thus the process of scanning the @@ -12327,13 +13473,14 @@ struct NameContext { ** ** Note: NC_MinMaxAgg must have the same value as SF_MinMaxAgg and ** SQLITE_FUNC_MINMAX. -** +** */ #define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ #define NC_HasAgg 0x0002 /* One or more aggregate functions seen */ #define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ #define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* @@ -12359,13 +13506,13 @@ struct NameContext { struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ - u16 selFlags; /* Various SF_* values */ + LogEst nSelectRow; /* Estimated number of result rows */ + u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ #if SELECTTRACE_ENABLED char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ #endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ - u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -12382,21 +13529,23 @@ struct Select { ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". */ -#define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_All 0x0002 /* Includes the ALL keyword */ -#define SF_Resolved 0x0004 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0008 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0010 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0020 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0040 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x0080 /* Part of a compound query */ -#define SF_Values 0x0100 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x0200 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x0400 /* Part of a parenthesized FROM clause */ -#define SF_MaybeConvert 0x0800 /* Need convertCompoundSelectToSubquery() */ -#define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ -#define SF_Recursive 0x2000 /* The recursive part of a recursive CTE */ -#define SF_Converted 0x4000 /* By convertCompoundSelectToSubquery() */ +#define SF_Distinct 0x00001 /* Output should be DISTINCT */ +#define SF_All 0x00002 /* Includes the ALL keyword */ +#define SF_Resolved 0x00004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x00008 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x00010 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x00020 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x00040 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x00080 /* Part of a compound query */ +#define SF_Values 0x00100 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x00200 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x00400 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x00800 /* Need convertCompoundSelectToSubquery() */ +#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ +#define SF_Converted 0x08000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x10000 /* Include hidden columns in output */ /* @@ -12404,7 +13553,7 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index +** SRT_Union Store results as a key in a temporary index ** identified by pDest->iSDParm. ** ** SRT_Except Remove results from the temporary index pDest->iSDParm. @@ -12428,7 +13577,7 @@ struct Select { ** of the query. This destination implies "LIMIT 1". ** ** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. +** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing ** results. Used to implement "IN (SELECT ...)". ** @@ -12496,11 +13645,11 @@ struct SelectDest { }; /* -** During code generation of statements that do inserts into AUTOINCREMENT +** During code generation of statements that do inserts into AUTOINCREMENT ** tables, the following information is attached to the Table.u.autoInc.p ** pointer of each autoincrement table to record some side information that ** the code generator needs. We have to keep per-table autoincrement -** information in case inserts are down within triggers. Triggers do not +** information in case inserts are done within triggers. Triggers do not ** normally coordinate their activities, but we do need to coordinate the ** loading and saving of autoincrement information. */ @@ -12519,7 +13668,7 @@ struct AutoincInfo { #endif /* -** At least one instance of the following structure is created for each +** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE ** statement. All such objects are stored in the linked list headed at ** Parse.pTriggerPrg and deleted once statement compilation has been @@ -12532,7 +13681,7 @@ struct AutoincInfo { ** values for both pTrigger and orconf. ** ** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns -** accessed (or set to 0 for triggers fired as a result of INSERT +** accessed (or set to 0 for triggers fired as a result of INSERT ** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to ** a mask of new.* columns used by the program. */ @@ -12573,7 +13722,7 @@ struct TriggerPrg { ** is constant but the second part is reset at the beginning and end of ** each recursion. ** -** The nTableLock and aTableLock variables are only used if the shared-cache +** The nTableLock and aTableLock variables are only used if the shared-cache ** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are ** used to store the set of table-locks required by the statement being ** compiled. Function sqlite3TableLock() is used to add entries to the @@ -12592,6 +13741,7 @@ struct Parse { u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ + u8 disableLookaside; /* Number of times lookaside has been disabled */ int aTempReg[8]; /* Holding area for temporary registers */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -12601,9 +13751,10 @@ struct Parse { int nSet; /* Number of sets used so far */ int nOnce; /* Number of OP_Once instructions so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ + int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ - int iPartIdxTab; /* Table corresponding to a partial index */ + int iSelfTab; /* Table of an index whose exprs are being coded */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ @@ -12652,10 +13803,9 @@ struct Parse { ** in the recursive region. ************************************************************************/ - int nVar; /* Number of '?' variables seen in the SQL so far */ + ynVar nVar; /* Number of '?' variables seen in the SQL so far */ int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ - u8 bFreeWith; /* True if pWith should be freed with parser */ u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ @@ -12682,6 +13832,7 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ + With *pWithToFree; /* Free this WITH object at the end of the parse */ }; /* @@ -12705,7 +13856,8 @@ struct AuthContext { /* ** Bitfield flags for P5 value in various opcodes. */ -#define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ +#define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ + /* Also used in P2 (not P5) of OP_Delete */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ @@ -12715,15 +13867,18 @@ struct AuthContext { #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ -#define OPFLAG_P2ISREG 0x04 /* P2 to OP_Open** is a register number */ +#define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ +#define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ +#define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ /* * Each trigger present in the database schema is stored as an instance of - * struct Trigger. + * struct Trigger. * * Pointers to instances of struct Trigger are stored in two ways. - * 1. In the "trigHash" hash table (part of the sqlite3* that represents the + * 1. In the "trigHash" hash table (part of the sqlite3* that represents the * database). This allows Trigger structures to be retrieved by name. * 2. All triggers associated with a single table form a linked list, using the * pNext member of struct Trigger. A pointer to the first element of the @@ -12749,7 +13904,7 @@ struct Trigger { /* ** A trigger is either a BEFORE or an AFTER trigger. The following constants -** determine which. +** determine which. ** ** If there are multiple triggers, you might of some BEFORE and some AFTER. ** In that cases, the constants below can be ORed together. @@ -12759,15 +13914,15 @@ struct Trigger { /* * An instance of struct TriggerStep is used to store a single SQL statement - * that is a part of a trigger-program. + * that is a part of a trigger-program. * * Instances of struct TriggerStep are stored in a singly linked list (linked - * using the "pNext" member) referenced by the "step_list" member of the + * using the "pNext" member) referenced by the "step_list" member of the * associated struct Trigger instance. The first element of the linked list is * the first step of the trigger-program. - * + * * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or - * "SELECT" statement. The meanings of the other members is determined by the + * "SELECT" statement. The meanings of the other members is determined by the * value of "op" as follows: * * (op == TK_INSERT) @@ -12777,7 +13932,7 @@ struct Trigger { * zTarget -> Dequoted name of the table to insert into. * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then * this stores values to be inserted. Otherwise NULL. - * pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... + * pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... * statement, then this stores the column-names to be * inserted into. * @@ -12785,7 +13940,7 @@ struct Trigger { * zTarget -> Dequoted name of the table to delete from. * pWhere -> The WHERE clause of the DELETE statement if one is specified. * Otherwise NULL. - * + * * (op == TK_UPDATE) * zTarget -> Dequoted name of the table to update. * pWhere -> The WHERE clause of the UPDATE statement if one is specified. @@ -12793,7 +13948,7 @@ struct Trigger { * pExprList -> A list of the columns to update and the expressions to update * them to. See sqlite3Update() documentation of "pChanges" * argument. - * + * */ struct TriggerStep { u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ @@ -12811,7 +13966,7 @@ struct TriggerStep { /* ** The following structure contains information used by the sqliteFix... ** routines as they walk the parse tree to make database references -** explicit. +** explicit. */ typedef struct DbFixer DbFixer; struct DbFixer { @@ -12831,13 +13986,20 @@ struct StrAccum { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - int nChar; /* Length of the string so far */ - int nAlloc; /* Amount of space allocated in zText */ - int mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ + u32 nChar; /* Length of the string so far */ + u32 nAlloc; /* Amount of space allocated in zText */ + u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ + u8 printfFlags; /* SQLITE_PRINTF flags below */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 +#define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ +#define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ + +#define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) + /* ** A pointer to this structure is used to communicate information @@ -12865,6 +14027,7 @@ struct Sqlite3Config { int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ + int nStmtSpill; /* Stmt-journal spill-to-disk threshold */ sqlite3_mem_methods m; /* Low-level memory allocation interface */ sqlite3_mutex_methods mutex; /* Low-level mutex interface */ sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ @@ -12932,10 +14095,10 @@ struct Sqlite3Config { ** Context pointer passed down through the tree-walk. */ struct Walker { + Parse *pParse; /* Parser context. */ int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ - Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ u8 eCode; /* A small processing code */ union { /* Extra data for callback */ @@ -12944,6 +14107,8 @@ struct Walker { int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ + int *aiCol; /* array of column indexes */ } u; }; @@ -12953,6 +14118,7 @@ SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); /* ** Return code from the parse-tree walking primitives and their @@ -12973,7 +14139,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zErr; /* Error message for circular references */ + const char *zCteErr; /* Error message for circular references */ } a[1]; }; @@ -13011,7 +14177,23 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); #define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int); +SQLITE_PRIVATE int sqlite3IoerrnomemError(int); +# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) +# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) +#else +# define SQLITE_NOMEM_BKPT SQLITE_NOMEM +# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM +#endif +/* +** FTS3 and FTS4 both require virtual table support +*/ +#if defined(SQLITE_OMIT_VIRTUALTABLE) +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif /* ** FTS4 is really an extension for FTS3. It is enabled using the @@ -13060,8 +14242,9 @@ SQLITE_PRIVATE int sqlite3IsIdChar(u8); /* ** Internal function prototypes */ -#define sqlite3StrICmp sqlite3_stricmp +SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); SQLITE_PRIVATE int sqlite3Strlen30(const char*); +SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp SQLITE_PRIVATE int sqlite3MallocInit(void); @@ -13070,6 +14253,7 @@ SQLITE_PRIVATE void *sqlite3Malloc(u64); SQLITE_PRIVATE void *sqlite3MallocZero(u64); SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, u64); SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, u64); +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3*, u64); SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, u64); SQLITE_PRIVATE void *sqlite3Realloc(void*, u64); @@ -13099,7 +14283,7 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) # define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) -# define sqlite3StackFree(D,P) +# define sqlite3StackFree(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) # define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) @@ -13121,11 +14305,16 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); SQLITE_PRIVATE int sqlite3MutexInit(void); SQLITE_PRIVATE int sqlite3MutexEnd(void); #endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) +SQLITE_PRIVATE void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier() +#endif SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); SQLITE_PRIVATE void sqlite3StatusUp(int, int); SQLITE_PRIVATE void sqlite3StatusDown(int, int); -SQLITE_PRIVATE void sqlite3StatusSet(int, int); +SQLITE_PRIVATE void sqlite3StatusHighwater(int, int); /* Access to mutexes used by sqlite3_status() */ SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); @@ -13147,10 +14336,8 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; -#define SQLITE_PRINTF_INTERNAL 0x01 -#define SQLITE_PRINTF_SQLFUNC 0x02 -SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); +SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, const char*, va_list); +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, const char*, ...); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) @@ -13164,12 +14351,14 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); #endif SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); SQLITE_PRIVATE int sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); @@ -13178,6 +14367,9 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); +#endif SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); @@ -13187,6 +14379,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); @@ -13197,18 +14390,24 @@ SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); -SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); -SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*); +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); +#else +# define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ +#endif +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*,Token*); SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); -SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); @@ -13240,7 +14439,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); -SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); @@ -13270,6 +14469,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*) SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); @@ -13281,7 +14481,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList* SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u16,Expr*,Expr*); + Expr*,ExprList*,u32,Expr*,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); @@ -13293,14 +14493,19 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo*); +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ +#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ +#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); @@ -13310,14 +14515,16 @@ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); @@ -13354,18 +14561,22 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); +#endif SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); -SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, - u8,u8,int,int*); + u8,u8,int,int*,int*); SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); -SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); +SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); SQLITE_PRIVATE void sqlite3MultiWrite(Parse*); SQLITE_PRIVATE void sqlite3MayAbort(Parse*); @@ -13382,11 +14593,11 @@ SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); #else # define sqlite3SelectSetName(A,B) #endif -SQLITE_PRIVATE void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); -SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3*); +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); +SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); -SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); @@ -13417,6 +14628,7 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) @@ -13426,6 +14638,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif @@ -13463,7 +14676,11 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); #ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); +#endif /* ** Routines to read and write variable-length integers. These used to @@ -13489,7 +14706,7 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); #define putVarint sqlite3PutVarint -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); @@ -13498,6 +14715,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); +SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -13530,7 +14748,7 @@ SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); -SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, +SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); @@ -13540,11 +14758,12 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; +SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; -SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte; #endif @@ -13561,6 +14780,7 @@ SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -13578,7 +14798,6 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); -SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); SQLITE_PRIVATE void sqlite3SchemaClear(void *); SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); @@ -13589,11 +14808,13 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif -SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, +SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), FuncDestructor *pDestructor ); +SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); +SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); @@ -13650,7 +14871,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabRollback(X) # define sqlite3VtabCommit(X) # define sqlite3VtabInSync(db) 0 -# define sqlite3VtabLock(X) +# define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK @@ -13669,6 +14890,8 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); @@ -13706,7 +14929,7 @@ SQLITE_PRIVATE void sqlite3WithPush(Parse*, With*, u8); ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign ** key functionality is available. If OMIT_TRIGGER is defined but ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In -** this case foreign keys are parsed, but no other functionality is +** this case foreign keys are parsed, but no other functionality is ** provided (enforcement of FK constraints requires the triggers sub-system). */ #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) @@ -13767,19 +14990,14 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*); +SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); #ifdef SQLITE_ENABLE_ATOMIC_WRITE -SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); -SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p); -#else - #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) - #define sqlite3JournalExists(p) 1 #endif +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p); SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); -SQLITE_PRIVATE int sqlite3MemJournalSize(void); -SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *); SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 @@ -13810,7 +15028,7 @@ SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); /* ** If the SQLITE_ENABLE IOTRACE exists then the global variable ** sqlite3IoTrace is a pointer to a printf-like routine used to -** print I/O tracing messages. +** print I/O tracing messages. */ #ifdef SQLITE_ENABLE_IOTRACE # define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } @@ -13844,7 +15062,7 @@ SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); ** that allocations that might have been satisfied by lookaside are not ** passed back to non-lookaside free() routines. Asserts such as the ** example above are placed on the non-lookaside free() routines to verify -** this constraint. +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. @@ -14040,6 +15258,18 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_SORTER_PMASZ 250 #endif +/* Statement journals spill to disk when their size exceeds the following +** threashold (in bytes). 0 means that statement journals are created and +** written to disk immediately (the default behavior for SQLite versions +** before 3.12.0). -1 means always keep the entire statement journal in +** memory. (The statement journal is also always held entirely in memory +** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this +** setting.) +*/ +#ifndef SQLITE_STMTJRNL_SPILL +# define SQLITE_STMTJRNL_SPILL (64*1024) +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -14054,6 +15284,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* neverCorrupt */ 128, /* szLookaside */ 500, /* nLookaside */ + SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ @@ -14100,7 +15331,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { ** database connections. After initialization, this table is ** read-only. */ -SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; /* ** Constant tokens for values 0 and 1. @@ -14142,6 +15373,11 @@ SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; */ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; +/* +** Name of the default collating sequence +*/ +SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; + /************** End of global.c **********************************************/ /************** Begin file ctime.c *******************************************/ /* @@ -14209,6 +15445,9 @@ static const char * const azCompileOpt[] = { #if SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#if SQLITE_ENABLE_8_3_NAMES + "ENABLE_8_3_NAMES", +#endif #if SQLITE_ENABLE_API_ARMOR "ENABLE_API_ARMOR", #endif @@ -14242,12 +15481,18 @@ static const char * const azCompileOpt[] = { #if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", #endif +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif #if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif #if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif #if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif @@ -14298,6 +15543,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_INT64_TYPE "INT64_TYPE", #endif +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + "LIKE_DOESNT_MATCH_BLOBS", +#endif #if SQLITE_LOCK_TRACE "LOCK_TRACE", #endif @@ -14624,6 +15872,17 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N){ #endif /* +** VDBE_DISPLAY_P4 is true or false depending on whether or not the +** "explain" P4 display logic is enabled. +*/ +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +# define VDBE_DISPLAY_P4 1 +#else +# define VDBE_DISPLAY_P4 0 +#endif + +/* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance ** of the following structure. @@ -14644,42 +15903,52 @@ typedef struct Explain Explain; /* Elements of the linked list at Vdbe.pAuxData */ typedef struct AuxData AuxData; +/* Types of VDBE cursors */ +#define CURTYPE_BTREE 0 +#define CURTYPE_SORTER 1 +#define CURTYPE_VTAB 2 +#define CURTYPE_PSEUDO 3 + /* -** A cursor is a pointer into a single BTree within a database file. -** The cursor can seek to a BTree entry with a particular key, or -** loop over all entries of the Btree. You can also insert new BTree -** entries or retrieve the key or data from the entry that the cursor -** is currently pointing to. +** A VdbeCursor is an superclass (a wrapper) for various cursor objects: ** -** Cursors can also point to virtual tables, sorters, or "pseudo-tables". -** A pseudo-table is a single-row table implemented by registers. -** -** Every cursor that the virtual machine has open is represented by an -** instance of the following structure. +** * A b-tree cursor +** - In the main database or in an ephemeral database +** - On either an index or a table +** * A sorter +** * A virtual table +** * A one-row "pseudotable" stored in a single register */ +typedef struct VdbeCursor VdbeCursor; struct VdbeCursor { - BtCursor *pCursor; /* The cursor structure of the backend */ - Btree *pBt; /* Separate file holding temporary table */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int seekResult; /* Result of previous sqlite3BtreeMoveto() */ - int pseudoTableReg; /* Register holding pseudotable content. */ - i16 nField; /* Number of fields in the header */ - u16 nHdrParsed; /* Number of header fields parsed so far */ -#ifdef SQLITE_DEBUG - u8 seekOp; /* Most recent seek operation on this cursor */ -#endif + u8 eCurType; /* One of the CURTYPE_* values above */ i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ u8 nullRow; /* True if pointing to a row with no data */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + u8 isTable; /* True for rowid tables. False for indexes */ +#ifdef SQLITE_DEBUG + u8 seekOp; /* Most recent seek operation on this cursor */ + u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ +#endif Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ - Bool isTable:1; /* True if a table requiring integer keys */ - Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ + Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ Pgno pgnoRoot; /* Root page of the open btree cursor */ - sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ + i16 nField; /* Number of fields in the header */ + u16 nHdrParsed; /* Number of header fields parsed so far */ + union { + BtCursor *pCursor; /* CURTYPE_BTREE. Btree cursor */ + sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ + int pseudoTableReg; /* CURTYPE_PSEUDO. Reg holding content. */ + VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ + } uc; + Btree *pBt; /* Separate file holding temporary table */ + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + int seekResult; /* Result of previous sqlite3BtreeMoveto() */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ + int *aAltMap; /* Mapping from table to index column numbers */ #ifdef SQLITE_ENABLE_COLUMN_USED_MASK u64 maskUsed; /* Mask of columns used by this cursor */ #endif @@ -14704,7 +15973,6 @@ struct VdbeCursor { ** static element declared in the structure. nField total array slots for ** aType[] and nField+1 array slots for aOffset[] */ }; -typedef struct VdbeCursor VdbeCursor; /* ** When a sub-program is executed (OP_Program), a structure of this type @@ -14738,6 +16006,7 @@ struct VdbeFrame { VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + AuxData *pAuxData; /* Linked list of auxdata allocations */ int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ @@ -14772,6 +16041,7 @@ struct Mem { } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ @@ -14814,7 +16084,7 @@ struct Mem { #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0x01ff /* Mask of type bits */ +#define MEM_TypeMask 0x81ff /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -14828,11 +16098,18 @@ struct Mem { #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ #define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ #ifdef SQLITE_OMIT_INCRBLOB #undef MEM_Zero #define MEM_Zero 0x0000 #endif +/* Return TRUE if Mem X contains dynamically allocated content - anything +** that needs to be deallocated to avoid a leak. +*/ +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) + /* ** Clear any existing type flags from a Mem and replace them with f */ @@ -14949,16 +16226,16 @@ struct Vdbe { #endif u16 nResColumn; /* Number of columns in one row of the result set */ u8 errorAction; /* Recovery action to do in case of an error */ + bft expired:1; /* True if the VM needs to be recompiled */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ bft explain:2; /* True if EXPLAIN present on SQL command */ bft changeCntOn:1; /* True to update the change-counter */ - bft expired:1; /* True if the VM needs to be recompiled */ bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ bft isPrepareV2:1; /* True if prepared with prepare_v2() */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ int nChange; /* Number of db changes made since last reset */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ @@ -15002,16 +16279,17 @@ struct Vdbe { SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); #endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); @@ -15047,8 +16325,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); -#define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); SQLITE_PRIVATE const char *sqlite3OpcodeName(int); SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); @@ -15067,11 +16343,15 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); -#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +#if !defined(SQLITE_OMIT_SHARED_CACHE) SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); #else # define sqlite3VdbeEnter(X) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); +#else # define sqlite3VdbeLeave(X) #endif @@ -15109,15 +16389,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); /* ** Variables in which to record status information. */ -typedef struct sqlite3StatType sqlite3StatType; -static SQLITE_WSD struct sqlite3StatType { #if SQLITE_PTRSIZE>4 - sqlite3_int64 nowValue[10]; /* Current value */ - sqlite3_int64 mxValue[10]; /* Maximum value */ +typedef sqlite3_int64 sqlite3StatValueType; #else - u32 nowValue[10]; /* Current value */ - u32 mxValue[10]; /* Maximum value */ +typedef u32 sqlite3StatValueType; #endif +typedef struct sqlite3StatType sqlite3StatType; +static SQLITE_WSD struct sqlite3StatType { + sqlite3StatValueType nowValue[10]; /* Current value */ + sqlite3StatValueType mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; /* @@ -15198,18 +16478,24 @@ SQLITE_PRIVATE void sqlite3StatusDown(int op, int N){ } /* -** Set the value of a status to X. The highwater mark is adjusted if -** necessary. The caller must hold the appropriate mutex. +** Adjust the highwater mark if necessary. +** The caller must hold the appropriate mutex. */ -SQLITE_PRIVATE void sqlite3StatusSet(int op, int X){ +SQLITE_PRIVATE void sqlite3StatusHighwater(int op, int X){ + sqlite3StatValueType newValue; wsdStatInit; + assert( X>=0 ); + newValue = (sqlite3StatValueType)X; assert( op>=0 && op<ArraySize(wsdStat.nowValue) ); assert( op>=0 && op<ArraySize(statMutex) ); assert( sqlite3_mutex_held(statMutex[op] ? sqlite3Pcache1Mutex() : sqlite3MallocMutex()) ); - wsdStat.nowValue[op] = X; - if( wsdStat.nowValue[op]>wsdStat.mxValue[op] ){ - wsdStat.mxValue[op] = wsdStat.nowValue[op]; + assert( op==SQLITE_STATUS_MALLOC_SIZE + || op==SQLITE_STATUS_PAGECACHE_SIZE + || op==SQLITE_STATUS_SCRATCH_SIZE + || op==SQLITE_STATUS_PARSER_STACK ); + if( newValue>wsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = newValue; } } @@ -15342,10 +16628,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( + pSchema->idxHash.count + pSchema->fkeyHash.count ); - nByte += sqlite3MallocSize(pSchema->tblHash.ht); - nByte += sqlite3MallocSize(pSchema->trigHash.ht); - nByte += sqlite3MallocSize(pSchema->idxHash.ht); - nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + nByte += sqlite3_msize(pSchema->tblHash.ht); + nByte += sqlite3_msize(pSchema->trigHash.ht); + nByte += sqlite3_msize(pSchema->idxHash.ht); + nByte += sqlite3_msize(pSchema->fkeyHash.ht); for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); @@ -15498,38 +16784,54 @@ struct DateTime { char validHMS; /* True (1) if h,m,s are valid */ char validJD; /* True (1) if iJD is valid */ char validTZ; /* True (1) if tz is valid */ + char tzSet; /* Timezone was set explicitly */ }; /* -** Convert zDate into one or more integers. Additional arguments -** come in groups of 5 as follows: +** Convert zDate into one or more integers according to the conversion +** specifier zFormat. ** -** N number of digits in the integer -** min minimum allowed value of the integer -** max maximum allowed value of the integer -** nextC first character after the integer -** pVal where to write the integers value. +** zFormat[] contains 4 characters for each integer converted, except for +** the last integer which is specified by three characters. The meaning +** of a four-character format specifiers ABCD is: +** +** A: number of digits to convert. Always "2" or "4". +** B: minimum value. Always "0" or "1". +** C: maximum value, decoded as: +** a: 12 +** b: 14 +** c: 24 +** d: 31 +** e: 59 +** f: 9999 +** D: the separator character, or \000 to indicate this is the +** last number to convert. +** +** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would +** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". +** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates +** the 2-digit day which is the last integer in the set. ** -** Conversions continue until one with nextC==0 is encountered. ** The function returns the number of successful conversions. */ -static int getDigits(const char *zDate, ...){ +static int getDigits(const char *zDate, const char *zFormat, ...){ + /* The aMx[] array translates the 3rd character of each format + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; va_list ap; - int val; - int N; - int min; - int max; - int nextC; - int *pVal; int cnt = 0; - va_start(ap, zDate); + char nextC; + va_start(ap, zFormat); do{ - N = va_arg(ap, int); - min = va_arg(ap, int); - max = va_arg(ap, int); - nextC = va_arg(ap, int); - pVal = va_arg(ap, int*); + char N = zFormat[0] - '0'; + char min = zFormat[1] - '0'; + int val = 0; + u16 max; + + assert( zFormat[2]>='a' && zFormat[2]<='f' ); + max = aMx[zFormat[2] - 'a']; + nextC = zFormat[3]; val = 0; while( N-- ){ if( !sqlite3Isdigit(*zDate) ){ @@ -15538,12 +16840,13 @@ static int getDigits(const char *zDate, ...){ val = val*10 + *zDate - '0'; zDate++; } - if( val<min || val>max || (nextC!=0 && nextC!=*zDate) ){ + if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ goto end_getDigits; } - *pVal = val; + *va_arg(ap,int*) = val; zDate++; cnt++; + zFormat += 4; }while( nextC ); end_getDigits: va_end(ap); @@ -15584,13 +16887,14 @@ static int parseTimezone(const char *zDate, DateTime *p){ return c!=0; } zDate++; - if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ + if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ return 1; } zDate += 5; p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tzSet = 1; return *zDate!=0; } @@ -15604,13 +16908,13 @@ zulu_time: static int parseHhMmSs(const char *zDate, DateTime *p){ int h, m, s; double ms = 0.0; - if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){ + if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ return 1; } zDate += 5; if( *zDate==':' ){ zDate++; - if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){ + if( getDigits(zDate, "20e", &s)!=1 ){ return 1; } zDate += 2; @@ -15698,7 +17002,7 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ }else{ neg = 0; } - if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){ + if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ return 1; } zDate += 10; @@ -16023,13 +17327,18 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ } #ifndef SQLITE_OMIT_LOCALTIME else if( strcmp(z, "utc")==0 ){ - sqlite3_int64 c1; - computeJD(p); - c1 = localtimeOffset(p, pCtx, &rc); - if( rc==SQLITE_OK ){ - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + if( p->tzSet==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } + p->tzSet = 1; + }else{ + rc = SQLITE_OK; } } #endif @@ -16377,7 +17686,7 @@ static void strftimeFunc( sqlite3_result_error_toobig(context); return; }else{ - z = sqlite3DbMallocRaw(db, (int)n); + z = sqlite3DbMallocRawNN(db, (int)n); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -16546,29 +17855,23 @@ static void currentTimeFunc( ** external linkage. */ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ - static SQLITE_WSD FuncDef aDateTimeFuncs[] = { + static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aDateTimeFuncs); - - for(i=0; i<ArraySize(aDateTimeFuncs); i++){ - sqlite3FuncDefInsert(pHash, &aFunc[i]); - } + sqlite3InsertBuiltinFuncs(aDateTimeFuncs, ArraySize(aDateTimeFuncs)); } /************** End of date.c ************************************************/ @@ -16593,13 +17896,35 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ #undef _SQLITE_OS_C_ /* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#if defined(SQLITE_TEST) +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#endif /* defined(SQLITE_TEST) */ + +/* +** When testing, also keep a count of the number of open files. +*/ +#if defined(SQLITE_TEST) +SQLITE_API int sqlite3_open_file_count = 0; +#endif /* defined(SQLITE_TEST) */ + +/* ** The default SQLite sqlite3_vfs implementations do not allocate ** memory (actually, os_unix.c allocates a small amount of memory ** from within OsOpen()), but some third-party implementations may. ** So we test the effects of a malloc() failing and the sqlite3OsXXX() ** function returning SQLITE_IOERR_NOMEM using the DO_OS_MALLOC_TEST macro. ** -** The following functions are instrumented for malloc() failure +** The following functions are instrumented for malloc() failure ** testing: ** ** sqlite3OsRead() @@ -16619,9 +17944,9 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ #if defined(SQLITE_TEST) SQLITE_API int sqlite3_memdebug_vfs_oom_test = 1; #define DO_OS_MALLOC_TEST(x) \ - if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3IsMemJournal(x))) { \ + if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \ void *pTstAlloc = sqlite3Malloc(10); \ - if (!pTstAlloc) return SQLITE_IOERR_NOMEM; \ + if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT; \ sqlite3_free(pTstAlloc); \ } #else @@ -16685,8 +18010,8 @@ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_TEST if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite - ** is using a regular VFS, it is called after the corresponding - ** transaction has been committed. Injecting a fault at this point + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM ** but the transaction is committed anyway. ** @@ -16755,10 +18080,10 @@ SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ ** VFS methods. */ SQLITE_PRIVATE int sqlite3OsOpen( - sqlite3_vfs *pVfs, - const char *zPath, - sqlite3_file *pFile, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, int *pFlagsOut ){ int rc; @@ -16777,18 +18102,18 @@ SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dir return pVfs->xDelete(pVfs, zPath, dirSync); } SQLITE_PRIVATE int sqlite3OsAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + int flags, int *pResOut ){ DO_OS_MALLOC_TEST(0); return pVfs->xAccess(pVfs, zPath, flags, pResOut); } SQLITE_PRIVATE int sqlite3OsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nPathOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, char *zPathOut ){ DO_OS_MALLOC_TEST(0); @@ -16815,6 +18140,9 @@ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufO SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() @@ -16834,13 +18162,13 @@ SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p } SQLITE_PRIVATE int sqlite3OsOpenMalloc( - sqlite3_vfs *pVfs, - const char *zFile, - sqlite3_file **ppFile, + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, int flags, int *pOutFlags ){ - int rc = SQLITE_NOMEM; + int rc; sqlite3_file *pFile; pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); if( pFile ){ @@ -16850,6 +18178,8 @@ SQLITE_PRIVATE int sqlite3OsOpenMalloc( }else{ *ppFile = pFile; } + }else{ + rc = SQLITE_NOMEM_BKPT; } return rc; } @@ -16869,7 +18199,7 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *pFile){ */ SQLITE_PRIVATE int sqlite3OsInit(void){ void *p = sqlite3_malloc(10); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; sqlite3_free(p); return sqlite3_os_init(); } @@ -17296,10 +18626,11 @@ static void sqlite3MemFree(void *pPrior){ */ static int sqlite3MemSize(void *pPrior){ #ifdef SQLITE_MALLOCSIZE - return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; + assert( pPrior!=0 ); + return (int)SQLITE_MALLOCSIZE(pPrior); #else sqlite3_int64 *p; - if( pPrior==0 ) return 0; + assert( pPrior!=0 ); p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; @@ -18429,7 +19760,7 @@ static void memsys3FreeUnsafe(void *pOld){ */ static int memsys3Size(void *p){ Mem3Block *pBlock; - if( p==0 ) return 0; + assert( p!=0 ); pBlock = (Mem3Block*)p; assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; @@ -18668,7 +19999,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** ** This memory allocator uses the following algorithm: ** -** 1. All memory allocations sizes are rounded up to a power of 2. +** 1. All memory allocation sizes are rounded up to a power of 2. ** ** 2. If two adjacent free blocks are the halves of a larger block, ** then the two blocks are coalesced into the single larger block. @@ -18745,6 +20076,7 @@ static SQLITE_WSD struct Mem5Global { */ sqlite3_mutex *mutex; +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Performance statistics */ @@ -18756,11 +20088,12 @@ static SQLITE_WSD struct Mem5Global { u32 maxOut; /* Maximum instantaneous currentOut */ u32 maxCount; /* Maximum instantaneous currentCount */ u32 maxRequest; /* Largest allocation (exclusive of internal frag) */ +#endif /* ** Lists of free blocks. aiFreelist[0] is a list of free blocks of ** size mem5.szAtom. aiFreelist[1] holds blocks of size szAtom*2. - ** and so forth. + ** aiFreelist[2] holds free blocks of size szAtom*4. And so forth. */ int aiFreelist[LOGMAX+1]; @@ -18826,9 +20159,7 @@ static void memsys5Link(int i, int iLogsize){ } /* -** If the STATIC_MEM mutex is not already held, obtain it now. The mutex -** will already be held (obtained by code in malloc.c) if -** sqlite3GlobalConfig.bMemStat is true. +** Obtain or release the mutex needed to access global data structures. */ static void memsys5Enter(void){ sqlite3_mutex_enter(mem5.mutex); @@ -18838,17 +20169,15 @@ static void memsys5Leave(void){ } /* -** Return the size of an outstanding allocation, in bytes. The -** size returned omits the 8-byte header overhead. This only -** works for chunks that are currently checked out. +** Return the size of an outstanding allocation, in bytes. +** This only works for chunks that are currently checked out. */ static int memsys5Size(void *p){ - int iSize = 0; - if( p ){ - int i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); - assert( i>=0 && i<mem5.nBlock ); - iSize = mem5.szAtom * (1 << (mem5.aCtrl[i]&CTRL_LOGSIZE)); - } + int iSize, i; + assert( p!=0 ); + i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); + assert( i>=0 && i<mem5.nBlock ); + iSize = mem5.szAtom * (1 << (mem5.aCtrl[i]&CTRL_LOGSIZE)); return iSize; } @@ -18871,21 +20200,20 @@ static void *memsys5MallocUnsafe(int nByte){ /* nByte must be a positive */ assert( nByte>0 ); + /* No more than 1GiB per allocation */ + if( nByte > 0x40000000 ) return 0; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* Keep track of the maximum allocation request. Even unfulfilled ** requests are counted */ if( (u32)nByte>mem5.maxRequest ){ mem5.maxRequest = nByte; } +#endif - /* Abort if the requested allocation size is larger than the largest - ** power of two that we can represent using 32-bit signed integers. - */ - if( nByte > 0x40000000 ){ - return 0; - } /* Round nByte up to the next valid power of two */ - for(iFullSz=mem5.szAtom, iLogsize=0; iFullSz<nByte; iFullSz *= 2, iLogsize++){} + for(iFullSz=mem5.szAtom,iLogsize=0; iFullSz<nByte; iFullSz*=2,iLogsize++){} /* Make sure mem5.aiFreelist[iLogsize] contains at least one free ** block. If not, then split a block of the next larger power of @@ -18909,6 +20237,7 @@ static void *memsys5MallocUnsafe(int nByte){ } mem5.aCtrl[i] = iLogsize; +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* Update allocator performance statistics. */ mem5.nAlloc++; mem5.totalAlloc += iFullSz; @@ -18917,6 +20246,7 @@ static void *memsys5MallocUnsafe(int nByte){ mem5.currentOut += iFullSz; if( mem5.maxCount<mem5.currentCount ) mem5.maxCount = mem5.currentCount; if( mem5.maxOut<mem5.currentOut ) mem5.maxOut = mem5.currentOut; +#endif #ifdef SQLITE_DEBUG /* Make sure the allocated memory does not assume that it is set to zero @@ -18951,23 +20281,26 @@ static void memsys5FreeUnsafe(void *pOld){ mem5.aCtrl[iBlock] |= CTRL_FREE; mem5.aCtrl[iBlock+size-1] |= CTRL_FREE; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) assert( mem5.currentCount>0 ); assert( mem5.currentOut>=(size*mem5.szAtom) ); mem5.currentCount--; mem5.currentOut -= size*mem5.szAtom; assert( mem5.currentOut>0 || mem5.currentCount==0 ); assert( mem5.currentCount>0 || mem5.currentOut==0 ); +#endif mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; while( ALWAYS(iLogsize<LOGMAX) ){ int iBuddy; if( (iBlock>>iLogsize) & 1 ){ iBuddy = iBlock - size; + assert( iBuddy>=0 ); }else{ iBuddy = iBlock + size; + if( iBuddy>=mem5.nBlock ) break; } - assert( iBuddy>=0 ); - if( (iBuddy+(1<<iLogsize))>mem5.nBlock ) break; if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; memsys5Unlink(iBuddy, iLogsize); iLogsize++; @@ -19042,13 +20375,11 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ if( nBytes<=nOld ){ return pPrior; } - memsys5Enter(); - p = memsys5MallocUnsafe(nBytes); + p = memsys5Malloc(nBytes); if( p ){ memcpy(p, pPrior, nOld); - memsys5FreeUnsafe(pPrior); + memsys5Free(pPrior); } - memsys5Leave(); return p; } @@ -19244,7 +20575,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -19275,8 +20606,10 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ pTo->xMutexLeave = pFrom->xMutexLeave; pTo->xMutexHeld = pFrom->xMutexHeld; pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -19311,6 +20644,7 @@ SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int id){ if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -19319,6 +20653,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -19327,6 +20662,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -19337,6 +20673,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -19348,6 +20685,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -19361,6 +20699,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -19371,9 +20710,11 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ ** intended for use inside assert() statements. */ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif @@ -19652,7 +20993,9 @@ struct sqlite3_mutex { #endif }; #if SQLITE_MUTEX_NREF -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 } +#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0} +#elif defined(SQLITE_ENABLE_API_ARMOR) +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 } #else #define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } #endif @@ -19683,6 +21026,19 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ #endif /* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + +/* ** Initialize and deinitialize the mutex subsystem. */ static int pthreadMutexInit(void){ return SQLITE_OK; } @@ -20033,8 +21389,8 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -20144,14 +21500,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -20177,17 +21533,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -20345,6 +21701,24 @@ static int winMutexNotheld(sqlite3_mutex *p){ #endif /* +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ + defined(_MSC_VER) && _MSC_VER>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) + MemoryBarrier(); +#endif +} + +/* ** Initialize and deinitialize the mutex subsystem. */ static sqlite3_mutex winMutex_staticMutexes[] = { @@ -20697,16 +22071,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -20723,7 +22088,7 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -20734,50 +22099,21 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ return mem0.mutex; } -/* -** This routine runs when the memory allocator sees that the -** total memory allocation is about to exceed the soft heap -** limit. -*/ -static void softHeapLimitEnforcer( - void *NotUsed, - sqlite3_int64 NotUsed2, - int allocSize -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_release_memory(allocSize); -} - -/* -** Change the alarm callback -*/ -static int sqlite3MemoryAlarm( - void(*xCallback)(void *pArg, sqlite3_int64 used,int N), - void *pArg, - sqlite3_int64 iThreshold -){ - sqlite3_int64 nUsed; - sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; - mem0.alarmThreshold = iThreshold; - nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); - sqlite3_mutex_leave(mem0.mutex); - return SQLITE_OK; -} - #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite3MemoryAlarm. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ - return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); + (void)xCallback; + (void)pArg; + (void)iThreshold; + return SQLITE_OK; } #endif @@ -20788,19 +22124,21 @@ SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - if( n<0 ) return priorLimit; - if( n>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); - }else{ - sqlite3MemoryAlarm(0, 0, 0); + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; @@ -20819,9 +22157,7 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); - if( sqlite3GlobalConfig.bCoreMutex ){ - mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); - } + mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 && sqlite3GlobalConfig.nScratch>0 ){ int i, n, sz; @@ -20897,19 +22233,10 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag){ ** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ - void (*xCallback)(void*,sqlite3_int64,int); - sqlite3_int64 nowUsed; - void *pArg; - if( mem0.alarmCallback==0 ) return; - xCallback = mem0.alarmCallback; - nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - pArg = mem0.alarmArg; - mem0.alarmCallback = 0; + if( mem0.alarmThreshold<=0 ) return; sqlite3_mutex_leave(mem0.mutex); - xCallback(pArg, nowUsed, nByte); + sqlite3_release_memory(nByte); sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; } /* @@ -20921,8 +22248,8 @@ static int mallocWithAlarm(int n, void **pp){ void *p; assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - if( mem0.alarmCallback!=0 ){ + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; @@ -20933,7 +22260,7 @@ static int mallocWithAlarm(int n, void **pp){ } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmCallback ){ + if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } @@ -21013,7 +22340,7 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; @@ -21057,7 +22384,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ scratchAllocOut--; #endif - if( p>=sqlite3GlobalConfig.pScratch && p<mem0.pScratchEnd ){ + if( SQLITE_WITHIN(p, sqlite3GlobalConfig.pScratch, mem0.pScratchEnd) ){ /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */ ScratchFreeslot *pSlot; pSlot = (ScratchFreeslot*)p; @@ -21093,7 +22420,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, void *p){ - return p>=db->lookaside.pStart && p<db->lookaside.pEnd; + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); } #else #define isLookaside(A,B) 0 @@ -21108,25 +22435,27 @@ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return sqlite3MallocSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; + assert( p!=0 ); + if( db==0 || !isLookaside(db,p) ){ +#if SQLITE_DEBUG + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - return sqlite3GlobalConfig.m.xSize(p); } +#endif + return sqlite3GlobalConfig.m.xSize(p); + }else{ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.sz; } } SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); + return p ? sqlite3GlobalConfig.m.xSize(p) : 0; } /* @@ -21214,14 +22543,14 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); - if( pNew==0 && mem0.alarmCallback ){ + if( pNew==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } @@ -21272,16 +22601,31 @@ SQLITE_PRIVATE void *sqlite3MallocZero(u64 n){ ** the mallocFailed flag in the connection pointer. */ SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ - void *p = sqlite3DbMallocRaw(db, n); - if( p ){ - memset(p, 0, (size_t)n); - } + void *p; + testcase( db==0 ); + p = sqlite3DbMallocRaw(db, n); + if( p ) memset(p, 0, (size_t)n); + return p; +} + + +/* Finish the work of sqlite3DbMallocRawNN for the unusual and +** slower case when the allocation cannot be fulfilled using lookaside. +*/ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p; + assert( db!=0 ); + p = sqlite3Malloc(n); + if( !p ) sqlite3OomFault(db); + sqlite3MemdebugSetType(p, + (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } /* -** Allocate and zero memory. If the allocation fails, make -** the mallocFailed flag in the connection pointer. +** Allocate memory, either lookaside (if possible) or heap. +** If the allocation fails, set the mallocFailed flag in +** the connection pointer. ** ** If db!=0 and db->mallocFailed is true (indicating a prior malloc ** failure on the same database connection) then always return 0. @@ -21296,64 +22640,73 @@ SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ ** ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. +** +** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is +** not a NULL pointer. */ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ void *p; - assert( db==0 || sqlite3_mutex_held(db->mutex) ); - assert( db==0 || db->pnBytesFreed==0 ); + if( db ) return sqlite3DbMallocRawNN(db, n); + p = sqlite3Malloc(n); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + return p; +} +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ #ifndef SQLITE_OMIT_LOOKASIDE - if( db ){ - LookasideSlot *pBuf; - if( db->mallocFailed ){ - return 0; - } - if( db->lookaside.bEnabled ){ - if( n>db->lookaside.sz ){ - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; - db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; - } - return (void*)pBuf; + LookasideSlot *pBuf; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->lookaside.bDisable==0 ){ + assert( db->mallocFailed==0 ); + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; } + return (void*)pBuf; } + }else if( db->mallocFailed ){ + return 0; } #else - if( db && db->mallocFailed ){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->mallocFailed ){ return 0; } #endif - p = sqlite3Malloc(n); - if( !p && db ){ - db->mallocFailed = 1; - } - sqlite3MemdebugSetType(p, - (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); - return p; + return dbMallocRawFinish(db, n); } +/* Forward declaration */ +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); + /* ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ - void *pNew = 0; assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; + return dbReallocFinish(db, p, n); +} +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ + void *pNew = 0; + assert( db!=0 ); + assert( p!=0 ); if( db->mallocFailed==0 ){ - if( p==0 ){ - return sqlite3DbMallocRaw(db, n); - } if( isLookaside(db, p) ){ - if( n<=db->lookaside.sz ){ - return p; - } - pNew = sqlite3DbMallocRaw(db, n); + pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ memcpy(pNew, p, db->lookaside.sz); sqlite3DbFree(db, p); @@ -21364,10 +22717,10 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } sqlite3MemdebugSetType(pNew, - (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; @@ -21409,11 +22762,12 @@ SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; + assert( db!=0 ); if( z==0 ){ return 0; } assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRaw(db, n+1); + zNew = sqlite3DbMallocRawNN(db, n+1); if( zNew ){ memcpy(zNew, z, (size_t)n); zNew[n] = 0; @@ -21430,12 +22784,44 @@ SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ } /* +** Call this routine to record the fact that an OOM (out-of-memory) error +** has happened. This routine will set db->mallocFailed, and also +** temporarily disable the lookaside memory allocator and interrupt +** any running VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ + db->u1.isInterrupted = 1; + } + db->lookaside.bDisable++; + } +} + +/* +** This routine reactivates the memory allocator and clears the +** db->mallocFailed flag as necessary. +** +** The memory allocator is not restarted if there are running +** VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; + db->u1.isInterrupted = 0; + assert( db->lookaside.bDisable>0 ); + db->lookaside.bDisable--; + } +} + +/* ** Take actions at the end of an API call to indicate an OOM error */ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ - db->mallocFailed = 0; + sqlite3OomClear(db); sqlite3Error(db, SQLITE_NOMEM); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* @@ -21638,7 +23024,6 @@ static char *getTextArg(PrintfArguments *p){ */ SQLITE_PRIVATE void sqlite3VXPrintf( StrAccum *pAccum, /* Accumulate results here */ - u32 bFlags, /* SQLITE_PRINTF_* flags */ const char *fmt, /* Format string */ va_list ap /* arguments */ ){ @@ -21678,11 +23063,11 @@ SQLITE_PRIVATE void sqlite3VXPrintf( char buf[etBUFSIZE]; /* Conversion buffer */ bufpt = 0; - if( bFlags ){ - if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ + if( pAccum->printfFlags ){ + if( (bArgList = (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ pArgList = va_arg(ap, PrintfArguments*); } - useIntern = bFlags & SQLITE_PRINTF_INTERNAL; + useIntern = pAccum->printfFlags & SQLITE_PRINTF_INTERNAL; }else{ bArgList = useIntern = 0; } @@ -21737,6 +23122,12 @@ SQLITE_PRIVATE void sqlite3VXPrintf( testcase( wx>0x7fffffff ); width = wx & 0x7fffffff; } + assert( width>=0 ); +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif /* Get the precision */ if( c=='.' ){ @@ -21763,6 +23154,14 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ precision = -1; } + assert( precision>=(-1) ); +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + + /* Get the conversion type modifier */ if( c=='l' ){ flag_long = 1; @@ -21935,21 +23334,16 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( realvalue>0.0 ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } - while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } realvalue /= scale; while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - if( prefix=='-' ){ - bufpt = "-Inf"; - }else if( prefix=='+' ){ - bufpt = "+Inf"; - }else{ - bufpt = "Inf"; - } - length = sqlite3Strlen30(bufpt); + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); break; } } @@ -22098,12 +23492,13 @@ SQLITE_PRIVATE void sqlite3VXPrintf( case etDYNSTRING: if( bArgList ){ bufpt = getTextArg(pArgList); + xtype = etSTRING; }else{ bufpt = va_arg(ap,char*); } if( bufpt==0 ){ bufpt = ""; - }else if( xtype==etDYNSTRING && !bArgList ){ + }else if( xtype==etDYNSTRING ){ zExtra = bufpt; } if( precision>=0 ){ @@ -22112,9 +23507,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = sqlite3Strlen30(bufpt); } break; - case etSQLESCAPE: - case etSQLESCAPE2: - case etSQLESCAPE3: { + case etSQLESCAPE: /* Escape ' characters */ + case etSQLESCAPE2: /* Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -22133,7 +23528,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; - n += i + 1 + needQuote*2; + n += i + 3; if( n>etBUFSIZE ){ bufpt = zExtra = sqlite3Malloc( n ); if( bufpt==0 ){ @@ -22197,7 +23592,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); if( zExtra ){ - sqlite3_free(zExtra); + sqlite3DbFree(pAccum->db, zExtra); zExtra = 0; } }/* End for loop over the format string */ @@ -22223,8 +23618,9 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ setStrAccumError(p, STRACCUM_TOOBIG); return N; }else{ - char *zOld = (p->zText==p->zBase ? 0 : p->zText); + char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar; + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); szNew += N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, @@ -22245,9 +23641,10 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); - if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + if( !isMalloced(p) && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; p->nAlloc = sqlite3DbMallocSize(p->db, zNew); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); @@ -22265,6 +23662,7 @@ SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ return; } + assert( (p->zText==p->zBase)==!isMalloced(p) ); while( (N--)>0 ) p->zText[p->nChar++] = c; } @@ -22282,6 +23680,7 @@ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* @@ -22317,11 +23716,13 @@ SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ */ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ + assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; - if( p->mxAlloc>0 && p->zText==p->zBase ){ + if( p->mxAlloc>0 && !isMalloced(p) ){ p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ setStrAccumError(p, STRACCUM_NOMEM); } @@ -22334,8 +23735,10 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ ** Reset an StrAccum string. Reclaim all malloced memory. */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ - if( p->zText!=p->zBase ){ + assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); + if( isMalloced(p) ){ sqlite3DbFree(p->db, p->zText); + p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } p->zText = 0; } @@ -22361,6 +23764,7 @@ SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, i p->nAlloc = n; p->mxAlloc = mx; p->accError = 0; + p->printfFlags = 0; } /* @@ -22374,10 +23778,11 @@ SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list a assert( db!=0 ); sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); + acc.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3VXPrintf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return z; } @@ -22414,7 +23819,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -22459,7 +23864,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int n, char *zBuf, const char } #endif sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); return sqlite3StrAccumFinish(&acc); } SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ @@ -22490,7 +23895,7 @@ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); } @@ -22519,7 +23924,7 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ char zBuf[500]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); @@ -22529,12 +23934,13 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ /* -** variable-argument wrapper around sqlite3VXPrintf(). +** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. */ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ va_list ap; va_start(ap,zFormat); - sqlite3VXPrintf(p, bFlags, zFormat, ap); + sqlite3VXPrintf(p, zFormat, ap); va_end(ap); } @@ -22605,7 +24011,7 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); } va_start(ap, zFormat); - sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1); sqlite3StrAccumFinish(&acc); @@ -22621,96 +24027,151 @@ static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ sqlite3TreeViewLine(p, "%s", zLabel); } - /* -** Generate a human-readable description of a the Select object. +** Generate a human-readable description of a WITH clause. */ -SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ - int n = 0; - pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags - ); - if( p->pSrc && p->pSrc->nSrc ) n++; - if( p->pWhere ) n++; - if( p->pGroupBy ) n++; - if( p->pHaving ) n++; - if( p->pOrderBy ) n++; - if( p->pLimit ) n++; - if( p->pOffset ) n++; - if( p->pPrior ) n++; - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; i<p->pSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ + int i; + if( pWith==0 ) return; + if( pWith->nCte==0 ) return; + if( pWith->pOuter ){ + sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter); + }else{ + sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith); + } + if( pWith->nCte>0 ){ + pView = sqlite3TreeViewPush(pView, 1); + for(i=0; i<pWith->nCte; i++){ StrAccum x; - char zLine[100]; + char zLine[1000]; + const struct Cte *pCte = &pWith->a[i]; sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, 0, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); - } - if( pItem->jointype & JT_LEFT ){ - sqlite3XPrintf(&x, 0, " LEFT-JOIN"); + sqlite3XPrintf(&x, "%s", pCte->zName); + if( pCte->pCols && pCte->pCols->nExpr>0 ){ + char cSep = '('; + int j; + for(j=0; j<pCte->pCols->nExpr; j++){ + sqlite3XPrintf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); + cSep = ','; + } + sqlite3XPrintf(&x, ")"); } + sqlite3XPrintf(&x, " AS"); sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, i<p->pSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } + sqlite3TreeViewItem(pView, zLine, i<pWith->nCte-1); + sqlite3TreeViewSelect(pView, pCte->pSelect, 0); sqlite3TreeViewPop(pView); } sqlite3TreeViewPop(pView); } - if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", (n--)>0); - sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); - } - if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); - } - if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", (n--)>0); - sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); - } - if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); - } - if( p->pPrior ){ - const char *zOp = "UNION"; - switch( p->op ){ - case TK_ALL: zOp = "UNION ALL"; break; - case TK_INTERSECT: zOp = "INTERSECT"; break; - case TK_EXCEPT: zOp = "EXCEPT"; break; - } - sqlite3TreeViewItem(pView, zOp, (n--)>0); - sqlite3TreeViewSelect(pView, p->pPrior, 0); - sqlite3TreeViewPop(pView); +} + + +/* +** Generate a human-readable description of a the Select object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + int cnt = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( p->pWith ){ + sqlite3TreeViewWith(pView, p->pWith, 1); + cnt = 1; + sqlite3TreeViewPush(pView, 1); } + do{ + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, + (int)p->nSelectRow + ); + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + } + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; i<p->pSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3XPrintf(&x, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, i<p->pSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); sqlite3TreeViewPop(pView); } @@ -22785,11 +24246,6 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); break; } - case TK_AS: { - sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } case TK_ID: { sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); break; @@ -22831,6 +24287,12 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_SPAN: { + sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_COLLATE: { sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); @@ -22964,7 +24426,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExprList( }else{ sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; i<pList->nExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + if( j ){ + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + } sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1); + if( j ) sqlite3TreeViewPop(pView); } } sqlite3TreeViewPop(pView); @@ -23176,10 +24644,14 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; }else{ @@ -23198,7 +24670,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; if( p->done ){ *ppOut = p->pOut; rc = SQLITE_OK; @@ -23263,8 +24735,13 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM; - if( sqlite3GlobalConfig.bCoreMutex==0 ){ + if( p==0 ) return SQLITE_NOMEM_BKPT; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ memset(p, 0, sizeof(*p)); }else{ p->xTask = xTask; @@ -23290,9 +24767,9 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ BOOL bRc; assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; if( p->xTask==0 ){ - assert( p->id==GetCurrentThreadId() ); + /* assert( p->id==GetCurrentThreadId() ); */ rc = WAIT_OBJECT_0; assert( p->tid==0 ); }else{ @@ -23338,7 +24815,7 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ p->xTask = xTask; p->pIn = pIn; @@ -23354,7 +24831,7 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ @@ -23365,7 +24842,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ #if defined(SQLITE_TEST) { void *pTstAlloc = sqlite3Malloc(10); - if (!pTstAlloc) return SQLITE_NOMEM; + if (!pTstAlloc) return SQLITE_NOMEM_BKPT; sqlite3_free(pTstAlloc); } #endif @@ -23418,13 +24895,13 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /* #include <assert.h> */ /* #include "vdbeInt.h" */ -#ifndef SQLITE_AMALGAMATION +#if !defined(SQLITE_AMALGAMATION) && SQLITE_BYTEORDER==0 /* ** The following constant value is used by the SQLITE_BIGENDIAN and ** SQLITE_LITTLEENDIAN macros. */ SQLITE_PRIVATE const int sqlite3one = 1; -#endif /* SQLITE_AMALGAMATION */ +#endif /* SQLITE_AMALGAMATION && SQLITE_BYTEORDER==0 */ /* ** This lookup table is used to help decode the first byte of @@ -23612,7 +25089,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired rc = sqlite3VdbeMemMakeWriteable(pMem); if( rc!=SQLITE_OK ){ assert( rc==SQLITE_NOMEM ); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } zIn = (u8*)pMem->z; zTerm = &zIn[pMem->n&~1]; @@ -23654,7 +25131,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired zTerm = &zIn[pMem->n]; zOut = sqlite3DbMallocRaw(pMem->db, len); if( !zOut ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } z = zOut; @@ -23697,7 +25174,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired c = pMem->flags; sqlite3VdbeMemRelease(pMem); - pMem->flags = MEM_Str|MEM_Term|(c&MEM_AffMask); + pMem->flags = MEM_Str|MEM_Term|(c&(MEM_AffMask|MEM_Subtype)); pMem->enc = desiredEnc; pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; @@ -24024,12 +25501,48 @@ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ } /* +** Return the declared type of a column. Or return zDflt if the column +** has no declared type. +** +** The column type is an extra string stored after the zero-terminator on +** the column name if and only if the COLFLAG_HASTYPE flag is set. +*/ +SQLITE_PRIVATE char *sqlite3ColumnType(Column *pCol, char *zDflt){ + if( (pCol->colFlags & COLFLAG_HASTYPE)==0 ) return zDflt; + return pCol->zName + strlen(pCol->zName) + 1; +} + +/* +** Helper function for sqlite3Error() - called rarely. Broken out into +** a separate routine to avoid unnecessary register saves on entry to +** sqlite3Error(). +*/ +static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ + if( db->pErr ) sqlite3ValueSetNull(db->pErr); + sqlite3SystemError(db, err_code); +} + +/* ** Set the current error code to err_code and clear any prior error message. +** Also set iSysErrno (by calling sqlite3System) if the err_code indicates +** that would be appropriate. */ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ assert( db!=0 ); db->errCode = err_code; - if( db->pErr ) sqlite3ValueSetNull(db->pErr); + if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); +} + +/* +** Load the sqlite3.iSysErrno field if that is an appropriate thing +** to do based on the SQLite error code in rc. +*/ +SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ + if( rc==SQLITE_IOERR_NOMEM ) return; + rc &= 0xff; + if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ + db->iSysErrno = sqlite3OsGetLastError(db->pVfs); + } } /* @@ -24056,6 +25569,7 @@ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; + sqlite3SystemError(db, err_code); if( zFormat==0 ){ sqlite3Error(db, err_code); }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ @@ -24148,6 +25662,14 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ return j; } +/* +** Generate a Token object from a string +*/ +SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ + p->z = z; + p->n = sqlite3Strlen30(z); +} + /* Convenient short-hand */ #define UpperToLower sqlite3UpperToLower @@ -24162,16 +25684,25 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ ** independence" that SQLite uses internally when comparing identifiers. */ SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *zLeft, const char *zRight){ - register unsigned char *a, *b; if( zLeft==0 ){ return zRight ? -1 : 0; }else if( zRight==0 ){ return 1; } + return sqlite3StrICmp(zLeft, zRight); +} +SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ + unsigned char *a, *b; + int c; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; - while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return UpperToLower[*a] - UpperToLower[*b]; + for(;;){ + c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; + if( c || *a==0 ) break; + a++; + b++; + } + return c; } SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; @@ -24469,7 +26000,8 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr || nonNum ){ + if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) + || i>19*incr || nonNum ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; @@ -24758,7 +26290,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* a: p0<<28 | p2<<14 | p4 (unmasked) */ if (!(a&0x80)) { - /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* we can skip these cause they were (effectively) done above + ** while calculating s */ /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ /* b &= (0x7f<<14)|(0x7f); */ b = b<<7; @@ -24979,11 +26512,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ ** 64-bit integer. */ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 && ALWAYS(i<9) ); + int i; + for(i=1; (v >>= 7)!=0; i++){ assert( i<10 ); } return i; } @@ -24996,11 +26526,13 @@ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 u32 x; memcpy(&x,p,4); return _byteswap_ulong(x); @@ -25012,10 +26544,12 @@ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ #if SQLITE_BYTEORDER==4321 memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 u32 x = __builtin_bswap32(v); memcpy(p,&x,4); -#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 u32 x = _byteswap_ulong(v); memcpy(p,&x,4); #else @@ -25055,7 +26589,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ char *zBlob; int i; - zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); + zBlob = (char *)sqlite3DbMallocRawNN(db, n/2 + 1); n--; if( zBlob ){ for(i=0; i<n; i+=2){ @@ -25294,8 +26828,14 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. +** +** Note that this routine is only used when one or more of various +** non-standard compile-time options is enabled. */ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ u64 n; @@ -25304,11 +26844,17 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; - if( x>=3 ){ - return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3); - } - return (n+8)>>(3-x); +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + if( x>60 ) return (u64)LARGEST_INT64; +#else + /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input + ** possible to this routine is 310, resulting in a maximum x of 31 */ + assert( x<=60 ); +#endif + return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); } +#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ @@ -25583,175 +27129,179 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ -/* See the mkopcodec.awk script for details. */ -#if !defined(SQLITE_OMIT_EXPLAIN) || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* See the tool/mkopcodec.tcl script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) \ + || defined(VDBE_PROFILE) \ + || defined(SQLITE_DEBUG) #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) # define OpHelp(X) "\0" X #else # define OpHelp(X) #endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ - static const char *const azName[] = { "?", - /* 1 */ "Savepoint" OpHelp(""), - /* 2 */ "AutoCommit" OpHelp(""), - /* 3 */ "Transaction" OpHelp(""), - /* 4 */ "SorterNext" OpHelp(""), - /* 5 */ "PrevIfOpen" OpHelp(""), - /* 6 */ "NextIfOpen" OpHelp(""), - /* 7 */ "Prev" OpHelp(""), - /* 8 */ "Next" OpHelp(""), - /* 9 */ "Checkpoint" OpHelp(""), - /* 10 */ "JournalMode" OpHelp(""), - /* 11 */ "Vacuum" OpHelp(""), - /* 12 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 13 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 14 */ "Goto" OpHelp(""), - /* 15 */ "Gosub" OpHelp(""), - /* 16 */ "Return" OpHelp(""), - /* 17 */ "InitCoroutine" OpHelp(""), - /* 18 */ "EndCoroutine" OpHelp(""), - /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Yield" OpHelp(""), - /* 21 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 22 */ "Halt" OpHelp(""), - /* 23 */ "Integer" OpHelp("r[P2]=P1"), - /* 24 */ "Int64" OpHelp("r[P2]=P4"), - /* 25 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 26 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 27 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 28 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 29 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 30 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 31 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 32 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 33 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 34 */ "CollSeq" OpHelp(""), - /* 35 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), - /* 36 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 38 */ "MustBeInt" OpHelp(""), - /* 39 */ "RealAffinity" OpHelp(""), - /* 40 */ "Cast" OpHelp("affinity(r[P1])"), - /* 41 */ "Permutation" OpHelp(""), - /* 42 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 43 */ "Jump" OpHelp(""), - /* 44 */ "Once" OpHelp(""), - /* 45 */ "If" OpHelp(""), - /* 46 */ "IfNot" OpHelp(""), - /* 47 */ "Column" OpHelp("r[P3]=PX"), - /* 48 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 49 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 50 */ "Count" OpHelp("r[P2]=count()"), - /* 51 */ "ReadCookie" OpHelp(""), - /* 52 */ "SetCookie" OpHelp(""), - /* 53 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 54 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 55 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 56 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 57 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 58 */ "SorterOpen" OpHelp(""), - /* 59 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 60 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 61 */ "Close" OpHelp(""), - /* 62 */ "ColumnsUsed" OpHelp(""), - /* 63 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 64 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 65 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 66 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 67 */ "Seek" OpHelp("intkey=r[P2]"), - /* 68 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 69 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 70 */ "Found" OpHelp("key=r[P3@P4]"), - /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), - /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 73 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 74 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 75 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), - /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), - /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), - /* 79 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), - /* 80 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), - /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), - /* 82 */ "Lt" OpHelp("if r[P1]<r[P3] goto P2"), - /* 83 */ "Ge" OpHelp("if r[P1]>=r[P3] goto P2"), - /* 84 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), - /* 88 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), - /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 95 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 97 */ "String8" OpHelp("r[P2]='P4'"), - /* 98 */ "Delete" OpHelp(""), - /* 99 */ "ResetCount" OpHelp(""), - /* 100 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 101 */ "SorterData" OpHelp("r[P2]=data"), - /* 102 */ "RowKey" OpHelp("r[P2]=key"), - /* 103 */ "RowData" OpHelp("r[P2]=data"), - /* 104 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 105 */ "NullRow" OpHelp(""), - /* 106 */ "Last" OpHelp(""), - /* 107 */ "SorterSort" OpHelp(""), - /* 108 */ "Sort" OpHelp(""), - /* 109 */ "Rewind" OpHelp(""), - /* 110 */ "SorterInsert" OpHelp(""), - /* 111 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 112 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 113 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 114 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 115 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 116 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 117 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 118 */ "Destroy" OpHelp(""), - /* 119 */ "Clear" OpHelp(""), - /* 120 */ "ResetSorter" OpHelp(""), - /* 121 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), - /* 122 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), - /* 123 */ "ParseSchema" OpHelp(""), - /* 124 */ "LoadAnalysis" OpHelp(""), - /* 125 */ "DropTable" OpHelp(""), - /* 126 */ "DropIndex" OpHelp(""), - /* 127 */ "DropTrigger" OpHelp(""), - /* 128 */ "IntegrityCk" OpHelp(""), - /* 129 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 130 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 131 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 132 */ "Program" OpHelp(""), - /* 133 */ "Real" OpHelp("r[P2]=P4"), - /* 134 */ "Param" OpHelp(""), - /* 135 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 136 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 137 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 138 */ "IfPos" OpHelp("if r[P1]>0 goto P2"), - /* 139 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), - /* 140 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]+=P3, goto P2"), - /* 141 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 142 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), - /* 143 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 144 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 145 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 146 */ "IncrVacuum" OpHelp(""), - /* 147 */ "Expire" OpHelp(""), - /* 148 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 149 */ "VBegin" OpHelp(""), - /* 150 */ "VCreate" OpHelp(""), - /* 151 */ "VDestroy" OpHelp(""), - /* 152 */ "VOpen" OpHelp(""), - /* 153 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 154 */ "VNext" OpHelp(""), - /* 155 */ "VRename" OpHelp(""), - /* 156 */ "Pagecount" OpHelp(""), - /* 157 */ "MaxPgcnt" OpHelp(""), - /* 158 */ "Init" OpHelp("Start at P2"), - /* 159 */ "Noop" OpHelp(""), - /* 160 */ "Explain" OpHelp(""), + static const char *const azName[] = { + /* 0 */ "Savepoint" OpHelp(""), + /* 1 */ "AutoCommit" OpHelp(""), + /* 2 */ "Transaction" OpHelp(""), + /* 3 */ "SorterNext" OpHelp(""), + /* 4 */ "PrevIfOpen" OpHelp(""), + /* 5 */ "NextIfOpen" OpHelp(""), + /* 6 */ "Prev" OpHelp(""), + /* 7 */ "Next" OpHelp(""), + /* 8 */ "Checkpoint" OpHelp(""), + /* 9 */ "JournalMode" OpHelp(""), + /* 10 */ "Vacuum" OpHelp(""), + /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 13 */ "Goto" OpHelp(""), + /* 14 */ "Gosub" OpHelp(""), + /* 15 */ "Return" OpHelp(""), + /* 16 */ "InitCoroutine" OpHelp(""), + /* 17 */ "EndCoroutine" OpHelp(""), + /* 18 */ "Yield" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 21 */ "Halt" OpHelp(""), + /* 22 */ "Integer" OpHelp("r[P2]=P1"), + /* 23 */ "Int64" OpHelp("r[P2]=P4"), + /* 24 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 25 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 26 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 27 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 28 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 29 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 30 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 31 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 32 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 33 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 34 */ "CollSeq" OpHelp(""), + /* 35 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), + /* 36 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), + /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 38 */ "MustBeInt" OpHelp(""), + /* 39 */ "RealAffinity" OpHelp(""), + /* 40 */ "Cast" OpHelp("affinity(r[P1])"), + /* 41 */ "Permutation" OpHelp(""), + /* 42 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 43 */ "Jump" OpHelp(""), + /* 44 */ "Once" OpHelp(""), + /* 45 */ "If" OpHelp(""), + /* 46 */ "IfNot" OpHelp(""), + /* 47 */ "Column" OpHelp("r[P3]=PX"), + /* 48 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 49 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 50 */ "Count" OpHelp("r[P2]=count()"), + /* 51 */ "ReadCookie" OpHelp(""), + /* 52 */ "SetCookie" OpHelp(""), + /* 53 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 54 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 55 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 56 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 57 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 58 */ "SorterOpen" OpHelp(""), + /* 59 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 60 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 61 */ "Close" OpHelp(""), + /* 62 */ "ColumnsUsed" OpHelp(""), + /* 63 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 64 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 65 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 66 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 67 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 68 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 69 */ "Found" OpHelp("key=r[P3@P4]"), + /* 70 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 73 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 74 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 75 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), + /* 79 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), + /* 80 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), + /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), + /* 82 */ "Lt" OpHelp("if r[P1]<r[P3] goto P2"), + /* 83 */ "Ge" OpHelp("if r[P1]>=r[P3] goto P2"), + /* 84 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), + /* 88 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), + /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 95 */ "Delete" OpHelp(""), + /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), + /* 97 */ "String8" OpHelp("r[P2]='P4'"), + /* 98 */ "ResetCount" OpHelp(""), + /* 99 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 100 */ "SorterData" OpHelp("r[P2]=data"), + /* 101 */ "RowKey" OpHelp("r[P2]=key"), + /* 102 */ "RowData" OpHelp("r[P2]=data"), + /* 103 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 104 */ "NullRow" OpHelp(""), + /* 105 */ "Last" OpHelp(""), + /* 106 */ "SorterSort" OpHelp(""), + /* 107 */ "Sort" OpHelp(""), + /* 108 */ "Rewind" OpHelp(""), + /* 109 */ "SorterInsert" OpHelp(""), + /* 110 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 111 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 112 */ "Seek" OpHelp("Move P3 to P1.rowid"), + /* 113 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 114 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 115 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 116 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 117 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 118 */ "Destroy" OpHelp(""), + /* 119 */ "Clear" OpHelp(""), + /* 120 */ "ResetSorter" OpHelp(""), + /* 121 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), + /* 122 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), + /* 123 */ "ParseSchema" OpHelp(""), + /* 124 */ "LoadAnalysis" OpHelp(""), + /* 125 */ "DropTable" OpHelp(""), + /* 126 */ "DropIndex" OpHelp(""), + /* 127 */ "DropTrigger" OpHelp(""), + /* 128 */ "IntegrityCk" OpHelp(""), + /* 129 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 130 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 131 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 132 */ "Program" OpHelp(""), + /* 133 */ "Real" OpHelp("r[P2]=P4"), + /* 134 */ "Param" OpHelp(""), + /* 135 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 136 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 137 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 138 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 139 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 140 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), + /* 141 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 142 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), + /* 143 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 144 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 145 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 146 */ "IncrVacuum" OpHelp(""), + /* 147 */ "Expire" OpHelp(""), + /* 148 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 149 */ "VBegin" OpHelp(""), + /* 150 */ "VCreate" OpHelp(""), + /* 151 */ "VDestroy" OpHelp(""), + /* 152 */ "VOpen" OpHelp(""), + /* 153 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 154 */ "VNext" OpHelp(""), + /* 155 */ "VRename" OpHelp(""), + /* 156 */ "Pagecount" OpHelp(""), + /* 157 */ "MaxPgcnt" OpHelp(""), + /* 158 */ "Init" OpHelp("Start at P2"), + /* 159 */ "CursorHint" OpHelp(""), + /* 160 */ "Noop" OpHelp(""), + /* 161 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -25832,6 +27382,19 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # endif #endif +/* Use pread() and pwrite() if they are available */ +#if defined(__APPLE__) +# define HAVE_PREAD 1 +# define HAVE_PWRITE 1 +#endif +#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) +# undef USE_PREAD +# define USE_PREAD64 1 +#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) +# undef USE_PREAD64 +# define USE_PREAD 1 +#endif + /* ** standard include files. */ @@ -25910,6 +27473,11 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define MAX_PATHNAME 512 +/* +** Maximum supported symbolic links +*/ +#define SQLITE_MAX_SYMLINKS 100 + /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ #define osGetpid(X) (pid_t)getpid() @@ -26019,8 +27587,6 @@ static pid_t randomnessPid = 0; #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ -#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings issued */ -#define UNIXFILE_BLOCK 0x0200 /* Next SHM lock might block */ /* ** Include code that is common to all os_*.c files @@ -26064,8 +27630,8 @@ static pid_t randomnessPid = 0; */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -26175,14 +27741,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -26208,17 +27774,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -26283,19 +27849,6 @@ static int posixOpen(const char *zFile, int flags, int mode){ return open(zFile, flags, mode); } -/* -** On some systems, calls to fchown() will trigger a message in a security -** log if they come from non-root processes. So avoid calling fchown() if -** we are not running as root. -*/ -static int posixFchown(int fd, uid_t uid, gid_t gid){ -#if OS_VXWORKS - return 0; -#else - return geteuid() ? 0 : fchown(fd,uid,gid); -#endif -} - /* Forward reference */ static int openDirectory(const char*, int*); static int unixGetpagesize(void); @@ -26382,7 +27935,7 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[13].pCurrent) - { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -26404,29 +27957,74 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) - { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, +#if defined(HAVE_FCHOWN) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) + #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 - { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, -#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#else + { "mmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, -#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) +#else + { "munmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent) -#if HAVE_MREMAP +#if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[24].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, -#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) +#else + { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) +#if defined(HAVE_READLINK) + { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) + +#if defined(HAVE_LSTAT) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#else + { "lstat", (sqlite3_syscall_ptr)0, 0 }, #endif +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) }; /* End of the overrideable system calls */ + +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int robustFchown(int fd, uid_t uid, gid_t gid){ +#if defined(HAVE_FCHOWN) + return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; +#endif +} + /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the @@ -26711,23 +28309,12 @@ static int robust_ftruncate(int h, sqlite3_int64 sz){ ** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); switch (posixError) { -#if 0 - /* At one point this code was not commented out. In theory, this branch - ** should never be hit, as this function should only be called after - ** a locking-related function (i.e. fcntl()) has returned non-zero with - ** the value of errno as the first argument. Since a system call has failed, - ** errno should be non-zero. - ** - ** Despite this, if errno really is zero, we still don't want to return - ** SQLITE_OK. The system call failed, and *some* SQLite error should be - ** propagated back to the caller. Commenting this branch out means errno==0 - ** will be handled by the "default:" case below. - */ - case 0: - return SQLITE_OK; -#endif - + case EACCES: case EAGAIN: case ETIMEDOUT: case EBUSY: @@ -26737,41 +28324,9 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { * introspection, in which it actually means what it says */ return SQLITE_BUSY; - case EACCES: - /* EACCES is like EAGAIN during locking operations, but not any other time*/ - if( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || - (sqliteIOErr == SQLITE_IOERR_RDLOCK) || - (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ - return SQLITE_BUSY; - } - /* else fall through */ case EPERM: return SQLITE_PERM; -#if EOPNOTSUPP!=ENOTSUP - case EOPNOTSUPP: - /* something went terribly awry, unless during file system support - * introspection, in which it actually means what it says */ -#endif -#ifdef ENOTSUP - case ENOTSUP: - /* invalid fd, unless during file system support introspection, in which - * it actually means what it says */ -#endif - case EIO: - case EBADF: - case EINVAL: - case ENOTCONN: - case ENODEV: - case ENXIO: - case ENOENT: -#ifdef ESTALE /* ESTALE is not defined on Interix systems */ - case ESTALE: -#endif - case ENOSYS: - /* these should force the client to close the file and reconnect */ - default: return sqliteIOErr; } @@ -27055,7 +28610,7 @@ static unixInodeInfo *inodeList = 0; /* ** -** This function - unixLogError_x(), is only ever called via the macro +** This function - unixLogErrorAtLine(), is only ever called via the macro ** unixLogError(). ** ** It is invoked after an error occurs in an OS function and errno has been @@ -27224,7 +28779,7 @@ static int findInodeInfo( rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); -#ifdef EOVERFLOW +#if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; #endif return SQLITE_IOERR; @@ -27269,7 +28824,7 @@ static int findInodeInfo( if( pInode==0 ){ pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); @@ -27311,30 +28866,25 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; - if( pFile->ctrlFlags & UNIXFILE_WARNED ){ - /* One or more of the following warnings have already been issued. Do not - ** repeat them so as not to clutter the error log */ - return; - } + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } - if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + if( buf.st_nlink==0 ){ sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( buf.st_nlink>1 ){ sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( fileHasMoved(pFile) ){ sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); - pFile->ctrlFlags |= UNIXFILE_WARNED; return; } } @@ -27354,6 +28904,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); unixEnterMutex(); /* Because pFile->pInode is shared across threads */ /* Check if a thread in this process holds such a lock */ @@ -27410,9 +28961,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); assert( pInode!=0 ); - if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) - && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) - ){ + if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; assert( pInode->nLock==0 ); @@ -27764,9 +29313,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(rc) ){ - storeLastErrno(pFile, tErrno); - } + storeLastErrno(pFile, tErrno); goto end_unlock; } lock.l_type = F_RDLCK; @@ -27788,9 +29335,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(rc) ){ - storeLastErrno(pFile, tErrno); - } + storeLastErrno(pFile, tErrno); goto end_unlock; } }else @@ -28041,17 +29586,7 @@ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - /* Either this connection or some other connection in the same process - ** holds a lock on the file. No need to check further. */ - reserved = 1; - }else{ - /* The lock is held if and only if the lockfile exists */ - const char *zLockFile = (const char*)pFile->lockingContext; - reserved = osAccess(zLockFile, 0)==0; - } + reserved = osAccess((const char*)pFile->lockingContext, 0)==0; OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; @@ -28113,7 +29648,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { rc = SQLITE_BUSY; } else { rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ + if( rc!=SQLITE_BUSY ){ storeLastErrno(pFile, tErrno); } } @@ -28160,14 +29695,12 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); rc = osRmdir(zLockFile); - if( rc<0 && errno==ENOTDIR ) rc = osUnlink(zLockFile); if( rc<0 ){ int tErrno = errno; - rc = 0; - if( ENOENT != tErrno ){ + if( tErrno==ENOENT ){ + rc = SQLITE_OK; + }else{ rc = SQLITE_IOERR_UNLOCK; - } - if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } return rc; @@ -28180,14 +29713,11 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. Make sure the lock has been released before closing. */ static int dotlockClose(sqlite3_file *id) { - int rc = SQLITE_OK; - if( id ){ - unixFile *pFile = (unixFile*)id; - dotlockUnlock(id, NO_LOCK); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - } - return rc; + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + return closeUnixFile(id); } /****************** End of the dot-file lock implementation ******************* ******************************************************************************/ @@ -28253,10 +29783,8 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ int tErrno = errno; /* unlock failed with an error */ lrc = SQLITE_IOERR_UNLOCK; - if( IS_LOCK_ERROR(lrc) ){ - storeLastErrno(pFile, tErrno); - rc = lrc; - } + storeLastErrno(pFile, tErrno); + rc = lrc; } } else { int tErrno = errno; @@ -28389,12 +29917,9 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. */ static int flockClose(sqlite3_file *id) { - int rc = SQLITE_OK; - if( id ){ - flockUnlock(id, NO_LOCK); - rc = closeUnixFile(id); - } - return rc; + assert( id!=0 ); + flockUnlock(id, NO_LOCK); + return closeUnixFile(id); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ @@ -29019,23 +30544,22 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { */ static int afpClose(sqlite3_file *id) { int rc = SQLITE_OK; - if( id ){ - unixFile *pFile = (unixFile*)id; - afpUnlock(id, NO_LOCK); - unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); - } - releaseInodeInfo(pFile); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - unixLeaveMutex(); + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pInode && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); return rc; } @@ -29105,7 +30629,6 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); - cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -29115,13 +30638,9 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ SimulateIOError( got = -1 ); #else newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - storeLastErrno((unixFile*)id, errno); - }else{ - storeLastErrno((unixFile*)id, 0); - } + SimulateIOError( newOffset = -1 ); + if( newOffset<0 ){ + storeLastErrno((unixFile*)id, errno); return -1; } got = osRead(id->h, pBuf, cnt); @@ -29220,6 +30739,7 @@ static int seekAndWriteFd( assert( nBuf==(nBuf&0x1ffff) ); assert( fd>2 ); + assert( piErrno!=0 ); nBuf &= 0x1ffff; TIMER_START; @@ -29230,11 +30750,10 @@ static int seekAndWriteFd( #else do{ i64 iSeek = lseek(fd, iOff, SEEK_SET); - SimulateIOError( iSeek-- ); - - if( iSeek!=iOff ){ - if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); - return -1; + SimulateIOError( iSeek = -1 ); + if( iSeek<0 ){ + rc = -1; + break; } rc = osWrite(fd, pBuf, nBuf); }while( rc<0 && errno==EINTR ); @@ -29243,7 +30762,7 @@ static int seekAndWriteFd( TIMER_END; OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); - if( rc<0 && piErrno ) *piErrno = errno; + if( rc<0 ) *piErrno = errno; return rc; } @@ -29306,7 +30825,7 @@ static int unixWrite( } #endif -#if SQLITE_MAX_MMAP_SIZE>0 +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offset<pFile->mmapSize ){ @@ -29322,8 +30841,8 @@ static int unixWrite( } } #endif - - while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))<amt && wrote>0 ){ amt -= wrote; offset += wrote; pBuf = &((char*)pBuf)[wrote]; @@ -29331,7 +30850,7 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>0 ){ + if( amt>wrote ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; @@ -29427,10 +30946,15 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ #endif /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op + ** no-op. But go ahead and call fstat() to validate the file + ** descriptor as we need a method to provoke a failure during + ** coverate testing. */ #ifdef SQLITE_NO_SYNC - rc = SQLITE_OK; + { + struct stat buf; + rc = osFstat(fd, &buf); + } #elif HAVE_FULLFSYNC if( fullSync ){ rc = osFcntl(fd, F_FULLFSYNC, 0); @@ -29496,16 +31020,20 @@ static int openDirectory(const char *zFilename, int *pFd){ char zDirname[MAX_PATHNAME+1]; sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); - for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); + for(ii=(int)strlen(zDirname); ii>0 && zDirname[ii]!='/'; ii--); if( ii>0 ){ zDirname[ii] = '\0'; - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); - if( fd>=0 ){ - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); - } + }else{ + if( zDirname[0]!='/' ) zDirname[0] = '.'; + zDirname[1] = 0; + } + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } *pFd = fd; - return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); + if( fd>=0 ) return SQLITE_OK; + return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname); } /* @@ -29558,10 +31086,11 @@ static int unixSync(sqlite3_file *id, int flags){ OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, HAVE_FULLFSYNC, isFullsync)); rc = osOpenDirectory(pFile->zPath, &dirfd); - if( rc==SQLITE_OK && dirfd>=0 ){ + if( rc==SQLITE_OK ){ full_fsync(dirfd, 0, 0); robust_close(pFile, dirfd, __LINE__); - }else if( rc==SQLITE_CANTOPEN ){ + }else{ + assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; @@ -29693,18 +31222,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ int nWrite = 0; /* Number of bytes written by seekAndWrite */ i64 iWrite; /* Next offset to write to */ - iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + iWrite = (buf.st_size/nBlk)*nBlk + nBlk - 1; assert( iWrite>=buf.st_size ); - assert( (iWrite/nBlk)==((buf.st_size+nBlk-1)/nBlk) ); assert( ((iWrite+1)%nBlk)==0 ); - for(/*no-op*/; iWrite<nSize; iWrite+=nBlk ){ + for(/*no-op*/; iWrite<nSize+nBlk-1; iWrite+=nBlk ){ + if( iWrite>=nSize ) iWrite = nSize - 1; nWrite = seekAndWrite(pFile, iWrite, "", 1); if( nWrite!=1 ) return SQLITE_IOERR_WRITE; } - if( nWrite==0 || (nSize%nBlk) ){ - nWrite = seekAndWrite(pFile, nSize-1, "", 1); - if( nWrite!=1 ) return SQLITE_IOERR_WRITE; - } #endif } } @@ -29752,10 +31277,6 @@ static int unixGetTempname(int nBuf, char *zBuf); static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ - case SQLITE_FCNTL_WAL_BLOCK: { - /* pFile->ctrlFlags |= UNIXFILE_BLOCK; // Deferred feature */ - return SQLITE_OK; - } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; @@ -30082,10 +31603,9 @@ static int unixShmSystemLock( assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ - assert( n>=1 && n<SQLITE_SHM_NLOCK ); + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); if( pShmNode->h>=0 ){ - int lkType; /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); f.l_type = lockType; @@ -30093,10 +31613,8 @@ static int unixShmSystemLock( f.l_start = ofst; f.l_len = n; - lkType = (pFile->ctrlFlags & UNIXFILE_BLOCK)!=0 ? F_SETLKW : F_SETLK; - rc = osFcntl(pShmNode->h, lkType, &f); + rc = osFcntl(pShmNode->h, F_SETLK, &f); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; - pFile->ctrlFlags &= ~UNIXFILE_BLOCK; } /* Update the global lock state and do debug tracing */ @@ -30163,7 +31681,7 @@ static int unixShmRegionPerMap(void){ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); - if( p && p->nRef==0 ){ + if( p && ALWAYS(p->nRef==0) ){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); @@ -30230,7 +31748,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Allocate space for the new unixShm object. */ p = sqlite3_malloc64( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); @@ -30250,7 +31768,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** a new *-shm file is created, an attempt will be made to create it ** with the same permissions. */ - if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ + if( osFstat(pDbFd->h, &sStat) ){ rc = SQLITE_IOERR_FSTAT; goto shm_open_err; } @@ -30262,7 +31780,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ #endif pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); @@ -30280,7 +31798,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->pInode = pDbFd->pInode; pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } @@ -30300,7 +31818,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -30437,7 +31955,8 @@ static int unixShmMap( /* Write to the last byte of each newly allocated or extended page */ assert( (nByte % pgsz)==0 ); for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ - if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + int x = 0; + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out; @@ -30452,7 +31971,7 @@ static int unixShmMap( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->apRegion = apNew; @@ -30472,7 +31991,7 @@ static int unixShmMap( }else{ pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } memset(pMem, 0, szRegion); @@ -30620,7 +32139,8 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - unixEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -30730,7 +32250,9 @@ static void unixRemapfile( assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); +#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; +#endif if( pOrig ){ #if HAVE_MREMAP @@ -30802,17 +32324,14 @@ static void unixRemapfile( ** recreated as a result of outstanding references) or an SQLite error ** code otherwise. */ -static int unixMapfile(unixFile *pFd, i64 nByte){ - i64 nMap = nByte; - int rc; - +static int unixMapfile(unixFile *pFd, i64 nMap){ assert( nMap>=0 || pFd->nFetchOut==0 ); + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( pFd->nFetchOut>0 ) return SQLITE_OK; if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(pFd->h, &statbuf); - if( rc!=SQLITE_OK ){ + if( osFstat(pFd->h, &statbuf) ){ return SQLITE_IOERR_FSTAT; } nMap = statbuf.st_size; @@ -30821,12 +32340,9 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ nMap = pFd->mmapSizeMax; } + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ - if( nMap>0 ){ - unixRemapfile(pFd, nMap); - }else{ - unixUnmapfile(pFd); - } + unixRemapfile(pFd, nMap); } return SQLITE_OK; @@ -31253,7 +32769,7 @@ static int fillInUnixFile( pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ ctrlFlags |= UNIXFILE_NOLOCK; - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } #endif @@ -31309,7 +32825,7 @@ static int fillInUnixFile( afpLockingContext *pCtx; pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ /* NB: zFilename exists and remains valid until the file is closed ** according to requirement F11141. So we do not need to make a @@ -31339,7 +32855,7 @@ static int fillInUnixFile( nFilename = (int)strlen(zFilename) + 6; zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); } @@ -31362,7 +32878,7 @@ static int fillInUnixFile( if( zSemName[n]=='/' ) zSemName[n] = '_'; pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); if( pNew->pInode->pSem == SEM_FAILED ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; pNew->pInode->aSemName[0] = '\0'; } } @@ -31397,19 +32913,17 @@ static const char *unixTempFileDir(void){ static const char *azDirs[] = { 0, 0, - 0, "/var/tmp", "/usr/tmp", "/tmp", - 0 /* List terminator */ + "." }; unsigned int i; struct stat buf; - const char *zDir = 0; + const char *zDir = sqlite3_temp_directory; - azDirs[0] = sqlite3_temp_directory; - if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR"); + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); for(i=0; i<sizeof(azDirs)/sizeof(azDirs[0]); zDir=azDirs[i++]){ if( zDir==0 ) continue; if( osStat(zDir, &buf) ) continue; @@ -31426,12 +32940,8 @@ static const char *unixTempFileDir(void){ ** pVfs->mxPathname bytes. */ static int unixGetTempname(int nBuf, char *zBuf){ - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - unsigned int i, j; const char *zDir; + int iLimit = 0; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -31440,24 +32950,14 @@ static int unixGetTempname(int nBuf, char *zBuf){ SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); - if( zDir==0 ) zDir = "."; - - /* Check that the output buffer is large enough for the temporary file - ** name. If it is not, return SQLITE_ERROR. - */ - if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 18) >= (size_t)nBuf ){ - return SQLITE_ERROR; - } - do{ - sqlite3_snprintf(nBuf-18, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); - j = (int)strlen(zBuf); - sqlite3_randomness(15, &zBuf[j]); - for(i=0; i<15; i++, j++){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; - } - zBuf[j] = 0; - zBuf[j+1] = 0; + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; }while( osAccess(zBuf,0)==0 ); return SQLITE_OK; } @@ -31579,16 +33079,19 @@ static int findCreateFileMode( ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; -#ifdef SQLITE_ENABLE_8_3_NAMES - while( nDb>0 && sqlite3Isalnum(zPath[nDb]) ) nDb--; - if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; -#else while( zPath[nDb]!='-' ){ +#ifndef SQLITE_ENABLE_8_3_NAMES + /* In the normal case (8+3 filenames disabled) the journal filename + ** is guaranteed to contain a '-' character. */ assert( nDb>0 ); - assert( zPath[nDb]!='\n' ); + assert( sqlite3Isalnum(zPath[nDb]) ); +#else + /* If 8+3 names are possible, then the journal file might not contain + ** a '-' character. So check for that case and return early. */ + if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; +#endif nDb--; } -#endif memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; @@ -31716,7 +33219,7 @@ static int unixOpen( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } p->pUnused = pUnused; @@ -31729,7 +33232,7 @@ static int unixOpen( }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ assert(isDelete && !syncDir); - rc = unixGetTempname(MAX_PATHNAME+2, zTmpname); + rc = unixGetTempname(pVfs->mxPathname, zTmpname); if( rc!=SQLITE_OK ){ return rc; } @@ -31762,7 +33265,8 @@ static int unixOpen( } fd = robust_open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); - if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ + assert( !isExclusive || (openFlags & O_CREAT)!=0 ); + if( fd<0 && errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); @@ -31781,7 +33285,7 @@ static int unixOpen( ** the same as the original database. */ if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - osFchown(fd, uid, gid); + robustFchown(fd, uid, gid); } } assert( fd>=0 ); @@ -31801,7 +33305,7 @@ static int unixOpen( zPath = sqlite3_mprintf("%s", zName); if( zPath==0 ){ robust_close(p, fd, __LINE__); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } #else osUnlink(zName); @@ -31812,9 +33316,6 @@ static int unixOpen( p->openFlags = openFlags; } #endif - - noLock = eType!=SQLITE_OPEN_MAIN_DB; - #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ @@ -31833,6 +33334,7 @@ static int unixOpen( /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -31909,16 +33411,12 @@ static int unixDelete( int fd; rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ -#if OS_VXWORKS - if( fsync(fd)==-1 ) -#else - if( fsync(fd) ) -#endif - { + if( full_fsync(fd,0,0) ){ rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } robust_close(0, fd, __LINE__); - }else if( rc==SQLITE_CANTOPEN ){ + }else{ + assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } } @@ -31942,33 +33440,49 @@ static int unixAccess( int flags, /* What do we want to learn about the zPath file? */ int *pResOut /* Write result boolean here */ ){ - int amode = 0; UNUSED_PARAMETER(NotUsed); SimulateIOError( return SQLITE_IOERR_ACCESS; ); - switch( flags ){ - case SQLITE_ACCESS_EXISTS: - amode = F_OK; - break; - case SQLITE_ACCESS_READWRITE: - amode = W_OK|R_OK; - break; - case SQLITE_ACCESS_READ: - amode = R_OK; - break; + assert( pResOut!=0 ); - default: - assert(!"Invalid flags argument"); - } - *pResOut = (osAccess(zPath, amode)==0); - if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + + if( flags==SQLITE_ACCESS_EXISTS ){ struct stat buf; - if( 0==osStat(zPath, &buf) && buf.st_size==0 ){ - *pResOut = 0; - } + *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); + }else{ + *pResOut = osAccess(zPath, W_OK|R_OK)==0; } return SQLITE_OK; } +/* +** +*/ +static int mkFullPathname( + const char *zPath, /* Input path */ + char *zOut, /* Output buffer */ + int nOut /* Allocated size of buffer zOut */ +){ + int nPath = sqlite3Strlen30(zPath); + int iOff = 0; + if( zPath[0]!='/' ){ + if( osGetcwd(zOut, nOut-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + iOff = sqlite3Strlen30(zOut); + zOut[iOff++] = '/'; + } + if( (iOff+nPath+1)>nOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + return SQLITE_OK; +} /* ** Turn a relative pathname into a full pathname. The relative path @@ -31985,6 +33499,17 @@ static int unixFullPathname( int nOut, /* Size of output buffer in bytes */ char *zOut /* Output buffer */ ){ +#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) + return mkFullPathname(zPath, zOut, nOut); +#else + int rc = SQLITE_OK; + int nByte; + int nLink = 1; /* Number of symbolic links followed so far */ + const char *zIn = zPath; /* Input path for each iteration of loop */ + char *zDel = 0; + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -31993,21 +33518,62 @@ static int unixFullPathname( */ SimulateIOError( return SQLITE_ERROR ); - assert( pVfs->mxPathname==MAX_PATHNAME ); - UNUSED_PARAMETER(pVfs); + do { - zOut[nOut-1] = '\0'; - if( zPath[0]=='/' ){ - sqlite3_snprintf(nOut, zOut, "%s", zPath); - }else{ - int nCwd; - if( osGetcwd(zOut, nOut-1)==0 ){ - return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + /* Call stat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + struct stat buf; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else{ + bLink = S_ISLNK(buf.st_mode); } - nCwd = (int)strlen(zOut); - sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); - } - return SQLITE_OK; + + if( bLink ){ + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nOut-1); + if( nByte<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + + assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zOut ){ + rc = mkFullPathname(zIn, zOut, nOut); + } + if( bLink==0 ) break; + zIn = zOut; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + return rc; +#endif /* HAVE_READLINK && HAVE_LSTAT */ } @@ -32176,11 +33742,8 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; #else struct timeval sNow; - if( gettimeofday(&sNow, 0)==0 ){ - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; - }else{ - rc = SQLITE_ERROR; - } + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; #endif #ifdef SQLITE_TEST @@ -32192,6 +33755,7 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ return rc; } +#ifndef SQLITE_OMIT_DEPRECATED /* ** Find the current time (in Universal Coordinated Time). Write the ** current time and date as a Julian Day number into *prNow and @@ -32205,19 +33769,21 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ *prNow = i/86400000.0; return rc; } +#else +# define unixCurrentTime 0 +#endif /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return 0; + return errno; } @@ -32472,7 +34038,7 @@ static int proxyCreateLockPath(const char *lockPath){ } buf[i] = lockPath[i]; } - OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, osGetpid(0))); + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); return 0; } @@ -32508,7 +34074,7 @@ static int proxyCreateUnixFile( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } if( fd<0 ){ @@ -32541,7 +34107,7 @@ static int proxyCreateUnixFile( pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); @@ -32884,7 +34450,7 @@ static int proxyTakeConch(unixFile *pFile){ writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); - fsync(conchFile->h); + full_fsync(conchFile->h,0,0); /* If we created a new conch file (not just updated the contents of a ** valid conch file), try to match the permissions of the database */ @@ -32954,7 +34520,7 @@ static int proxyTakeConch(unixFile *pFile){ if( tempLockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); if( !pCtx->lockProxyPath ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -33019,7 +34585,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ ** the name of the original database file. */ *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(conchPath, dbPath, len+1); @@ -33135,7 +34701,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pCtx, 0, sizeof(*pCtx)); @@ -33171,7 +34737,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { if( rc==SQLITE_OK ){ pCtx->dbPath = sqlite3DbStrDup(0, dbPath); if( pCtx->dbPath==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK ){ @@ -33357,7 +34923,7 @@ static int proxyUnlock(sqlite3_file *id, int eFileLock) { ** Close a file that uses proxy locks. */ static int proxyClose(sqlite3_file *id) { - if( id ){ + if( ALWAYS(id) ){ unixFile *pFile = (unixFile*)id; proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *lockProxy = pCtx->lockProxy; @@ -33501,7 +35067,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==25 ); + assert( ArraySize(aSyscall)==28 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ @@ -33584,8 +35150,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -33695,14 +35261,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -33728,17 +35294,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif +#endif /* defined(SQLITE_TEST) */ /* ** When testing, keep a count of the number of open files. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif +#endif /* defined(SQLITE_TEST) */ #endif /* !defined(_OS_COMMON_H_) */ @@ -33801,6 +35367,10 @@ SQLITE_API int sqlite3_open_file_count = 0; # define NTDDI_WINBLUE 0x06030000 #endif +#ifndef NTDDI_WINTHRESHOLD +# define NTDDI_WINTHRESHOLD 0x06040000 +#endif + /* ** Check to see if the GetVersionEx[AW] functions are deprecated on the ** target system. GetVersionEx was first deprecated in Win8.1. @@ -33814,6 +35384,19 @@ SQLITE_API int sqlite3_open_file_count = 0; #endif /* +** Check to see if the CreateFileMappingA function is supported on the +** target system. It is unavailable when using "mincore.lib" on Win10. +** When compiling for Windows 10, always assume "mincore.lib" is in use. +*/ +#ifndef SQLITE_WIN32_CREATEFILEMAPPINGA +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINTHRESHOLD +# define SQLITE_WIN32_CREATEFILEMAPPINGA 0 +# else +# define SQLITE_WIN32_CREATEFILEMAPPINGA 1 +# endif +#endif + +/* ** This constant should already be defined (in the "WinDef.h" SDK file). */ #ifndef MAX_PATH @@ -34053,10 +35636,22 @@ struct winFile { #endif /* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + +/* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) #endif @@ -34219,8 +35814,9 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ + SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -34450,8 +36046,7 @@ static struct win_syscall { #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_GETVERSIONEX) && \ - SQLITE_WIN32_GETVERSIONEX +#if defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_GETVERSIONEX { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, #else { "GetVersionExA", (SYSCALL)0, 0 }, @@ -34461,7 +36056,7 @@ static struct win_syscall { LPOSVERSIONINFOA))aSyscall[34].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX + SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else { "GetVersionExW", (SYSCALL)0, 0 }, @@ -34930,7 +36525,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_compact_heap(LPUINT pnLargest){ if( lastErrno==NO_ERROR ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapCompact (no space), heap=%p", (void*)hHeap); - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_log(SQLITE_ERROR, "failed to HeapCompact (%lu), heap=%p", osGetLastError(), (void*)hHeap); @@ -35072,7 +36667,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** the LockFileEx() API. */ -#if !defined(SQLITE_WIN32_GETVERSIONEX) || !SQLITE_WIN32_GETVERSIONEX +#if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) #elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) @@ -35093,7 +36688,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void){ ** kernel. */ return 1; -#elif defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX +#elif SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -35250,7 +36845,7 @@ static int winMemInit(void *pAppData){ "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, dwMaximumSize); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = TRUE; assert( pWinMemData->bOwned ); @@ -35260,7 +36855,7 @@ static int winMemInit(void *pAppData){ if( !pWinMemData->hHeap ){ sqlite3_log(SQLITE_NOMEM, "failed to GetProcessHeap (%lu)", osGetLastError()); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = FALSE; assert( !pWinMemData->bOwned ); @@ -35497,7 +37092,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_set_directory(DWORD type, LPCWSTR zV if( zValue && zValue[0] ){ zValueUtf8 = winUnicodeToUtf8(zValue); if ( zValueUtf8==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } sqlite3_free(*ppDirectory); @@ -35774,7 +37369,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ zName = winUtf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Initialize the local lockdata */ @@ -36327,7 +37922,7 @@ static int winWrite( "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); -#if SQLITE_MAX_MMAP_SIZE>0 +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offset<pFile->mmapSize ){ @@ -36875,7 +38470,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ res = 1; OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); }else{ - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0); if( res ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } @@ -36965,7 +38560,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -37323,12 +38918,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); - if( p==0 ) return SQLITE_IOERR_NOMEM; + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); @@ -37355,7 +38950,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shm_open_err; } @@ -37575,8 +39170,8 @@ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ winShmLeaveMutex(); } @@ -37660,7 +39255,7 @@ static int winShmMap( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->aRegion = apNew; @@ -37677,7 +39272,7 @@ static int winShmMap( hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); @@ -37821,17 +39416,19 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ DWORD flags = FILE_MAP_READ; winUnmapfile(pFd); +#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } +#endif #if SQLITE_OS_WINRT pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -38088,7 +39685,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Figure out the effective temporary directory. First, check if one @@ -38146,7 +39743,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ sqlite3_snprintf(nMax, zBuf, "%s", zDir); @@ -38159,7 +39756,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A, zDir, @@ -38180,7 +39777,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zConverted); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); @@ -38198,7 +39795,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zWidePath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathW(nMax, zWidePath)==0 ){ sqlite3_free(zWidePath); @@ -38216,7 +39813,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zWidePath); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -38226,7 +39823,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zMbcsPath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathA(nMax, zMbcsPath)==0 ){ sqlite3_free(zBuf); @@ -38241,7 +39838,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ }else{ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #endif /* SQLITE_WIN32_HAS_ANSI */ @@ -38433,7 +40030,7 @@ static int winOpen( if( zConverted==0 ){ sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ @@ -38633,7 +40230,7 @@ static int winDelete( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ do { @@ -38741,7 +40338,7 @@ static int winAccess( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ int cnt = 0; @@ -38868,7 +40465,7 @@ static int winFullPathname( */ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | @@ -38880,7 +40477,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", sqlite3_data_directory, winGetDirSep(), zUtf8); @@ -38890,7 +40487,7 @@ static int winFullPathname( }else{ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), @@ -38902,7 +40499,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); sqlite3_free(zUtf8); @@ -38962,7 +40559,7 @@ static int winFullPathname( } zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ LPWSTR zTemp; @@ -38976,7 +40573,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -39002,7 +40599,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -39021,7 +40618,7 @@ static int winFullPathname( sqlite3_free(zOut); return SQLITE_OK; }else{ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } #endif } @@ -39096,65 +40693,85 @@ static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ #define winDlClose 0 #endif +/* State information for the randomness gatherer. */ +typedef struct EntropyGatherer EntropyGatherer; +struct EntropyGatherer { + unsigned char *a; /* Gather entropy into this buffer */ + int na; /* Size of a[] in bytes */ + int i; /* XOR next input into a[i] */ + int nXor; /* Number of XOR operations done */ +}; + +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) +/* Mix sz bytes of entropy into p. */ +static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ + int j, k; + for(j=0, k=p->i; j<sz; j++){ + p->a[k++] ^= x[j]; + if( k>=p->na ) k = 0; + } + p->i = k; + p->nXor += sz; +} +#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ /* ** Write up to nBuf bytes of randomness into zBuf. */ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - int n = 0; - UNUSED_PARAMETER(pVfs); #if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) - n = nBuf; + UNUSED_PARAMETER(pVfs); memset(zBuf, 0, nBuf); + return nBuf; #else - if( sizeof(SYSTEMTIME)<=nBuf-n ){ + EntropyGatherer e; + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); +#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE + rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ +#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ + e.a = (unsigned char*)zBuf; + e.na = nBuf; + e.nXor = 0; + e.i = 0; + { SYSTEMTIME x; osGetSystemTime(&x); - memcpy(&zBuf[n], &x, sizeof(x)); - n += sizeof(x); + xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); } - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD pid = osGetCurrentProcessId(); - memcpy(&zBuf[n], &pid, sizeof(pid)); - n += sizeof(pid); + xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } #if SQLITE_OS_WINRT - if( sizeof(ULONGLONG)<=nBuf-n ){ + { ULONGLONG cnt = osGetTickCount64(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); } #else - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD cnt = osGetTickCount(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif - if( sizeof(LARGE_INTEGER)<=nBuf-n ){ +#endif /* SQLITE_OS_WINRT */ + { LARGE_INTEGER i; osQueryPerformanceCounter(&i); - memcpy(&zBuf[n], &i, sizeof(i)); - n += sizeof(i); + xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID - if( sizeof(UUID)<=nBuf-n ){ + { UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreate(&id); - memcpy(&zBuf[n], &id, sizeof(UUID)); - n += sizeof(UUID); - } - if( sizeof(UUID)<=nBuf-n ){ - UUID id; + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); memset(&id, 0, sizeof(UUID)); osUuidCreateSequential(&id); - memcpy(&zBuf[n], &id, sizeof(UUID)); - n += sizeof(UUID); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif -#endif /* defined(SQLITE_TEST) || defined(SQLITE_ZERO_PRNG_SEED) */ - return n; +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ + return e.nXor>nBuf ? nBuf : e.nXor; +#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -39270,8 +40887,10 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; } /* @@ -39409,7 +41028,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ -#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) +#define BITVEC_USIZE \ + (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) /* Type of the array "element" for the bitmap representation. ** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. @@ -39544,7 +41164,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ i = i%p->iDivisor; if( p->u.apSub[bin]==0 ){ p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); - if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; } p = p->u.apSub[bin]; } @@ -39579,7 +41199,7 @@ bitvec_set_rehash: int rc; u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); if( aiValues==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); @@ -39800,8 +41420,9 @@ bitvec_end: struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ + int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ u8 bPurgeable; /* True if pages are on backing store */ @@ -39891,10 +41512,8 @@ static void pcacheUnpin(PgHdr *p){ } /* -** Compute the number of pages of cache requested. p->szCache is the +** Compute the number of pages of cache requested. p->szCache is the ** cache size requested by the "PRAGMA cache_size" statement. -** -** */ static int numberOfCachePages(PCache *p){ if( p->szCache>=0 ){ @@ -39957,6 +41576,7 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( p->xStress = xStress; p->pStress = pStress; p->szCache = 100; + p->szSpill = 1; return sqlite3PcacheSetPageSize(p, szPage); } @@ -39965,14 +41585,14 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), pCache->bPurgeable ); - if( pNew==0 ) return SQLITE_NOMEM; + if( pNew==0 ) return SQLITE_NOMEM_BKPT; sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); if( pCache->pCache ){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); @@ -40052,36 +41672,37 @@ SQLITE_PRIVATE int sqlite3PcacheFetchStress( PgHdr *pPg; if( pCache->eCreate==2 ) return 0; - - /* Find a dirty page to write-out and recycle. First try to find a - ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other - ** unreferenced dirty page. - */ - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); - pPg=pPg->pDirtyPrev - ); - pCache->pSynced = pPg; - if( !pPg ){ - for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); - } - if( pPg ){ - int rc; + if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; #ifdef SQLITE_LOG_CACHE_SPILL - sqlite3_log(SQLITE_FULL, - "spill page %d making room for %d - cache used: %d/%d", - pPg->pgno, pgno, - sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), numberOfCachePages(pCache)); #endif - rc = pCache->xStress(pCache->pStress, pPg); - if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - return rc; + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; + } } } *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); - return *ppPage==0 ? SQLITE_NOMEM : SQLITE_OK; + return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; } /* @@ -40132,9 +41753,7 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } + pCache->nRefSum++; pPgHdr->nRef++; return pPgHdr; } @@ -40145,9 +41764,8 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( */ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - p->pCache->nRef--; + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); }else if( p->pDirtyPrev!=0 ){ @@ -40163,6 +41781,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); p->nRef++; + p->pCache->nRefSum++; } /* @@ -40175,7 +41794,7 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRef--; + p->pCache->nRefSum--; sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -40271,11 +41890,11 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->nRef ){ + if( pgno==0 && pCache->nRefSum ){ sqlite3_pcache_page *pPage1; pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because - ** pCache->nRef>0 */ + ** pCache->nRefSum>0 */ memset(pPage1->pBuf, 0, pCache->szPage); pgno = 1; } @@ -40381,10 +42000,13 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ } /* -** Return the total number of referenced pages held by the cache. +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. */ SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; + return pCache->nRefSum; } /* @@ -40422,6 +42044,25 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ } /* +** Set the suggested cache-spill value. Make no changes if if the +** argument is zero. Return the effective cache-spill size, which will +** be the larger of the szSpill and szCache. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ + int res; + assert( p->pCache!=0 ); + if( mxPage ){ + if( mxPage<0 ){ + mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra)); + } + p->szSpill = mxPage; + } + res = numberOfCachePages(p); + if( res<p->szSpill ) res = p->szSpill; + return res; +} + +/* ** Free up as much memory as possible from the page cache. */ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ @@ -40519,7 +42160,7 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** that is allocated when the page cache is created. The size of the local ** bulk allocation can be adjusted using ** -** sqlite3_config(SQLITE_CONFIG_PCACHE, 0, 0, N). +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). ** ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. @@ -40541,6 +42182,24 @@ typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -40569,7 +42228,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -40608,23 +42267,6 @@ struct PCache1 { }; /* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - u8 isBulkLocal; /* This page from bulk local storage */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - -/* ** Free slots in the allocator used to divide up the global page cache ** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. */ @@ -40684,6 +42326,7 @@ static SQLITE_WSD struct PCacheGlobal { /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -40743,6 +42386,7 @@ static int pcache1InitBulk(PCache1 *pCache){ pX->page.pBuf = zBulk; pX->page.pExtra = &pX[1]; pX->isBulkLocal = 1; + pX->isAnchor = 0; pX->pNext = pCache->pFree; pCache->pFree = pX; zBulk += pCache->szAlloc; @@ -40771,7 +42415,7 @@ static void *pcache1Alloc(int nByte){ pcache1.nFreeSlot--; pcache1.bUnderPressure = pcache1.nFreeSlot<pcache1.nReserve; assert( pcache1.nFreeSlot>=0 ); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); } sqlite3_mutex_leave(pcache1.mutex); @@ -40785,7 +42429,7 @@ static void *pcache1Alloc(int nByte){ if( p ){ int sz = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); sqlite3_mutex_leave(pcache1.mutex); } @@ -40799,9 +42443,8 @@ static void *pcache1Alloc(int nByte){ ** Free an allocated buffer obtained from pcache1Alloc(). */ static void pcache1Free(void *p){ - int nFreed = 0; if( p==0 ) return; - if( p>=pcache1.pStart && p<pcache1.pEnd ){ + if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ PgFreeslot *pSlot; sqlite3_mutex_enter(pcache1.mutex); sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_USED, 1); @@ -40816,10 +42459,13 @@ static void pcache1Free(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - nFreed = sqlite3MallocSize(p); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); - sqlite3_mutex_leave(pcache1.mutex); + { + int nFreed = 0; + nFreed = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); + sqlite3_mutex_leave(pcache1.mutex); + } #endif sqlite3_free(p); } @@ -40846,7 +42492,7 @@ static int pcache1MemSize(void *p){ /* ** Allocate a new page object initially associated with cache pCache. */ -static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ +static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ PgHdr1 *p = 0; void *pPg; @@ -40864,6 +42510,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ assert( pCache->pGroup==&pcache1.grp ); pcache1LeaveMutex(pCache->pGroup); #endif + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } #ifdef SQLITE_PCACHE_SEPARATE_HEADER pPg = pcache1Alloc(pCache->szPage); p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); @@ -40876,6 +42523,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ pPg = pcache1Alloc(pCache->szAlloc); p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif + if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif @@ -40883,6 +42531,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; + p->isAnchor = 0; } if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; @@ -41009,22 +42658,16 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - assert( pPage->pLruNext || pPage==pCache->pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pCache->pGroup->pLruHead ); + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pCache->pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pCache->pGroup->pLruTail = pPage->pLruPrev; - } + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; return pPage; } @@ -41057,9 +42700,11 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ */ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); @@ -41193,6 +42838,10 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; @@ -41300,11 +42949,11 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( /* Step 4. Try to recycle a page. */ if( pCache->bPurgeable - && pGroup->pLruTail + && !pGroup->lru.pLruPrev->isAnchor && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) ){ PCache1 *pOther; - pPage = pGroup->pLruTail; + pPage = pGroup->lru.pLruPrev; assert( pPage->isPinned==0 ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); @@ -41321,9 +42970,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** attempt to allocate a new one. */ if( !pPage ){ - if( createFlag==1 ){ sqlite3BeginBenignMalloc(); } - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ){ sqlite3EndBenignMalloc(); } + pPage = pcache1AllocPage(pCache, createFlag==1); } if( pPage ){ @@ -41415,7 +43062,10 @@ static PgHdr1 *pcache1FetchNoMutex( pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: Abort if no existing page is found and createFlag is 0 */ + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ if( pPage ){ if( !pPage->isPinned ){ return pcache1PinPage(pPage); @@ -41492,21 +43142,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -41644,7 +43289,10 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFree<nReq) && ((p=pcache1.grp.pLruTail)!=0) ){ + while( (nReq<0 || nFree<nReq) + && (p=pcache1.grp.lru.pLruPrev)!=0 + && p->isAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); @@ -41672,7 +43320,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } @@ -41868,7 +43516,7 @@ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); if( p->nFresh==0 ){ struct RowSetChunk *pNew; - pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ return 0; } @@ -42266,6 +43914,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 +# define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -42348,6 +43997,11 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); */ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). @@ -42355,6 +44009,9 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); #endif +/* Return the sqlite3_file object for the WAL file */ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ @@ -42767,6 +44424,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define MAX_SECTOR_SIZE 0x10000 + /* ** An instance of the following structure is allocated for each active ** savepoint and statement transaction in the system. All such structures @@ -42962,6 +44620,7 @@ struct Pager { u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 extraSync; /* sync directory after journal delete */ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ @@ -42986,7 +44645,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ - u8 hasBeenUsed; /* True if any content previously read */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -43667,6 +45326,7 @@ static i64 journalHdrOffset(Pager *pPager){ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ @@ -44048,7 +45708,7 @@ static void releaseAllSavepoints(Pager *pPager){ for(ii=0; ii<pPager->nSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); @@ -44286,8 +45946,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ - if( sqlite3IsMemJournal(pPager->jfd) ){ - assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ @@ -44313,16 +45973,17 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. + ** the database file, it will do so using an in-memory journal. */ - int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); } } } @@ -44455,6 +46116,20 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +#ifdef SQLITE_HAS_CODEC +/* +** Make sure the number of reserved bits is the same in the destination +** pager as it is in the source. This comes up when a VACUUM changes the +** number of reserved bits to the "optimal" amount. +*/ +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ + if( pDest->nReserve!=pSrc->nReserve ){ + pDest->nReserve = pSrc->nReserve; + pagerReportSize(pDest); + } +} +#endif + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -44634,9 +46309,9 @@ static int pager_playback_one_page( pPager->dbFileSize = pgno; } if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to @@ -44658,7 +46333,7 @@ static int pager_playback_one_page( assert( isSavepnt ); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); pPager->doNotSpill |= SPILLFLAG_ROLLBACK; - rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); + rc = sqlite3PagerGet(pPager, pgno, &pPg, 1); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; if( rc!=SQLITE_OK ) return rc; @@ -44708,7 +46383,7 @@ static int pager_playback_one_page( } /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3PcacheRelease(pPg); } return rc; @@ -44774,7 +46449,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); if( !pMaster ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); @@ -44791,7 +46466,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ nMasterPtr = pVfs->mxPathname+1; zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto delmaster_out; } zMasterPtr = &zMasterJournal[nMasterJournal+1]; @@ -45039,7 +46714,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value + ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; @@ -45261,7 +46936,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); @@ -45621,7 +47296,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( pSavepoint ){ pDone = sqlite3BitvecCreate(pSavepoint->nOrig); if( !pDone ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } @@ -45717,13 +47392,22 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ } /* -** Change the maximum number of in-memory pages that are allowed. +** Change the maximum number of in-memory pages that are allowed +** before attempting to recycle clean and unused pages. */ SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } /* +** Change the maximum number of in-memory pages that are allowed +** before attempting to spill pages to journal. +*/ +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){ + return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage); +} + +/* ** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. */ static void pagerFixMaplimit(Pager *pPager){ @@ -45759,7 +47443,7 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness ** of the database to damage due to OS crashes or power failures by ** changing the number of syncs()s when writing the journals. -** There are three levels: +** There are four levels: ** ** OFF sqlite3OsSync() is never called. This is the default ** for temporary and transient files. @@ -45779,6 +47463,10 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. ** +** EXTRA This is like FULL except that is also syncs the directory +** that contains the rollback journal after the rollback +** journal is unlinked. +** ** The above is for a rollback-journal mode. For WAL mode, OFF continues ** to mean that no syncs ever occur. NORMAL means that the WAL is synced ** prior to the start of checkpoint and that the database file is synced @@ -45786,7 +47474,8 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** was written back into the database. But no sync operations occur for ** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL ** file is synced following each commit operation, in addition to the -** syncs associated with NORMAL. +** syncs associated with NORMAL. There is no difference between FULL +** and EXTRA for WAL mode. ** ** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The ** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync @@ -45805,9 +47494,15 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - assert( level>=1 && level<=3 ); - pPager->noSync = (level==1 || pPager->tempFile) ?1:0; - pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + if( pPager->tempFile ){ + pPager->noSync = 1; + pPager->fullSync = 0; + pPager->extraSync = 0; + }else{ + pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; + pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + } if( pPager->noSync ){ pPager->syncFlags = 0; pPager->ckptSyncFlags = 0; @@ -45969,7 +47664,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR } if( rc==SQLITE_OK ){ pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM; + if( !pNew ) rc = SQLITE_NOMEM_BKPT; } if( rc==SQLITE_OK ){ @@ -46245,7 +47940,7 @@ static int pagerAcquireMapPage( *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; p->flags = PGHDR_MMAP; @@ -46603,7 +48298,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pList->pgno==1 ) pager_write_changecounter(pList); /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); @@ -46648,11 +48343,14 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - sqlite3MemJournalOpen(pPager->sjfd); - }else{ - rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + nStmtSpill = -1; } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); } return rc; } @@ -46690,7 +48388,7 @@ static int subjournalPage(PgHdr *pPg){ i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ @@ -46798,6 +48496,25 @@ static int pagerStress(void *p, PgHdr *pPg){ return pager_error(pPager, rc); } +/* +** Flush all unreferenced dirty pages to disk. +*/ +SQLITE_PRIVATE int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} /* ** Allocate and initialize a new Pager object and put a pointer to it @@ -46854,18 +48571,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). This - ** is the maximum space required for an in-memory journal file handle - ** and a regular journal file-handle. Note that a "regular journal-handle" - ** may be a wrapper capable of caching the first portion of the journal - ** file in memory to implement the atomic-write optimization (see - ** source file journal.c). - */ - if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); - }else{ - journalFileSize = ROUND8(sqlite3MemJournalSize()); - } + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; @@ -46875,7 +48582,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( memDb = 1; if( zFilename && zFilename[0] ){ zPathname = sqlite3DbStrDup(0, zFilename); - if( zPathname==0 ) return SQLITE_NOMEM; + if( zPathname==0 ) return SQLITE_NOMEM_BKPT; nPathname = sqlite3Strlen30(zPathname); zFilename = 0; } @@ -46891,7 +48598,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( nPathname = pVfs->mxPathname+1; zPathname = sqlite3DbMallocRaw(0, nPathname*2); if( zPathname==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); @@ -46944,7 +48651,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pPager = (Pager*)(pPtr); pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); @@ -47093,11 +48800,13 @@ act_like_temp_file: pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); + assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; + pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; @@ -47275,7 +48984,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ /* ** This function is called to obtain a shared lock on the database file. -** It is illegal to call sqlite3PagerAcquire() until after this function +** It is illegal to call sqlite3PagerGet() until after this function ** has been successfully called. If a shared-lock is already held when ** this function is called, it is a no-op. ** @@ -47436,10 +49145,10 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && pPager->hasBeenUsed ){ + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ /* The shared-lock has just been acquired then check to ** see if the database has been modified. If the database has changed, - ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** flush the cache. The hasHeldSharedLock flag prevents this from ** occurring on the very first access to a file, in order to save a ** single unnecessary sqlite3OsRead() call at the start-up. ** @@ -47509,6 +49218,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -47577,7 +49287,7 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** Since Lookup() never goes to disk, it never has to deal with locks ** or journal files. */ -SQLITE_PRIVATE int sqlite3PagerAcquire( +SQLITE_PRIVATE int sqlite3PagerGet( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ @@ -47592,21 +49302,25 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } - pPager->hasBeenUsed = 1; + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -47653,7 +49367,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( if( rc!=SQLITE_OK ) goto pager_acquire_err; if( pBase==0 ){ pPg = *ppPage = 0; - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto pager_acquire_err; } } @@ -47761,7 +49475,7 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); - assert( pPage==0 || pPager->hasBeenUsed ); + assert( pPage==0 || pPager->hasHeldSharedLock ); if( pPage==0 ) return 0; return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } @@ -47827,7 +49541,7 @@ static int pager_open_journal(Pager *pPager){ if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Open the journal file if it is not already open. */ @@ -47919,7 +49633,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory if( rc!=SQLITE_OK ){ return rc; } - sqlite3WalExclusiveMode(pPager->pWal, 1); + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to @@ -47982,7 +49696,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); cksum = pager_cksum(pPager, (u8*)pData2); /* Even if an IO or diskfull error occurs while journalling the @@ -48159,7 +49873,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ PgHdr *pPage; if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage); + rc = sqlite3PagerGet(pPager, pg, &pPage, 0); if( rc==SQLITE_OK ){ rc = pager_write(pPage); if( pPage->flags&PGHDR_NEED_SYNC ){ @@ -48216,9 +49930,10 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->errCode ){ + return pPager->errCode; + }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ @@ -48318,7 +50033,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ assert( !pPager->tempFile && isOpen(pPager->fd) ); /* Open page 1 of the file for writing. */ - rc = sqlite3PagerGet(pPager, 1, &pPgHdr); + rc = sqlite3PagerGet(pPager, 1, &pPgHdr, 0); assert( pPgHdr==0 || rc==SQLITE_OK ); /* If page one was fetched successfully, and this function is not @@ -48338,7 +50053,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -48396,14 +50111,17 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** returned. */ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ - int rc = SQLITE_OK; - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); + int rc = pPager->errCode; assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } } return rc; } @@ -48470,7 +50188,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ - rc = sqlite3PagerGet(pPager, 1, &pPageOne); + rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0); pList = pPageOne; pList->pDirty = 0; } @@ -48728,7 +50446,7 @@ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ #ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); @@ -48834,7 +50552,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint ); if( !aNew ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); pPager->aSavepoint = aNew; @@ -48850,7 +50568,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ aNew[ii].iSubRec = pPager->nSubRec; aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( pagerUseWal(pPager) ){ sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); @@ -48928,7 +50646,7 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3IsMemJournal(pPager->sjfd) ){ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } @@ -48967,7 +50685,7 @@ SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ /* ** Return the VFS structure for the pager. */ -SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ return pPager->pVfs; } @@ -48981,18 +50699,22 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ } /* -** Return the full pathname of the journal file. +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. */ -SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ - return pPager->zJournal; +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif } /* -** Return true if fsync() calls are disabled for this pager. Return FALSE -** if fsync()s are executed normally. +** Return the full pathname of the journal file. */ -SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ - return pPager->noSync; +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; } #ifdef SQLITE_HAS_CODEC @@ -49175,7 +50897,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** the journal file twice, but that is not a problem. */ PgHdr *pPgHdr; - rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr, 0); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ assert( pPager->pTmpSpace!=0 ); @@ -49441,6 +51163,7 @@ SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } @@ -49589,6 +51312,34 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** If this is a WAL database, obtain a snapshot handle for the snapshot +** currently open. Otherwise, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_ERROR; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotGet(pPager->pWal, ppSnapshot); + } + return rc; +} + +/* +** If this is a WAL database, store a pointer to pSnapshot. Next time a +** read transaction is opened, attempt to read from the snapshot it +** identifies. If this is not a WAL database, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -49884,7 +51635,8 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; /* ** Indices of various locking bytes. WAL_NREADER is the number -** of available reader locks and should be at least 3. +** of available reader locks and should be at least 3. The default +** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. */ #define WAL_WRITE_LOCK 0 #define WAL_ALL_BUT_WRITE 1 @@ -49904,7 +51656,10 @@ typedef struct WalCkptInfo WalCkptInfo; ** The following object holds a copy of the wal-index header content. ** ** The actual header in the wal-index consists of two copies of this -** object. +** object followed by one instance of the WalCkptInfo object. +** For all versions of SQLite through 3.10.0 and probably beyond, +** the locking bytes (WalCkptInfo.aLock) start at offset 120 and +** the total header size is 136 bytes. ** ** The szPage value can be any power of 2 between 512 and 32768, inclusive. ** Or it can be 1 to represent a 65536-byte page. The latter case was @@ -49937,6 +51692,16 @@ struct WalIndexHdr { ** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from ** mxFrame back to zero when the WAL is reset. ** +** nBackfillAttempted is the largest value of nBackfill that a checkpoint +** has attempted to achieve. Normally nBackfill==nBackfillAtempted, however +** the nBackfillAttempted is set before any backfilling is done and the +** nBackfill is only set after all backfilling completes. So if a checkpoint +** crashes, nBackfillAttempted might be larger than nBackfill. The +** WalIndexHdr.mxFrame must never be less than nBackfillAttempted. +** +** The aLock[] field is a set of bytes used for locking. These bytes should +** never be read or written. +** ** There is one entry in aReadMark[] for each reader lock. If a reader ** holds read-lock K, then the value in aReadMark[K] is no greater than ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) @@ -49976,6 +51741,9 @@ struct WalIndexHdr { struct WalCkptInfo { u32 nBackfill; /* Number of WAL frames backfilled into DB */ u32 aReadMark[WAL_NREADER]; /* Reader marks */ + u8 aLock[SQLITE_SHM_NLOCK]; /* Reserved space for locks */ + u32 nBackfillAttempted; /* WAL frames perhaps written, or maybe not */ + u32 notUsed0; /* Available for future enhancements */ }; #define READMARK_NOT_USED 0xffffffff @@ -49985,9 +51753,8 @@ struct WalCkptInfo { ** only support mandatory file-locks, we do not read or write data ** from the region of the file on which locks are applied. */ -#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) -#define WALINDEX_LOCK_RESERVED 16 -#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2+offsetof(WalCkptInfo,aLock)) +#define WALINDEX_HDR_SIZE (sizeof(WalIndexHdr)*2+sizeof(WalCkptInfo)) /* Size of header before each frame in wal */ #define WAL_FRAME_HDRSIZE 24 @@ -50040,11 +51807,16 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ + u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif +#ifdef SQLITE_ENABLE_SNAPSHOT + WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ +#endif }; /* @@ -50137,7 +51909,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset((void*)&apNew[pWal->nWiData], 0, sizeof(u32*)*(iPage+1-pWal->nWiData)); @@ -50149,7 +51921,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ if( pWal->apWiData[iPage]==0 ){ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] @@ -50290,14 +52062,18 @@ static void walEncodeFrame( assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); - memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + if( pWal->iReCksum==0 ){ + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); - nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); - walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - sqlite3Put4byte(&aFrame[16], aCksum[0]); - sqlite3Put4byte(&aFrame[20], aCksum[1]); + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); + }else{ + memset(&aFrame[8], 0, 16); + } } /* @@ -50400,10 +52176,9 @@ static void walUnlockShared(Wal *pWal, int lockIdx){ SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } -static int walLockExclusive(Wal *pWal, int lockIdx, int n, int fBlock){ +static int walLockExclusive(Wal *pWal, int lockIdx, int n){ int rc; if( pWal->exclusiveMode ) return SQLITE_OK; - if( fBlock ) sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_WAL_BLOCK, 0); rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, @@ -50689,7 +52464,7 @@ static int walIndexRecover(Wal *pWal){ assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock, 0); + rc = walLockExclusive(pWal, iLock, nLock); if( rc ){ return rc; } @@ -50761,7 +52536,7 @@ static int walIndexRecover(Wal *pWal){ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame); if( !aFrame ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -50810,6 +52585,7 @@ finished: */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; + pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; @@ -50881,7 +52657,11 @@ SQLITE_PRIVATE int sqlite3WalOpen( /* In the amalgamation, the os_unix.c and os_win.c source files come before ** this source file. Verify that the #defines of the locking byte offsets ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. + ** For that matter, if the lock offset ever changes from its initial design + ** value of 120, we need to know that so there is an assert() to check it. */ + assert( 120==WALINDEX_LOCK_OFFSET ); + assert( 136==WALINDEX_HDR_SIZE ); #ifdef WIN_SHM_BASE assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); #endif @@ -50894,7 +52674,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( *ppWal = 0; pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); if( !pRet ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pRet->pVfs = pVfs; @@ -51158,7 +52938,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ + iLast*sizeof(ht_slot); p = (WalIterator *)sqlite3_malloc64(nByte); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; @@ -51170,7 +52950,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } for(i=0; rc==SQLITE_OK && i<nSegment; i++){ @@ -51227,7 +53007,7 @@ static int walBusyLock( ){ int rc; do { - rc = walLockExclusive(pWal, lockIdx, n, 0); + rc = walLockExclusive(pWal, lockIdx, n); }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); return rc; } @@ -51267,6 +53047,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ memcpy(&pWal->hdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; + pInfo->nBackfillAttempted = 0; pInfo->aReadMark[1] = 0; for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); @@ -51376,6 +53157,8 @@ static int walCheckpoint( i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; + pInfo->nBackfillAttempted = mxSafeFrame; + /* Sync the WAL to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pWalFd, sync_flags); @@ -51668,7 +53451,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } - }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 1)) ){ + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ pWal->writeLock = 1; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); @@ -51759,6 +53542,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ int mxI; /* Index of largest aReadMark[] value */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ + u32 mxFrame; /* Wal frame to lock to */ assert( pWal->readLock<0 ); /* Not currently locked */ @@ -51822,7 +53606,12 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame +#ifdef SQLITE_ENABLE_SNAPSHOT + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 + || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) +#endif + ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. */ @@ -51860,70 +53649,88 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ */ mxReadMark = 0; mxI = 0; + mxFrame = pWal->hdr.mxFrame; +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pWal->pSnapshot && pWal->pSnapshot->mxFrame<mxFrame ){ + mxFrame = pWal->pSnapshot->mxFrame; + } +#endif for(i=1; i<WAL_NREADER; i++){ u32 thisMark = pInfo->aReadMark[i]; - if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ + if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; mxI = i; } } - /* There was once an "if" here. The extra "{" is to preserve indentation. */ - { - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMark<pWal->hdr.mxFrame || mxI==0) - ){ - for(i=1; i<WAL_NREADER; i++){ - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1, 0); - if( rc==SQLITE_OK ){ - mxReadMark = pInfo->aReadMark[i] = pWal->hdr.mxFrame; - mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; - } + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMark<mxFrame || mxI==0) + ){ + for(i=1; i<WAL_NREADER; i++){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + mxReadMark = pInfo->aReadMark[i] = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; } } - if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; - } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. - */ - walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; - } + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. + */ + pWal->minFrame = pInfo->nBackfill+1; + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; } return rc; } @@ -51946,6 +53753,14 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ +#ifdef SQLITE_ENABLE_SNAPSHOT + int bChanged = 0; + WalIndexHdr *pSnapshot = pWal->pSnapshot; + if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + bChanged = 1; + } +#endif + do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); @@ -51953,6 +53768,66 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( rc==SQLITE_OK ){ + if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + /* At this point the client has a lock on an aReadMark[] slot holding + ** a value equal to or smaller than pSnapshot->mxFrame, but pWal->hdr + ** is populated with the wal-index header corresponding to the head + ** of the wal file. Verify that pSnapshot is still valid before + ** continuing. Reasons why pSnapshot might no longer be valid: + ** + ** (1) The WAL file has been reset since the snapshot was taken. + ** In this case, the salt will have changed. + ** + ** (2) A checkpoint as been attempted that wrote frames past + ** pSnapshot->mxFrame into the database file. Note that the + ** checkpoint need not have completed for this to cause problems. + */ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + + assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); + assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); + + /* It is possible that there is a checkpointer thread running + ** concurrent with this code. If this is the case, it may be that the + ** checkpointer has already determined that it will checkpoint + ** snapshot X, where X is later in the wal file than pSnapshot, but + ** has not yet set the pInfo->nBackfillAttempted variable to indicate + ** its intent. To avoid the race condition this leads to, ensure that + ** there is no checkpointer process by taking a shared CKPT lock + ** before checking pInfo->nBackfillAttempted. */ + rc = walLockShared(pWal, WAL_CKPT_LOCK); + + if( rc==SQLITE_OK ){ + /* Check that the wal file has not been wrapped. Assuming that it has + ** not, also check that no checkpointer has attempted to checkpoint any + ** frames beyond pSnapshot->mxFrame. If either of these conditions are + ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** with *pSnapshot and set *pChanged as appropriate for opening the + ** snapshot. */ + if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + && pSnapshot->mxFrame>=pInfo->nBackfillAttempted + ){ + assert( pWal->readLock>0 ); + memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); + *pChanged = bChanged; + }else{ + rc = SQLITE_BUSY_SNAPSHOT; + } + + /* Release the shared CKPT lock obtained above. */ + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + + + if( rc!=SQLITE_OK ){ + sqlite3WalEndReadTransaction(pWal); + } + } + } +#endif return rc; } @@ -51984,6 +53859,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -52024,7 +53900,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -52039,7 +53916,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -52056,7 +53933,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ + assert( pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -52122,6 +54000,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); + assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ return SQLITE_READONLY; @@ -52130,7 +54009,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); if( rc ){ return rc; } @@ -52157,6 +54036,7 @@ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; + pWal->iReCksum = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; @@ -52275,7 +54155,7 @@ static int walRestartLog(Wal *pWal){ if( pInfo->nBackfill>0 ){ u32 salt1; sqlite3_randomness(4, &salt1); - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1, 0); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no ** readers are currently using the WAL), then the transactions @@ -52363,7 +54243,7 @@ static int walWriteOneFrame( void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ #if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM; + if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; #else pData = pPage->pData; #endif @@ -52375,6 +54255,59 @@ static int walWriteOneFrame( return rc; } +/* +** This function is called as part of committing a transaction within which +** one or more frames have been overwritten. It updates the checksums for +** all frames written to the wal file by the current transaction starting +** with the earliest to have been overwritten. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walRewriteChecksums(Wal *pWal, u32 iLast){ + const int szPage = pWal->szPage;/* Database page size */ + int rc = SQLITE_OK; /* Return code */ + u8 *aBuf; /* Buffer to load data from wal file into */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ + u32 iRead; /* Next frame to read from wal file */ + i64 iCksumOff; + + aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); + if( aBuf==0 ) return SQLITE_NOMEM_BKPT; + + /* Find the checksum values to use as input for the recalculating the + ** first checksum. If the first frame is frame 1 (implying that the current + ** transaction restarted the wal file), these values must be read from the + ** wal-file header. Otherwise, read them from the frame header of the + ** previous frame. */ + assert( pWal->iReCksum>0 ); + if( pWal->iReCksum==1 ){ + iCksumOff = 24; + }else{ + iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; + } + rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); + pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); + pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); + + iRead = pWal->iReCksum; + pWal->iReCksum = 0; + for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ + i64 iOff = walFrameOffset(iRead, szPage); + rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); + if( rc==SQLITE_OK ){ + u32 iPgno, nDbSize; + iPgno = sqlite3Get4byte(aBuf); + nDbSize = sqlite3Get4byte(&aBuf[4]); + + walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); + rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); + } + } + + sqlite3_free(aBuf); + return rc; +} + /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). @@ -52395,6 +54328,8 @@ SQLITE_PRIVATE int sqlite3WalFrames( int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ + u32 iFirst = 0; /* First frame that may be overwritten */ + WalIndexHdr *pLive; /* Pointer to shared header */ assert( pList ); assert( pWal->writeLock ); @@ -52410,6 +54345,11 @@ SQLITE_PRIVATE int sqlite3WalFrames( } #endif + pLive = (WalIndexHdr*)walIndexHdr(pWal); + if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ + iFirst = pLive->mxFrame+1; + } + /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ @@ -52474,6 +54414,33 @@ SQLITE_PRIVATE int sqlite3WalFrames( /* Write all frames into the log file exactly once */ for(p=pList; p; p=p->pDirty){ int nDbSize; /* 0 normally. Positive == commit flag */ + + /* Check if this page has already been written into the wal file by + ** the current transaction. If so, overwrite the existing frame and + ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that + ** checksums must be recomputed when the transaction is committed. */ + if( iFirst && (p->pDirty || isCommit==0) ){ + u32 iWrite = 0; + VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + assert( rc==SQLITE_OK || iWrite==0 ); + if( iWrite>=iFirst ){ + i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; + void *pData; + if( pWal->iReCksum==0 || iWrite<pWal->iReCksum ){ + pWal->iReCksum = iWrite; + } +#if defined(SQLITE_HAS_CODEC) + if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; +#else + pData = p->pData; +#endif + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + if( rc ) return rc; + p->flags &= ~PGHDR_WAL_APPEND; + continue; + } + } + iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; @@ -52481,6 +54448,13 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; pLast = p; iOffset += szFrame; + p->flags |= PGHDR_WAL_APPEND; + } + + /* Recalculate checksums within the wal file if required. */ + if( isCommit && pWal->iReCksum ){ + rc = walRewriteChecksums(pWal, iFrame); + if( rc ) return rc; } /* If this is the end of a transaction, then we might need to pad @@ -52532,6 +54506,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( */ iFrame = pWal->hdr.mxFrame; for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ + if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } @@ -52600,7 +54575,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive ** "checkpoint" lock on the database file. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1, 0); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ /* EVIDENCE-OF: R-10421-19736 If any other process is running a ** checkpoint operation at the same time, the lock cannot be obtained and @@ -52644,6 +54619,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ @@ -52759,6 +54735,35 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* Create a snapshot object. The content of a snapshot is opaque to +** every other subsystem, so the WAL module can put whatever it needs +** in the object. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_OK; + WalIndexHdr *pRet; + + assert( pWal->readLock>=0 && pWal->writeLock==0 ); + + pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); + if( pRet==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memcpy(pRet, &pWal->hdr, sizeof(WalIndexHdr)); + *ppSnapshot = (sqlite3_snapshot*)pRet; + } + + return rc; +} + +/* Try to open on pSnapshot when the next read-transaction starts +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a @@ -52771,6 +54776,12 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ } #endif +/* Return the sqlite3_file object for the WAL file +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ + return pWal->pWalFd; +} + #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ @@ -53072,7 +55083,6 @@ struct MemPage { u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ - u8 noPayload; /* True if internal intKey page (thus w/o data) */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ @@ -53266,7 +55276,6 @@ struct CellInfo { u8 *pPayload; /* Pointer to the start of payload */ u32 nPayload; /* Bytes of payload */ u16 nLocal; /* Amount of payload held locally, not on overflow */ - u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; @@ -53313,7 +55322,7 @@ struct BtCursor { int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ - u8 curPagerFlags; /* Flags to send to sqlite3PagerAcquire() */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 hints; /* As configured by CursorSetHints() */ /* All fields above are zeroed when the cursor is allocated. See @@ -53496,9 +55505,11 @@ struct IntegrityCk { */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) @@ -53659,21 +55670,6 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ #endif -#ifndef SQLITE_OMIT_INCRBLOB -/* -** Enter and leave a mutex on a Btree given a cursor owned by that -** Btree. These entry points are used by incremental I/O and can be -** omitted if that module is not used. -*/ -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ - sqlite3BtreeEnter(pCur->pBtree); -} -SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ - sqlite3BtreeLeave(pCur->pBtree); -} -#endif /* SQLITE_OMIT_INCRBLOB */ - - /* ** Enter the mutex on every Btree associated with a database ** connection. This is needed (for example) prior to parsing @@ -53707,14 +55703,6 @@ SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ } } -/* -** Return true if a particular Btree requires a lock. Return FALSE if -** no lock is ever required since it is not sharable. -*/ -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ - return p->sharable; -} - #ifndef NDEBUG /* ** Return true if the current thread holds the database connection @@ -53788,6 +55776,25 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ } } #endif /* if SQLITE_THREADSAFE */ + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter a mutex on a Btree given a cursor owned by that Btree. +** +** These entry points are used by incremental I/O only. Enter() is required +** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not +** the build is threadsafe. Leave() is only required by threadsafe builds. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +# if SQLITE_THREADSAFE +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +# endif +#endif /* ifndef SQLITE_OMIT_INCRBLOB */ + #endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ /************** End of btmutex.c *********************************************/ @@ -54144,7 +56151,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ if( !pLock ){ pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); if( !pLock ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pLock->iTable = iTable; pLock->pBtree = p; @@ -54244,6 +56251,10 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } +static int cursorOwnsBtShared(BtCursor *p){ + assert( cursorHoldsMutex(p) ); + return (p->pBtree->db==p->pBt->db); +} #endif /* @@ -54343,7 +56354,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ assert( pgno<=pBt->nPage ); pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); if( !pBt->pHasContent ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ @@ -54385,26 +56396,25 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } - /* -** Save the current cursor position in the variables BtCursor.nKey -** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. ** -** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. */ -static int saveCursorPosition(BtCursor *pCur){ +static int saveCursorKey(BtCursor *pCur){ int rc; - - assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - if( pCur->eState==CURSOR_SKIPNEXT ){ - pCur->eState = CURSOR_VALID; - }else{ - pCur->skipNext = 0; - } rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ @@ -54412,8 +56422,7 @@ static int saveCursorPosition(BtCursor *pCur){ ** stores the integer key in pCur->nKey. In this case this value is ** all that is required. Otherwise, if pCur is not open on an intKey ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ + ** data. */ if( 0==pCur->curIntKey ){ void *pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ @@ -54424,17 +56433,40 @@ static int saveCursorPosition(BtCursor *pCur){ sqlite3_free(pKey); } }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; + } + + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - invalidateOverflowCache(pCur); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); return rc; } @@ -54533,7 +56565,7 @@ static int btreeMoveto( pIdxKey = sqlite3VdbeAllocUnpackedRecord( pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree ); - if( pIdxKey==0 ) return SQLITE_NOMEM; + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ sqlite3DbFree(pCur->pKeyInfo->db, pFree); @@ -54559,7 +56591,7 @@ static int btreeMoveto( static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; int skipNext; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; @@ -54631,6 +56663,26 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) return SQLITE_OK; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Provide hints to the cursor. The particular hint given (and the type +** and number of the varargs parameters) is determined by the eHintType +** parameter. See the definitions of the BTREE_HINT_* macros for details. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ + /* Used only by system that substitute their own storage engine */ +} +#endif + +/* +** Provide flag hints to the cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ + assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); + pCur->hints = x; +} + + #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -54684,7 +56736,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ return; } iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -54727,7 +56779,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ assert( sqlite3_mutex_held(pBt->mutex) ); iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); if( rc!=0 ){ return rc; } @@ -54804,8 +56856,7 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( }else{ pInfo->nLocal = (u16)minLocal; } - pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); - pInfo->nSize = pInfo->iOverflow + 4; + pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; } /* @@ -54829,7 +56880,6 @@ static void btreeParseCellPtrNoPayload( ){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 ); - assert( pPage->noPayload ); assert( pPage->childPtrSize==4 ); #ifndef SQLITE_DEBUG UNUSED_PARAMETER(pPage); @@ -54837,7 +56887,6 @@ static void btreeParseCellPtrNoPayload( pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); pInfo->nPayload = 0; pInfo->nLocal = 0; - pInfo->iOverflow = 0; pInfo->pPayload = 0; return; } @@ -54852,8 +56901,6 @@ static void btreeParseCellPtr( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - assert( pPage->intKeyLeaf || pPage->noPayload ); - assert( pPage->noPayload==0 ); assert( pPage->intKeyLeaf ); assert( pPage->childPtrSize==0 ); pIter = pCell; @@ -54907,7 +56954,6 @@ static void btreeParseCellPtr( pInfo->nSize = nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; - pInfo->iOverflow = 0; }else{ btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } @@ -54923,7 +56969,6 @@ static void btreeParseCellPtrIndex( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf==0 ); - assert( pPage->noPayload==0 ); pIter = pCell + pPage->childPtrSize; nPayload = *pIter; if( nPayload>=0x80 ){ @@ -54946,7 +56991,6 @@ static void btreeParseCellPtrIndex( pInfo->nSize = nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; - pInfo->iOverflow = 0; }else{ btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } @@ -54985,7 +57029,6 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif - assert( pPage->noPayload==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -55062,8 +57105,8 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ if( *pRC ) return; assert( pCell!=0 ); pPage->xParseCell(pPage, pCell, &info); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&pCell[info.iOverflow]); + if( info.nLocal<info.nPayload ){ + Pgno ovfl = get4byte(&pCell[info.nSize-4]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } } @@ -55443,11 +57486,9 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->intKey = 1; if( pPage->leaf ){ pPage->intKeyLeaf = 1; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtr; }else{ pPage->intKeyLeaf = 0; - pPage->noPayload = 1; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; } @@ -55462,7 +57503,6 @@ static int decodeFlags(MemPage *pPage, int flagByte){ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; @@ -55654,11 +57694,14 @@ static void zeroPage(MemPage *pPage, int flags){ */ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - pPage->aData = sqlite3PagerGetData(pDbPage); - pPage->pDbPage = pDbPage; - pPage->pBt = pBt; - pPage->pgno = pgno; - pPage->hdrOffset = pgno==1 ? 100 : 0; + if( pgno!=pPage->pgno ){ + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pgno==1 ? 100 : 0; + } + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); return pPage; } @@ -55684,7 +57727,7 @@ static int btreeGetPage( assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -55749,24 +57792,25 @@ static int getAndInitPage( rc = SQLITE_CORRUPT_BKPT; goto getAndInitPage_error; } - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ goto getAndInitPage_error; } - *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); if( (*ppPage)->isInit==0 ){ + btreePageFromDbPage(pDbPage, pgno, pBt); rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); goto getAndInitPage_error; } } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); /* If obtaining a child page for a cursor, we must verify that the page is ** compatible with the root page. */ - if( pCur - && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) - ){ + if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ rc = SQLITE_CORRUPT_BKPT; releasePage(*ppPage); goto getAndInitPage_error; @@ -55933,7 +57977,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( } p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->inTrans = TRANS_NONE; p->db = db; @@ -55957,7 +58001,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( isMemdb ){ memcpy(zFullPathname, zFilename, nFilename); @@ -56025,7 +58069,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, @@ -56094,8 +58138,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ - rc = SQLITE_NOMEM; - db->mallocFailed = 0; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } } @@ -56118,12 +58161,12 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( for(i=0; i<db->nDb; i++){ if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( p->pBt<pSib->pBt ){ + if( (uptr)p->pBt<(uptr)pSib->pBt ){ p->pNext = pSib; p->pPrev = 0; pSib->pPrev = p; }else{ - while( pSib->pNext && pSib->pNext->pBt<p->pBt ){ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ pSib = pSib->pNext; } p->pNext = pSib->pNext; @@ -56305,19 +58348,11 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ } /* -** Change the limit on the number of pages allowed in the cache. -** -** The maximum number of cache pages is set to the absolute -** value of mxPage. If mxPage is negative, the pager will -** operate asynchronously - it will not stop to do fsync()s -** to insure data is written to the disk surface before -** continuing. Transactions still work if synchronous is off, -** and the database cannot be corrupted if this program -** crashes. But if the operating system crashes or there is -** an abrupt power failure when synchronous is off, the database -** could be left in an inconsistent and unrecoverable state. -** Synchronous is on by default so database corruption is not -** normally a worry. +** Change the "soft" limit on the number of pages in the cache. +** Unused and unmodified pages will be recycled when the number of +** pages in the cache exceeds this soft limit. But the size of the +** cache is allowed to grow larger than this limit if it contains +** dirty pages or pages still in active use. */ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ BtShared *pBt = p->pBt; @@ -56328,6 +58363,26 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the "spill" limit on the number of pages in the cache. +** If the number of pages exceeds this limit during a write transaction, +** the pager might attempt to "spill" pages to the journal early in +** order to free up memory. +** +** The value returned is the current spill size. If zero is passed +** as an argument, no changes are made to the spill size setting, so +** using mxPage of 0 is a way to query the current spill size. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + int res; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return res; +} + #if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be @@ -56366,21 +58421,6 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( #endif /* -** Return TRUE if the given btree is set to safety level 1. In other -** words, return TRUE if no sync() occurs on the disk files. -*/ -SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ - BtShared *pBt = p->pBt; - int rc; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - assert( pBt && pBt->pPager ); - rc = sqlite3PagerNosync(pBt->pPager); - sqlite3BtreeLeave(p); - return rc; -} - -/* ** Change the default pages size and the number of reserved bytes per page. ** Or, if the page size has already been fixed, return SQLITE_READONLY ** without changing anything. @@ -56625,9 +58665,25 @@ static int lockBtree(BtShared *pBt){ rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; - }else if( isOpen==0 ){ - releasePage(pPage1); - return SQLITE_OK; + }else{ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 + ){ + pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +#endif + if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; + } } rc = SQLITE_NOTADB; } @@ -56867,7 +58923,6 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** proceed. */ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ - sqlite3 *pBlock = 0; BtShared *pBt = p->pBt; int rc = SQLITE_OK; @@ -56890,27 +58945,30 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } #ifndef SQLITE_OMIT_SHARED_CACHE - /* If another database handle has already opened a write transaction - ** on this shared-btree structure and a second write transaction is - ** requested, return SQLITE_LOCKED. - */ - if( (wrflag && pBt->inTransaction==TRANS_WRITE) - || (pBt->btsFlags & BTS_PENDING)!=0 - ){ - pBlock = pBt->pWriter->db; - }else if( wrflag>1 ){ - BtLock *pIter; - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - if( pIter->pBtree!=p ){ - pBlock = pIter->pBtree->db; - break; + { + sqlite3 *pBlock = 0; + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } } } - } - if( pBlock ){ - sqlite3ConnectionBlocked(p->db, pBlock); - rc = SQLITE_LOCKED_SHAREDCACHE; - goto trans_begun; + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; + } } #endif @@ -57085,11 +59143,11 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; pPage->xParseCell(pPage, pCell, &info); - if( info.iOverflow - && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage - && iFrom==get4byte(&pCell[info.iOverflow]) + if( info.nLocal<info.nPayload + && pCell+info.nSize-1<=pPage->aData+pPage->maskPage + && iFrom==get4byte(pCell+info.nSize-4) ){ - put4byte(&pCell[info.iOverflow], iTo); + put4byte(pCell+info.nSize-4, iTo); break; } }else{ @@ -57792,13 +59850,13 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** on the database already. If a write-cursor is requested, then ** the caller is assumed to have an open write transaction. ** -** If wrFlag==0, then the cursor can only be used for reading. -** If wrFlag==1, then the cursor can be used for reading or for -** writing if other conditions for writing are also met. These -** are the conditions that must be met in order for writing to -** be allowed: +** If the BTREE_WRCSR bit of wrFlag is clear, then the cursor can only +** be used for reading. If the BTREE_WRCSR bit is set, then the cursor +** can be used for reading or for writing if other conditions for writing +** are also met. These are the conditions that must be met in order +** for writing to be allowed: ** -** 1: The cursor must have been opened with wrFlag==1 +** 1: The cursor must have been opened with wrFlag containing BTREE_WRCSR ** ** 2: Other database connections that share the same pager cache ** but which are not in the READ_UNCOMMITTED state may not have @@ -57810,6 +59868,16 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** ** 4: There must be an active transaction. ** +** The BTREE_FORDELETE bit of wrFlag may optionally be set if BTREE_WRCSR +** is set. If FORDELETE is set, that is a hint to the implementation that +** this cursor will only be used to seek to and delete entries of an index +** as part of a larger DELETE statement. The FORDELETE hint is not used by +** this implementation. But in a hypothetical alternative storage engine +** in which index entries are automatically deleted when corresponding table +** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE +** operations on this cursor can be no-ops and all READ operations can +** return a null row (2-bytes: 0x01 0x00). +** ** No checking is done to make sure that page iTable really is the ** root page of a b-tree. If it is not, then the cursor acquired ** will not work correctly. @@ -57828,13 +59896,16 @@ static int btreeCursor( BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 || wrFlag==1 ); + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + ); /* The following assert statements verify that if this is a sharable ** b-tree database, the connection is holding the required table locks, ** and that no other connection has any open cursor that conflicts with ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ @@ -57845,7 +59916,7 @@ static int btreeCursor( if( wrFlag ){ allocateTempSpace(pBt); - if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); @@ -57859,8 +59930,7 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); - pCur->curFlags = wrFlag; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; /* If there are two or more cursors on the same btree, then all such ** cursors *must* have the BTCF_Multiple flag set. */ @@ -58025,7 +60095,7 @@ SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ ** to return an integer result code for historical reasons. */ SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>=0 ); assert( pCur->iPage<BTCURSOR_MAX_DEPTH ); @@ -58203,8 +60273,13 @@ static int accessPayload( #endif assert( offset+amt <= pCur->info.nPayload ); - if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ - /* Trying to read or write past the end of the data is an error */ + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ return SQLITE_CORRUPT_BKPT; } @@ -58244,7 +60319,7 @@ static int accessPayload( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; @@ -58272,7 +60347,9 @@ static int accessPayload( /* If required, populate the overflow page-list cache. */ if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); pCur->aOverflow[iIdx] = nextPage; } @@ -58342,7 +60419,7 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ @@ -58403,7 +60480,7 @@ SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *p } #endif - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); @@ -58441,7 +60518,7 @@ static const void *fetchPayload( assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->aiIdx[pCur->iPage]<pCur->apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); @@ -58487,7 +60564,7 @@ SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ static int moveToChild(BtCursor *pCur, u32 newPgno){ BtShared *pBt = pCur->pBt; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage<BTCURSOR_MAX_DEPTH ); assert( pCur->iPage>=0 ); @@ -58533,7 +60610,7 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ ** the largest cell index. */ static void moveToParent(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); assert( pCur->apPage[pCur->iPage] ); @@ -58573,7 +60650,7 @@ static int moveToRoot(BtCursor *pCur){ MemPage *pRoot; int rc = SQLITE_OK; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); @@ -58652,7 +60729,7 @@ static int moveToLeftmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ assert( pCur->aiIdx[pCur->iPage]<pPage->nCell ); @@ -58677,7 +60754,7 @@ static int moveToRightmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage = 0; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); @@ -58698,7 +60775,7 @@ static int moveToRightmost(BtCursor *pCur){ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ int rc; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ @@ -58721,7 +60798,7 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ int rc; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ @@ -58786,6 +60863,8 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ ** *pRes>0 The cursor is left pointing at an entry that ** is larger than intKey/pIdxKey. ** +** For index tables, the pIdxKey->eqSeen field is set to 1 if there +** exists an entry in the table that exactly matches pIdxKey. */ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( BtCursor *pCur, /* The cursor to be moved */ @@ -58797,7 +60876,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int rc; RecordCompare xRecordCompare; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); @@ -58945,7 +61024,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } pCellKey = sqlite3Malloc( nCell+18 ); if( pCellKey==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; @@ -59045,7 +61124,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ int idx; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ @@ -59109,7 +61188,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ } SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -59154,7 +61233,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pRes!=0 ); assert( *pRes==0 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -59210,7 +61289,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ return rc; } SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -59725,13 +61804,13 @@ static int clearCell( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); pPage->xParseCell(pPage, pCell, &info); *pnSize = info.nSize; - if( info.iOverflow==0 ){ + if( info.nLocal==info.nPayload ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } - if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ + if( pCell+info.nSize-1 > pPage->aData+pPage->maskPage ){ return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } - ovflPgno = get4byte(&pCell[info.iOverflow]); + ovflPgno = get4byte(pCell + info.nSize - 4); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; @@ -59876,11 +61955,10 @@ static int fillInCell( { CellInfo info; pPage->xParseCell(pPage, pCell, &info); - assert( nHeader=(int)(info.pPayload - pCell) ); + assert( nHeader==(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); - assert( pPrior == &pCell[info.iOverflow] ); } #endif @@ -60201,7 +62279,7 @@ static int rebuildPage( pData = pEnd; for(i=0; i<nCell; i++){ u8 *pCell = apCell[i]; - if( pCell>aData && pCell<pEnd ){ + if( SQLITE_WITHIN(pCell,aData,pEnd) ){ pCell = &pTmp[pCell - aData]; } pData -= szCell[i]; @@ -60272,7 +62350,13 @@ static int pageInsertArray( if( pData<pBegin ) return 1; pSlot = pData; } - memcpy(pSlot, pCArray->apCell[i], sz); + /* pSlot and pCArray->apCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; } @@ -60306,7 +62390,7 @@ static int pageFreeArray( for(i=iFirst; i<iEnd; i++){ u8 *pCell = pCArray->apCell[i]; - if( pCell>=pStart && pCell<pEnd ){ + if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which @@ -60584,8 +62668,8 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ z = findCell(pPage, j); pPage->xParseCell(pPage, z, &info); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&z[info.iOverflow]); + if( info.nLocal<info.nPayload ){ + Pgno ovfl = get4byte(&z[info.nSize-4]); ptrmapGet(pBt, ovfl, &e, &n); assert( n==pPage->pgno && e==PTRMAP_OVERFLOW1 ); } @@ -60703,9 +62787,6 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) -#pragma optimize("", off) -#endif static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ @@ -60762,7 +62843,7 @@ static int balance_nonroot( assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); if( !aOvflSpace ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Find the sibling pages to balance. Also locate the cells in pParent @@ -60862,7 +62943,7 @@ static int balance_nonroot( assert( szScratch<=6*(int)pBt->pageSize ); b.apCell = sqlite3ScratchMalloc( szScratch ); if( b.apCell==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto balance_cleanup; } b.szCell = (u16*)&b.apCell[nMaxCells]; @@ -60921,9 +63002,8 @@ static int balance_nonroot( ** long be able to find the cells if a pointer to each cell is not saved ** first. */ - memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit); + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ - memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow); limit = pOld->aiOvfl[0]; for(j=0; j<limit; j++){ b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell)); @@ -61249,8 +63329,7 @@ static int balance_nonroot( ** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || pCell<aOld - || pCell>=&aOld[usableSize] + || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); @@ -61299,9 +63378,9 @@ static int balance_nonroot( ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. + ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" + ** and WITHOUT ROWID tables with exactly one column which is the + ** primary key. */ if( b.szCell[j]==4 ){ assert(leafCorrection==4); @@ -61397,7 +63476,7 @@ static int balance_nonroot( ** by smaller than the child due to the database header, and so all the ** free space needs to be up front. */ - assert( nNew==1 ); + assert( nNew==1 || CORRUPT_DB ); rc = defragmentPage(apNew[0]); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == @@ -61451,9 +63530,6 @@ balance_cleanup: return rc; } -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) -#pragma optimize("", on) -#endif /* @@ -61538,8 +63614,8 @@ static int balance(BtCursor *pCur){ u8 aBalanceQuickSpace[13]; u8 *pFree = 0; - TESTONLY( int balance_quick_called = 0 ); - TESTONLY( int balance_deeper_called = 0 ); + VVA_ONLY( int balance_quick_called = 0 ); + VVA_ONLY( int balance_deeper_called = 0 ); do { int iPage = pCur->iPage; @@ -61552,7 +63628,8 @@ static int balance(BtCursor *pCur){ ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. */ - assert( (balance_deeper_called++)==0 ); + assert( balance_deeper_called==0 ); + VVA_ONLY( balance_deeper_called++ ); rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; @@ -61591,7 +63668,8 @@ static int balance(BtCursor *pCur){ ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( (balance_quick_called++)==0 ); + assert( balance_quick_called==0 ); + VVA_ONLY( balance_quick_called++ ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else #endif @@ -61692,7 +63770,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( return pCur->skipNext; } - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); @@ -61820,10 +63898,23 @@ end_insert: } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at an arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then +** the cursor is left pointing at an arbitrary location after the delete. +** But if that bit is set, then the cursor is left in a state such that +** the next call to BtreeNext() or BtreePrev() moves it to the same row +** as it would have been on if the call to BtreeDelete() had been omitted. +** +** The BTREE_AUXDELETE bit of flags indicates that is one of several deletes +** associated with a single table entry and its indexes. Only one of those +** deletes is considered the "primary" delete. The primary delete occurs +** on a cursor that is not a BTREE_FORDELETE cursor. All but one delete +** operation on non-FORDELETE cursors is tagged with the AUXDELETE flag. +** The BTREE_AUXDELETE bit is a hint that is not used by this implementation, +** but which might be used by alternative storage engines. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -61832,8 +63923,10 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ + u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ - assert( cursorHoldsMutex(pCur) ); + assert( cursorOwnsBtShared(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( pCur->curFlags & BTCF_WriteFlag ); @@ -61841,6 +63934,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ assert( !hasReadConflicts(p, pCur->pgnoRoot) ); assert( pCur->aiIdx[pCur->iPage]<pCur->apPage[pCur->iPage]->nCell ); assert( pCur->eState==CURSOR_VALID ); + assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; @@ -61861,10 +63955,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ + ** making any modifications. */ if( pCur->curFlags & BTCF_Multiple ){ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; @@ -61876,6 +63967,31 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -61929,7 +64045,23 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); + assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } @@ -62236,6 +64368,14 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } + /* + ** It is illegal to drop the sqlite_master table on page 1. But again, + ** this error is caught long before reaching this point. + */ + if( NEVER(iTable<2) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -62246,76 +64386,67 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ *piMoved = 0; - if( iTable>1 ){ #ifdef SQLITE_OMIT_AUTOVACUUM - freePage(pPage, &rc); - releasePage(pPage); + freePage(pPage, &rc); + releasePage(pPage); #else - if( pBt->autoVacuum ){ - Pgno maxRootPgno; - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); - - if( iTable==maxRootPgno ){ - /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. - */ - freePage(pPage, &rc); - releasePage(pPage); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the - ** gap left by the deleted root-page. - */ - MemPage *pMove; - releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - freePage(pMove, &rc); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - *piMoved = maxRootPgno; - } + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); - /* Set the new 'max-root-page' value in the database header. This - ** is the old value less one, less one more if that happens to - ** be a root-page number, less one again if that is the - ** PENDING_BYTE_PAGE. + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. */ - maxRootPgno--; - while( maxRootPgno==PENDING_BYTE_PAGE(pBt) - || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ - maxRootPgno--; + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; } - assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - - rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); }else{ - freePage(pPage, &rc); + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; } -#endif - }else{ - /* If sqlite3BtreeDropTable was called on page 1. - ** This really never should happen except in a corrupt - ** database. + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. */ - zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); releasePage(pPage); } +#endif return rc; } SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ @@ -62494,7 +64625,6 @@ static void checkAppendMsg( ... ){ va_list ap; - char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -62503,10 +64633,9 @@ static void checkAppendMsg( sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); - sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); + sqlite3XPrintf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); } - sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); + sqlite3VXPrintf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==STRACCUM_NOMEM ){ pCheck->mallocFailed = 1; @@ -62610,7 +64739,7 @@ static void checkList( break; } if( checkRef(pCheck, iPage) ) break; - if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } @@ -62653,6 +64782,10 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); + + if( isFreeList && N<(iPage!=0) ){ + checkAppendMsg(pCheck, "free-page count in header is too small"); + } } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -62852,9 +64985,9 @@ static int checkTreePage( if( info.nPayload>info.nLocal ){ int nPage; /* Number of pages on the overflow chain */ Pgno pgnoOvfl; /* First page of the overflow chain */ - assert( pc + info.iOverflow <= usableSize ); + assert( pc + info.nSize - 4 <= usableSize ); nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); - pgnoOvfl = get4byte(&pCell[info.iOverflow]); + pgnoOvfl = get4byte(&pCell[info.nSize - 4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); @@ -63003,7 +65136,8 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 ); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); @@ -63016,6 +65150,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.aPgRef = 0; sCheck.heap = 0; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); + sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; if( sCheck.nPage==0 ){ goto integrity_ck_cleanup; } @@ -63255,7 +65390,7 @@ SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ */ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ int rc; - assert( cursorHoldsMutex(pCsr) ); + assert( cursorOwnsBtShared(pCsr) ); assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); assert( pCsr->curFlags & BTCF_Incrblob ); @@ -63344,22 +65479,12 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ } /* -** set the mask of hint flags for cursor pCsr. -*/ -SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ - assert( mask==BTREE_BULKLOAD || mask==BTREE_SEEK_EQ || mask==0 ); - pCsr->hints = mask; -} - -#ifdef SQLITE_DEBUG -/* ** Return true if the cursor has a hint specified. This routine is ** only used from within assert() statements */ SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ return (pCsr->hints & mask)!=0; } -#endif /* ** Return true if the given Btree is read-only. @@ -63373,6 +65498,15 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ */ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +#if !defined(SQLITE_OMIT_SHARED_CACHE) +/* +** Return true if the Btree passed as the only argument is sharable. +*/ +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} +#endif + /************** End of btree.c ***********************************************/ /************** Begin file backup.c ******************************************/ /* @@ -63465,7 +65599,7 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); if( pParse==0 ){ sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pParse->db = pDb; if( sqlite3OpenTempDatabase(pParse) ){ @@ -63559,7 +65693,7 @@ SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM); + sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); } } @@ -63670,7 +65804,7 @@ static int backupOnePage( DbPage *pDestPg = 0; Pgno iDest = (Pgno)(iOff/nDestPgsz)+1; if( iDest==PENDING_BYTE_PAGE(p->pDest->pBt) ) continue; - if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg, 0)) && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) ){ const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; @@ -63796,8 +65930,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, - PAGER_GET_READONLY); + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg,PAGER_GET_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); @@ -63897,7 +66030,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ DbPage *pPg; - rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + rc = sqlite3PagerGet(pDestPager, iPg, &pPg, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg); sqlite3PagerUnref(pPg); @@ -63917,7 +66050,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ ){ PgHdr *pSrcPg = 0; const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1); - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg, 0); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); @@ -63959,7 +66092,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ } if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } p->rc = rc; } @@ -64146,6 +66279,10 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; +#ifdef SQLITE_HAS_CODEC + sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +#endif + /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes @@ -64290,6 +66427,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); + testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already ** contain a valid string or blob value. */ @@ -64311,7 +66449,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre sqlite3VdbeMemSetNull(pMem); pMem->z = 0; pMem->szMalloc = 0; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } @@ -64369,7 +66507,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ f = pMem->flags; if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){ if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; @@ -64401,7 +66539,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ nByte = 1; } if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(&pMem->z[pMem->n], 0, pMem->u.nZero); @@ -64418,7 +66556,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ */ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; @@ -64467,7 +66605,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 @@ -64893,7 +67031,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ assert( db!=0 ); assert( (pMem->flags & MEM_RowSet)==0 ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRaw(db, 64); + pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; pMem->szMalloc = 0; @@ -64934,7 +67072,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + for(i=0, pX=pVdbe->aMem; i<pVdbe->nMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ pX->flags |= MEM_Undefined; pX->pScopyFrom = 0; @@ -65078,7 +67216,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( testcase( nAlloc==31 ); testcase( nAlloc==32 ); if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); }else if( xDel==SQLITE_DYNAMIC ){ @@ -65098,7 +67236,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( #ifndef SQLITE_OMIT_UTF16 if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } #endif @@ -65329,7 +67467,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** to be a scalar SQL function. If ** ** * all function arguments are SQL literals, -** * the SQLITE_FUNC_CONSTANT function flag is set, and +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and ** * the SQLITE_FUNC_NEEDCOLL function flag is not set, ** ** then this routine attempts to invoke the SQL function. Assuming no @@ -65359,7 +67497,6 @@ static int valueFromFunction( FuncDef *pFunc = 0; /* Function definition */ sqlite3_value *pVal = 0; /* New value */ int rc = SQLITE_OK; /* Return code */ - int nName; /* Size of function name in bytes */ ExprList *pList = 0; /* Function arguments */ int i; /* Iterator variable */ @@ -65367,10 +67504,9 @@ static int valueFromFunction( assert( (p->flags & EP_TokenOnly)==0 ); pList = p->x.pList; if( pList ) nVal = pList->nExpr; - nName = sqlite3Strlen30(p->u.zToken); - pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0); + pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); assert( pFunc ); - if( (pFunc->funcFlags & SQLITE_FUNC_CONSTANT)==0 + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; @@ -65379,7 +67515,7 @@ static int valueFromFunction( if( pList ){ apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal); if( apVal==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto value_from_function_out; } for(i=0; i<nVal; i++){ @@ -65390,7 +67526,7 @@ static int valueFromFunction( pVal = valueNew(db, pCtx); if( pVal==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto value_from_function_out; } @@ -65398,7 +67534,7 @@ static int valueFromFunction( memset(&ctx, 0, sizeof(ctx)); ctx.pOut = pVal; ctx.pFunc = pFunc; - pFunc->xFunc(&ctx, nVal, apVal); + pFunc->xSFunc(&ctx, nVal, apVal); if( ctx.isError ){ rc = ctx.isError; sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); @@ -65460,7 +67596,7 @@ static int valueFromExpr( *ppVal = 0; return SQLITE_OK; } - while( (op = pExpr->op)==TK_UPLUS ) pExpr = pExpr->pLeft; + while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; /* Compressed expressions only appear when parsing the DEFAULT clause @@ -65555,7 +67691,7 @@ static int valueFromExpr( return rc; no_mem: - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -65563,7 +67699,7 @@ no_mem: #else assert( pCtx==0 ); sqlite3ValueFree(pVal); #endif - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* @@ -65601,21 +67737,20 @@ static void recordFunc( sqlite3_value **argv ){ const int file_format = 1; - int iSerial; /* Serial type */ + u32 iSerial; /* Serial type */ int nSerial; /* Bytes of space for iSerial as varint */ - int nVal; /* Bytes of space required for argv[0] */ + u32 nVal; /* Bytes of space required for argv[0] */ int nRet; sqlite3 *db; u8 *aRet; UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format); + iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); nSerial = sqlite3VarintLen(iSerial); - nVal = sqlite3VdbeSerialTypeLen(iSerial); db = sqlite3_context_db_handle(context); nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRaw(db, nRet); + aRet = sqlite3DbMallocRawNN(db, nRet); if( aRet==0 ){ sqlite3_result_error_nomem(context); }else{ @@ -65631,15 +67766,10 @@ static void recordFunc( ** Register built-in functions used to help read ANALYZE data. */ SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { + static FuncDef aAnalyzeTableFuncs[] = { FUNCTION(sqlite_record, 1, 0, 0, recordFunc), }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); - for(i=0; i<ArraySize(aAnalyzeTableFuncs); i++){ - sqlite3FuncDefInsert(pHash, &aFunc[i]); - } + sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); } /* @@ -65818,7 +67948,7 @@ SQLITE_PRIVATE int sqlite3Stat4Column( if( iField>nRec ) return SQLITE_CORRUPT_BKPT; if( pMem==0 ){ pMem = *ppVal = sqlite3ValueNew(db); - if( pMem==0 ) return SQLITE_NOMEM; + if( pMem==0 ) return SQLITE_NOMEM_BKPT; } sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); pMem->enc = ENC(db); @@ -65931,6 +68061,7 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); assert( pParse->nOpAlloc==0 ); + assert( pParse->szOpAlloc==0 ); return p; } @@ -65964,7 +68095,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepa */ SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return (p && p->isPrepareV2) ? p->zSql : 0; + return p ? p->zSql : 0; } /* @@ -66020,10 +68151,11 @@ static int growOpArray(Vdbe *v, int nOp){ assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ - p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); + p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); + p->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } - return (pNew ? SQLITE_OK : SQLITE_NOMEM); + return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); } #ifdef SQLITE_DEBUG @@ -66053,17 +68185,21 @@ static void test_addop_breakpoint(void){ ** the sqlite3VdbeChangeP4() function to change the value of the P4 ** operand. */ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->pParse->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->pParse->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ int i; VdbeOp *pOp; i = p->nOp; assert( p->magic==VDBE_MAGIC_INIT ); - assert( op>0 && op<0xff ); + assert( op>=0 && op<0xff ); if( p->pParse->nOpAlloc<=i ){ - if( growOpArray(p, 1) ){ - return 1; - } + return growOp3(p, op, p1, p2, p3); } p->nOp++; pOp = &p->aOp[i]; @@ -66111,6 +68247,43 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } +/* Generate code for an unconditional jump to instruction iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ + return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); +} + +/* Generate code to cause the string zStr to be loaded into +** register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ + return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); +} + +/* +** Generate code that initializes multiple registers to string or integer +** constants. The registers begin with iDest and increase consecutively. +** One register is initialized for each characgter in zTypes[]. For each +** "s" character in zTypes[], the register is a string if the argument is +** not NULL, or OP_Null if the value is a null pointer. For each "i" character +** in zTypes[], the register is initialized to an integer. +*/ +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ + va_list ap; + int i; + char c; + va_start(ap, zTypes); + for(i=0; (c = zTypes[i])!=0; i++){ + if( c=='s' ){ + const char *z = va_arg(ap, const char*); + sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0); + }else{ + assert( c=='i' ); + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); + } + } + va_end(ap); +} /* ** Add an opcode that includes the p4 value as a pointer. @@ -66130,7 +68303,8 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( } /* -** Add an opcode that includes the p4 value with a P4_INT64 type. +** Add an opcode that includes the p4 value with a P4_INT64 or +** P4_REAL type. */ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( Vdbe *p, /* Add the opcode to this VM */ @@ -66141,7 +68315,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( const u8 *zP4, /* The P4 operand */ int p4type /* P4 operand type */ ){ - char *p4copy = sqlite3DbMallocRaw(sqlite3VdbeDb(p), 8); + char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); if( p4copy ) memcpy(p4copy, zP4, 8); return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); } @@ -66156,8 +68330,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( */ SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; - int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); - sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j); } @@ -66177,6 +68350,21 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( return addr; } +/* Insert the end of a co-routine +*/ +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + + /* Clear the temporary register cache, thereby ensuring that each + ** co-routine has its own independent set of registers, because co-routines + ** might expect their registers to be preserved across an OP_Yield, and + ** that could cause problems if two or more co-routines are using the same + ** temporary register. + */ + v->pParse->nTempReg = 0; + v->pParse->nRangeReg = 0; +} + /* ** Create a new symbolic label for an instruction that has yet to be ** coded. The symbolic label is really just a negative number. The @@ -66202,7 +68390,7 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ if( p->aLabel ){ p->aLabel[i] = -1; } - return -1-i; + return ADDR(i); } /* @@ -66212,10 +68400,11 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; - int j = -1-x; + int j = ADDR(x); assert( v->magic==VDBE_MAGIC_INIT ); assert( j<p->nLabel ); - if( ALWAYS(j>=0) && p->aLabel ){ + assert( j>=0 ); + if( p->aLabel ){ p->aLabel[j] = v->nOp; } p->iFixedOp = v->nOp - 1; @@ -66228,6 +68417,13 @@ SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ p->runOnlyOnce = 1; } +/* +** Mark the VDBE as one that can only be run multiple times. +*/ +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ + p->runOnlyOnce = 0; +} + #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* @@ -66359,17 +68555,21 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** Loop through the program looking for P2 values that are negative -** on jump instructions. Each such value is a label. Resolve the -** label by setting the P2 value to its correct non-zero value. +** This routine is called after all opcodes have been inserted. It loops +** through all the opcodes and fixes up some details. +** +** (1) For each jump instruction with a negative P2 value (a label) +** resolve the P2 value to an actual address. +** +** (2) Compute the maximum number of arguments used by any SQL function +** and store that value in *pMaxFuncArgs. ** -** This routine is called once after all opcodes have been inserted. +** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately +** indicate what the prepared statement actually does. ** -** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument -** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by -** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** (4) Initialize the p4.xAdvance pointer on opcodes that use it. ** -** The Op.opflags field is set on all opcodes. +** (5) Reclaim the memory allocated for storing labels. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ int i; @@ -66382,7 +68582,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ u8 opcode = pOp->opcode; - /* NOTE: Be sure to update mkopcodeh.awk when adding or removing + /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing ** cases from this switch! */ switch( opcode ){ case OP_Transaction: { @@ -66434,8 +68634,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pOp->opflags = sqlite3OpcodeProperty[opcode]; if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( -1-pOp->p2<pParse->nLabel ); - pOp->p2 = aLabel[-1-pOp->p2]; + assert( ADDR(pOp->p2)<pParse->nLabel ); + pOp->p2 = aLabel[ADDR(pOp->p2)]; } } sqlite3DbFree(p->db, pParse->aLabel); @@ -66454,6 +68654,20 @@ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ } /* +** Verify that at least N opcode slots are available in p without +** having to malloc for more space (except when compiled using +** SQLITE_TEST_REALLOC_STRESS). This interface is used during testing +** to verify that certain calls to sqlite3VdbeAddOpList() can never +** fail due to a OOM fault and hence that the return value from +** sqlite3VdbeAddOpList() will always be non-NULL. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ + assert( p->nOp + N <= p->pParse->nOpAlloc ); +} +#endif + +/* ** This function returns a pointer to the array of opcodes associated with ** the Vdbe passed as the first argument. It is the callers responsibility ** to arrange for the returned array to be eventually freed using the @@ -66478,51 +68692,54 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) } /* -** Add a whole list of operations to the operation stack. Return the -** address of the first operation added. +** Add a whole list of operations to the operation stack. Return a +** pointer to the first operation inserted. +** +** Non-zero P2 arguments to jump instructions are automatically adjusted +** so that the jump target is relative to the first operation inserted. */ -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ - int addr; +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( + Vdbe *p, /* Add opcodes to the prepared statement */ + int nOp, /* Number of opcodes to add */ + VdbeOpList const *aOp, /* The opcodes to be added */ + int iLineno /* Source-file line number of first opcode */ +){ + int i; + VdbeOp *pOut, *pFirst; + assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } - addr = p->nOp; - if( ALWAYS(nOp>0) ){ - int i; - VdbeOpList const *pIn = aOp; - for(i=0; i<nOp; i++, pIn++){ - int p2 = pIn->p2; - VdbeOp *pOut = &p->aOp[i+addr]; - pOut->opcode = pIn->opcode; - pOut->p1 = pIn->p1; - if( p2<0 ){ - assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); - pOut->p2 = addr + ADDR(p2); - }else{ - pOut->p2 = p2; - } - pOut->p3 = pIn->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; + pFirst = pOut = &p->aOp[p->nOp]; + for(i=0; i<nOp; i++, aOp++, pOut++){ + pOut->opcode = aOp->opcode; + pOut->p1 = aOp->p1; + pOut->p2 = aOp->p2; + assert( aOp->p2>=0 ); + if( (sqlite3OpcodeProperty[aOp->opcode] & OPFLG_JUMP)!=0 && aOp->p2>0 ){ + pOut->p2 += p->nOp; + } + pOut->p3 = aOp->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOut->zComment = 0; + pOut->zComment = 0; #endif #ifdef SQLITE_VDBE_COVERAGE - pOut->iSrcLine = iLineno+i; + pOut->iSrcLine = iLineno+i; #else - (void)iLineno; + (void)iLineno; #endif #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); - } -#endif + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]); } - p->nOp += nOp; +#endif } - return addr; + p->nOp += nOp; + return pFirst; } #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) @@ -66554,49 +68771,23 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( /* -** Change the value of the P1 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. +** Change the value of the opcode, or P1, P2, P3, or P5 operands +** for a specific instruction. */ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; +} SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p1 = val; - } + sqlite3VdbeGetOp(p,addr)->p1 = val; } - -/* -** Change the value of the P2 operand for a specific instruction. -** This routine is useful for setting a jump destination. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p2 = val; - } + sqlite3VdbeGetOp(p,addr)->p2 = val; } - -/* -** Change the value of the P3 operand for a specific instruction. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p3 = val; - } + sqlite3VdbeGetOp(p,addr)->p3 = val; } - -/* -** Change the value of the P5 operand for the most recently -** added operation. -*/ -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ - assert( p!=0 ); - if( p->aOp ){ - assert( p->nOp>0 ); - p->aOp[p->nOp-1].p5 = val; - } +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ + if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5; } /* @@ -66604,8 +68795,8 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ ** the address of the next instruction to be coded. */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - sqlite3VdbeChangeP2(p, addr, p->nOp); p->pParse->iFixedOp = p->nOp - 1; + sqlite3VdbeChangeP2(p, addr, p->nOp); } @@ -66643,6 +68834,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); break; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + sqlite3ExprDelete(db, (Expr*)p4); + break; + } +#endif case P4_MPRINTF: { if( db->pnBytesFreed==0 ) sqlite3_free(p4); break; @@ -66678,7 +68875,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - freeP4(db, pOp->p4type, pOp->p4.p); + if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif @@ -66700,15 +68897,16 @@ SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ /* ** Change the opcode at addr into OP_Noop */ -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - if( addr<p->nOp ){ - VdbeOp *pOp = &p->aOp[addr]; - sqlite3 *db = p->db; - freeP4(db, pOp->p4type, pOp->p4.p); - memset(pOp, 0, sizeof(pOp[0])); - pOp->opcode = OP_Noop; - if( addr==p->nOp-1 ) p->nOp--; - } +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + VdbeOp *pOp; + if( p->db->mallocFailed ) return 0; + assert( addr>=0 && addr<p->nOp ); + pOp = &p->aOp[addr]; + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = P4_NOTUSED; + pOp->p4.z = 0; + pOp->opcode = OP_Noop; + return 1; } /* @@ -66717,8 +68915,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ */ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ - sqlite3VdbeChangeToNoop(p, p->nOp-1); - return 1; + return sqlite3VdbeChangeToNoop(p, p->nOp-1); }else{ return 0; } @@ -66741,16 +68938,34 @@ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** ** If addr<0 then change P4 on the most recently inserted instruction. */ +static void SQLITE_NOINLINE vdbeChangeP4Full( + Vdbe *p, + Op *pOp, + const char *zP4, + int n +){ + if( pOp->p4type ){ + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = 0; + pOp->p4.p = 0; + } + if( n<0 ){ + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 || db->mallocFailed ){ - if( n!=P4_VTAB ){ - freeP4(db, n, (void*)*(char**)&zP4); - } + assert( p->aOp!=0 || db->mallocFailed ); + if( db->mallocFailed ){ + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); return; } assert( p->nOp>0 ); @@ -66759,34 +68974,20 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED - || pOp->p4type==P4_INT32 - || pOp->p4type==P4_KEYINFO ); - freeP4(db, pOp->p4type, pOp->p4.p); - pOp->p4.p = 0; + if( n>=0 || pOp->p4type ){ + vdbeChangeP4Full(p, pOp, zP4, n); + return; + } if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; - }else if( zP4==0 ){ - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - }else if( n==P4_KEYINFO ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_KEYINFO; - }else if( n==P4_VTAB ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_VTAB; - sqlite3VtabLock((VTable *)zP4); - assert( ((VTable *)zP4)->db==p->db ); - }else if( n<0 ){ + }else if( zP4!=0 ){ + assert( n<0 ); pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); } } @@ -66965,74 +69166,138 @@ static int displayComment( } #endif /* SQLITE_DEBUG */ +#if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) +/* +** Translate the P4.pExpr value for an OP_CursorHint opcode into text +** that can be displayed in the P4 column of EXPLAIN output. +*/ +static void displayP4Expr(StrAccum *p, Expr *pExpr){ + const char *zOp = 0; + switch( pExpr->op ){ + case TK_STRING: + sqlite3XPrintf(p, "%Q", pExpr->u.zToken); + break; + case TK_INTEGER: + sqlite3XPrintf(p, "%d", pExpr->u.iValue); + break; + case TK_NULL: + sqlite3XPrintf(p, "NULL"); + break; + case TK_REGISTER: { + sqlite3XPrintf(p, "r[%d]", pExpr->iTable); + break; + } + case TK_COLUMN: { + if( pExpr->iColumn<0 ){ + sqlite3XPrintf(p, "rowid"); + }else{ + sqlite3XPrintf(p, "c%d", (int)pExpr->iColumn); + } + break; + } + case TK_LT: zOp = "LT"; break; + case TK_LE: zOp = "LE"; break; + case TK_GT: zOp = "GT"; break; + case TK_GE: zOp = "GE"; break; + case TK_NE: zOp = "NE"; break; + case TK_EQ: zOp = "EQ"; break; + case TK_IS: zOp = "IS"; break; + case TK_ISNOT: zOp = "ISNOT"; break; + case TK_AND: zOp = "AND"; break; + case TK_OR: zOp = "OR"; break; + case TK_PLUS: zOp = "ADD"; break; + case TK_STAR: zOp = "MUL"; break; + case TK_MINUS: zOp = "SUB"; break; + case TK_REM: zOp = "REM"; break; + case TK_BITAND: zOp = "BITAND"; break; + case TK_BITOR: zOp = "BITOR"; break; + case TK_SLASH: zOp = "DIV"; break; + case TK_LSHIFT: zOp = "LSHIFT"; break; + case TK_RSHIFT: zOp = "RSHIFT"; break; + case TK_CONCAT: zOp = "CONCAT"; break; + case TK_UMINUS: zOp = "MINUS"; break; + case TK_UPLUS: zOp = "PLUS"; break; + case TK_BITNOT: zOp = "BITNOT"; break; + case TK_NOT: zOp = "NOT"; break; + case TK_ISNULL: zOp = "ISNULL"; break; + case TK_NOTNULL: zOp = "NOTNULL"; break; -#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ - || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) + default: + sqlite3XPrintf(p, "%s", "expr"); + break; + } + + if( zOp ){ + sqlite3XPrintf(p, "%s(", zOp); + displayP4Expr(p, pExpr->pLeft); + if( pExpr->pRight ){ + sqlite3StrAccumAppend(p, ",", 1); + displayP4Expr(p, pExpr->pRight); + } + sqlite3StrAccumAppend(p, ")", 1); + } +} +#endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ + + +#if VDBE_DISPLAY_P4 /* ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. */ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ char *zP4 = zTemp; + StrAccum x; assert( nTemp>=20 ); + sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); switch( pOp->p4type ){ case P4_KEYINFO: { - int i, j; + int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->aSortOrder!=0 ); - sqlite3_snprintf(nTemp, zTemp, "k(%d", pKeyInfo->nField); - i = sqlite3Strlen30(zTemp); + sqlite3XPrintf(&x, "k(%d", pKeyInfo->nField); for(j=0; j<pKeyInfo->nField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; - const char *zColl = pColl ? pColl->zName : "nil"; - int n = sqlite3Strlen30(zColl); - if( n==6 && memcmp(zColl,"BINARY",6)==0 ){ - zColl = "B"; - n = 1; - } - if( i+n>nTemp-6 ){ - memcpy(&zTemp[i],",...",4); - break; - } - zTemp[i++] = ','; - if( pKeyInfo->aSortOrder[j] ){ - zTemp[i++] = '-'; - } - memcpy(&zTemp[i], zColl, n+1); - i += n; + const char *zColl = pColl ? pColl->zName : ""; + if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; + sqlite3XPrintf(&x, ",%s%s", pKeyInfo->aSortOrder[j] ? "-" : "", zColl); } - zTemp[i++] = ')'; - zTemp[i] = 0; - assert( i<nTemp ); + sqlite3StrAccumAppend(&x, ")", 1); break; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + displayP4Expr(&x, pOp->p4.pExpr); + break; + } +#endif case P4_COLLSEQ: { CollSeq *pColl = pOp->p4.pColl; - sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); + sqlite3XPrintf(&x, "(%.20s)", pColl->zName); break; } case P4_FUNCDEF: { FuncDef *pDef = pOp->p4.pFunc; - sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } #ifdef SQLITE_DEBUG case P4_FUNCCTX: { FuncDef *pDef = pOp->p4.pCtx->pFunc; - sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } #endif case P4_INT64: { - sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); + sqlite3XPrintf(&x, "%lld", *pOp->p4.pI64); break; } case P4_INT32: { - sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); + sqlite3XPrintf(&x, "%d", pOp->p4.i); break; } case P4_REAL: { - sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); + sqlite3XPrintf(&x, "%.16g", *pOp->p4.pReal); break; } case P4_MEM: { @@ -67040,11 +69305,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ if( pMem->flags & MEM_Str ){ zP4 = pMem->z; }else if( pMem->flags & MEM_Int ){ - sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); + sqlite3XPrintf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->u.r); + sqlite3XPrintf(&x, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ - sqlite3_snprintf(nTemp, zTemp, "NULL"); + zP4 = "NULL"; }else{ assert( pMem->flags & MEM_Blob ); zP4 = "(blob)"; @@ -67054,16 +69319,24 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ #ifndef SQLITE_OMIT_VIRTUALTABLE case P4_VTAB: { sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3_snprintf(nTemp, zTemp, "vtab:%p", pVtab); + sqlite3XPrintf(&x, "vtab:%p", pVtab); break; } #endif case P4_INTARRAY: { - sqlite3_snprintf(nTemp, zTemp, "intarray"); + int i; + int *ai = pOp->p4.ai; + int n = ai[0]; /* The first element of an INTARRAY is always the + ** count of the number of elements to follow */ + for(i=1; i<n; i++){ + sqlite3XPrintf(&x, ",%d", ai[i]); + } + zTemp[0] = '['; + sqlite3StrAccumAppend(&x, "]", 1); break; } case P4_SUBPROGRAM: { - sqlite3_snprintf(nTemp, zTemp, "program"); + sqlite3XPrintf(&x, "program"); break; } case P4_ADVANCE: { @@ -67078,10 +69351,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } } } + sqlite3StrAccumFinish(&x); assert( zP4!=0 ); return zP4; } -#endif +#endif /* VDBE_DISPLAY_P4 */ /* ** Declare to the Vdbe that the BTree object at db->aDb[i] is used. @@ -67100,7 +69374,7 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ } } -#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +#if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** If SQLite is compiled to support shared-cache mode and to be threadsafe, ** this routine obtains the mutex associated with each BtShared structure @@ -67197,7 +69471,6 @@ static void releaseMemArray(Mem *p, int N){ if( p && N ){ Mem *pEnd = &p[N]; sqlite3 *db = p->db; - u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ do{ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); @@ -67233,7 +69506,6 @@ static void releaseMemArray(Mem *p, int N){ p->flags = MEM_Undefined; }while( (++p)<pEnd ); - db->mallocFailed = malloc_failed; } } @@ -67249,6 +69521,7 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ sqlite3VdbeFreeCursor(p->v, apCsr[i]); } releaseMemArray(aMem, p->nChildMem); + sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); sqlite3DbFree(p->v->db, p); } @@ -67291,10 +69564,10 @@ SQLITE_PRIVATE int sqlite3VdbeList( releaseMemArray(pMem, 8); p->pResultSet = 0; - if( p->rc==SQLITE_NOMEM ){ + if( p->rc==SQLITE_NOMEM_BKPT ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ - db->mallocFailed = 1; + sqlite3OomFault(db); return SQLITE_ERROR; } @@ -67395,12 +69668,12 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, 32); + zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); if( zP4!=pMem->z ){ sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); }else{ @@ -67492,43 +69765,46 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ } #endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ -/* -** Allocate space from a fixed size buffer and return a pointer to -** that space. If insufficient space is available, return NULL. -** -** The pBuf parameter is the initial value of a pointer which will -** receive the new memory. pBuf is normally NULL. If pBuf is not -** NULL, it means that memory space has already been allocated and that -** this routine should not allocate any new memory. When pBuf is not -** NULL simply return pBuf. Only allocate new memory space when pBuf -** is NULL. -** -** nByte is the number of bytes of space needed. +/* An instance of this object describes bulk memory available for use +** by subcomponents of a prepared statement. Space is allocated out +** of a ReusableSpace object by the allocSpace() routine below. +*/ +struct ReusableSpace { + u8 *pSpace; /* Available memory */ + int nFree; /* Bytes of available memory */ + int nNeeded; /* Total bytes that could not be allocated */ +}; + +/* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf +** from the ReusableSpace object. Return a pointer to the allocated +** memory on success. If insufficient memory is available in the +** ReusableSpace object, increase the ReusableSpace.nNeeded +** value by the amount needed and return NULL. ** -** *ppFrom points to available space and pEnd points to the end of the -** available space. When space is allocated, *ppFrom is advanced past -** the end of the allocated space. +** If pBuf is not initially NULL, that means that the memory has already +** been allocated by a prior call to this routine, so just return a copy +** of pBuf and leave ReusableSpace unchanged. ** -** *pnByte is a counter of the number of bytes of space that have failed -** to allocate. If there is insufficient space in *ppFrom to satisfy the -** request, then increment *pnByte by the amount of the request. +** This allocator is employed to repurpose unused slots at the end of the +** opcode array of prepared state for other memory needs of the prepared +** statement. */ static void *allocSpace( - void *pBuf, /* Where return pointer will be stored */ - int nByte, /* Number of bytes to allocate */ - u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ - u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ - int *pnByte /* If allocation cannot be made, increment *pnByte */ -){ - assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); - if( pBuf ) return pBuf; - nByte = ROUND8(nByte); - if( &(*ppFrom)[nByte] <= pEnd ){ - pBuf = (void*)*ppFrom; - *ppFrom += nByte; - }else{ - *pnByte += nByte; + struct ReusableSpace *p, /* Bulk memory available for allocation */ + void *pBuf, /* Pointer to a prior allocation */ + int nByte /* Bytes of memory needed */ +){ + assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); + if( pBuf==0 ){ + nByte = ROUND8(nByte); + if( nByte <= p->nFree ){ + p->nFree -= nByte; + pBuf = &p->pSpace[p->nFree]; + }else{ + p->nNeeded += nByte; + } } + assert( EIGHT_BYTE_ALIGNMENT(pBuf) ); return pBuf; } @@ -67551,14 +69827,13 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->magic = VDBE_MAGIC_RUN; #ifdef SQLITE_DEBUG - for(i=1; i<p->nMem; i++){ + for(i=0; i<p->nMem; i++){ assert( p->aMem[i].db==p->db ); } #endif p->pc = -1; p->rc = SQLITE_OK; p->errorAction = OE_Abort; - p->magic = VDBE_MAGIC_RUN; p->nChange = 0; p->cacheCtr = 1; p->minWriteFileFormat = 255; @@ -67601,9 +69876,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( int nArg; /* Number of arguments in subprograms */ int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ - u8 *zCsr; /* Memory available for allocation */ - u8 *zEnd; /* First byte past allocated memory */ - int nByte; /* How much extra memory is needed */ + struct ReusableSpace x; /* Reusable bulk memory */ assert( p!=0 ); assert( p->nOp>0 ); @@ -67619,61 +69892,59 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( nOnce = pParse->nOnce; if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ - /* For each cursor required, also allocate a memory cell. Memory - ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by - ** the vdbe program. Instead they are used to allocate space for - ** VdbeCursor/BtCursor structures. The blob of memory associated with - ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) - ** stores the blob of memory associated with cursor 1, etc. - ** + /* Each cursor uses a memory cell. The first cursor (cursor 0) can + ** use aMem[0] which is not otherwise used by the VDBE program. Allocate + ** space at the end of aMem[] for cursors 1 and greater. ** See also: allocateCursor(). */ nMem += nCursor; + if( nCursor==0 && nMem>0 ) nMem++; /* Space for aMem[0] even if not used */ - /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. + /* Figure out how much reusable memory is available at the end of the + ** opcode array. This extra memory will be reallocated for other elements + ** of the prepared statement. */ - zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ - zEnd = (u8*)&p->aOp[pParse->nOpAlloc]; /* First byte past end of zCsr[] */ + n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ + assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); + x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ + assert( x.nFree>=0 ); + if( x.nFree>0 ){ + memset(x.pSpace, 0, x.nFree); + assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); + } resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain && nMem<10 ){ nMem = 10; } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); p->expired = 0; - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the + /* Memory for registers, parameters, cursor, etc, is allocated in one or two + ** passes. On the first pass, we try to reuse unused memory at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. + ** pass will fill in the remainder using a fresh memory allocation. ** ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly + ** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); + x.nNeeded = 0; + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + p->aOnceFlag = allocSpace(&x, p->aOnceFlag, nOnce); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); #endif - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); + if( x.nNeeded==0 ) break; + x.pSpace = p->pFree = sqlite3DbMallocZero(db, x.nNeeded); + x.nFree = x.nNeeded; + }while( !db->mallocFailed ); p->nCursor = nCursor; p->nOnceFlag = nOnce; @@ -67684,15 +69955,13 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar[n].db = db; } } - if( p->azVar && pParse->nzVar>0 ){ - p->nzVar = pParse->nzVar; - memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); - memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); - } + p->nzVar = pParse->nzVar; + p->azVar = pParse->azVar; + pParse->nzVar = 0; + pParse->azVar = 0; if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ + p->nMem = nMem; + for(n=0; n<nMem; n++){ p->aMem[n].flags = MEM_Undefined; p->aMem[n].db = db; } @@ -67709,23 +69978,34 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } - sqlite3VdbeSorterClose(p->db, pCx); - if( pCx->pBt ){ - sqlite3BtreeClose(pCx->pBt); - /* The pCx->pCursor will be close automatically, if it exists, by - ** the call above. */ - }else if( pCx->pCursor ){ - sqlite3BtreeCloseCursor(pCx->pCursor); - } + assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE ); + switch( pCx->eCurType ){ + case CURTYPE_SORTER: { + sqlite3VdbeSorterClose(p->db, pCx); + break; + } + case CURTYPE_BTREE: { + if( pCx->pBt ){ + sqlite3BtreeClose(pCx->pBt); + /* The pCx->pCursor will be close automatically, if it exists, by + ** the call above. */ + }else{ + assert( pCx->uc.pCursor!=0 ); + sqlite3BtreeCloseCursor(pCx->uc.pCursor); + } + break; + } #ifndef SQLITE_OMIT_VIRTUALTABLE - else if( pCx->pVtabCursor ){ - sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; - const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; - assert( pVtabCursor->pVtab->nRef>0 ); - pVtabCursor->pVtab->nRef--; - pModule->xClose(pVtabCursor); - } + case CURTYPE_VTAB: { + sqlite3_vtab_cursor *pVCur = pCx->uc.pVCur; + const sqlite3_module *pModule = pVCur->pVtab->pModule; + assert( pVCur->pVtab->nRef>0 ); + pVCur->pVtab->nRef--; + pModule->xClose(pVCur); + break; + } #endif + } } /* @@ -67766,6 +70046,9 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; v->db->nChange = pFrame->nDbChange; + sqlite3VdbeDeleteAuxData(v->db, &v->pAuxData, -1, 0); + v->pAuxData = pFrame->pAuxData; + pFrame->pAuxData = 0; return pFrame->pc; } @@ -67788,7 +70071,7 @@ static void closeAllCursors(Vdbe *p){ assert( p->nFrame==0 ); closeCursorsInFrame(p); if( p->aMem ){ - releaseMemArray(&p->aMem[1], p->nMem); + releaseMemArray(p->aMem, p->nMem); } while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; @@ -67797,7 +70080,7 @@ static void closeAllCursors(Vdbe *p){ } /* Delete any auxdata allocations made by the VM */ - if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0); + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p->db, &p->pAuxData, -1, 0); assert( p->pAuxData==0 ); } @@ -67813,7 +70096,7 @@ static void Cleanup(Vdbe *p){ int i; if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 ); if( p->aMem ){ - for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + for(i=0; i<p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); } #endif @@ -67869,7 +70152,7 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( assert( var<COLNAME_N ); if( p->db->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); @@ -67886,7 +70169,9 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( */ static int vdbeCommit(sqlite3 *db, Vdbe *p){ int i; - int nTrans = 0; /* Number of databases with an active write-transaction */ + int nTrans = 0; /* Number of databases with an active write-transaction + ** that are candidates for a two-phase commit using a + ** master-journal */ int rc = SQLITE_OK; int needXcommit = 0; @@ -67914,10 +70199,28 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ + /* Whether or not a database might need a master journal depends upon + ** its journal mode (among other things). This matrix determines which + ** journal modes use a master journal and which do not */ + static const u8 aMJNeeded[] = { + /* DELETE */ 1, + /* PERSIST */ 1, + /* OFF */ 0, + /* TRUNCATE */ 1, + /* MEMORY */ 0, + /* WAL */ 0 + }; + Pager *pPager; /* Pager associated with pBt */ needXcommit = 1; - if( i!=1 ) nTrans++; sqlite3BtreeEnter(pBt); - rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + pPager = sqlite3BtreePager(pBt); + if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF + && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] + ){ + assert( i!=1 ); + nTrans++; + } + rc = sqlite3PagerExclusiveLock(pPager); sqlite3BtreeLeave(pBt); } } @@ -67975,7 +70278,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ #ifndef SQLITE_OMIT_DISKIO else{ sqlite3_vfs *pVfs = db->pVfs; - int needSync = 0; char *zMaster = 0; /* File-name for the master journal */ char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); sqlite3_file *pMaster = 0; @@ -67987,7 +70289,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Select a master journal file name */ nMainFile = sqlite3Strlen30(zMainFile); zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); - if( zMaster==0 ) return SQLITE_NOMEM; + if( zMaster==0 ) return SQLITE_NOMEM_BKPT; do { u32 iRandom; if( retryCount ){ @@ -68035,9 +70337,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ continue; /* Ignore TEMP and :memory: databases */ } assert( zFile[0]!=0 ); - if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ - needSync = 1; - } rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); offset += sqlite3Strlen30(zFile)+1; if( rc!=SQLITE_OK ){ @@ -68052,8 +70351,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Sync the master journal file. If the IOCAP_SEQUENTIAL device ** flag is set this is not required. */ - if( needSync - && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) + if( 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) ){ sqlite3OsCloseFree(pMaster); @@ -68089,7 +70387,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** doing this the directory is synced again before any individual ** transaction files are deleted. */ - rc = sqlite3OsDelete(pVfs, zMaster, needSync); + rc = sqlite3OsDelete(pVfs, zMaster, 1); sqlite3DbFree(db, zMaster); zMaster = 0; if( rc ){ @@ -68276,8 +70574,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; } if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); closeAllCursors(p); @@ -68437,8 +70735,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } p->magic = VDBE_MAGIC_HALT; checkActiveVdbeCnt(db); - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; } /* If the auto-commit flag is set to true, then any locks that were held @@ -68474,12 +70772,12 @@ SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ sqlite3 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ - u8 mallocFailed = db->mallocFailed; + db->bBenignMalloc++; sqlite3BeginBenignMalloc(); if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); sqlite3EndBenignMalloc(); - db->mallocFailed = mallocFailed; + db->bBenignMalloc--; db->errCode = rc; }else{ sqlite3Error(db, rc); @@ -68625,8 +70923,7 @@ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ ** * the corresponding bit in argument mask is clear (where the first ** function parameter corresponds to bit 0 etc.). */ -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ - AuxData **pp = &pVdbe->pAuxData; +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){ while( *pp ){ AuxData *pAux = *pp; if( (iOp<0) @@ -68637,7 +70934,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ pAux->xDelete(pAux->pAux); } *pp = pAux->pNext; - sqlite3DbFree(pVdbe->db, pAux); + sqlite3DbFree(db, pAux); }else{ pp= &pAux->pNext; } @@ -68664,6 +70961,7 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, pSub); } for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + sqlite3DbFree(db, p->azVar); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); @@ -68712,7 +71010,8 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ #endif assert( p->deferredMoveto ); assert( p->isTable ); - rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); + assert( p->eCurType==CURTYPE_BTREE ); + rc = sqlite3BtreeMovetoUnpacked(p->uc.pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; if( res!=0 ) return SQLITE_CORRUPT_BKPT; #ifdef SQLITE_TEST @@ -68732,9 +71031,10 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ */ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ int isDifferentRow, rc; - assert( p->pCursor!=0 ); - assert( sqlite3BtreeCursorHasMoved(p->pCursor) ); - rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow); + assert( p->eCurType==CURTYPE_BTREE ); + assert( p->uc.pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ); + rc = sqlite3BtreeCursorRestore(p->uc.pCursor, &isDifferentRow); p->cacheStatus = CACHE_STALE; if( isDifferentRow ) p->nullRow = 1; return rc; @@ -68745,7 +71045,8 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ ** if need be. Return any I/O error from the restore operation. */ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ - if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ + assert( p->eCurType==CURTYPE_BTREE ); + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ return handleMovedCursor(p); } return SQLITE_OK; @@ -68764,12 +71065,21 @@ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ ** If the cursor is already pointing to the correct row and that row has ** not been deleted out from under the cursor, then this routine is a no-op. */ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ - if( p->deferredMoveto ){ - return handleDeferredMoveto(p); - } - if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){ - return handleMovedCursor(p); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ + VdbeCursor *p = *pp; + if( p->eCurType==CURTYPE_BTREE ){ + if( p->deferredMoveto ){ + int iMap; + if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ + *pp = p->pAltCursor; + *piCol = iMap - 1; + return SQLITE_OK; + } + return handleDeferredMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); + } } return SQLITE_OK; } @@ -68819,11 +71129,13 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ /* ** Return the serial-type for the value stored in pMem. */ -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags; u32 n; + assert( pLen!=0 ); if( flags&MEM_Null ){ + *pLen = 0; return 0; } if( flags&MEM_Int ){ @@ -68837,15 +71149,23 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ u = i; } if( u<=127 ){ - return ((i&1)==i && file_format>=4) ? 8+(u32)u : 1; + if( (i&1)==i && file_format>=4 ){ + *pLen = 0; + return 8+(u32)u; + }else{ + *pLen = 1; + return 1; + } } - if( u<=32767 ) return 2; - if( u<=8388607 ) return 3; - if( u<=2147483647 ) return 4; - if( u<=MAX_6BYTE ) return 5; + if( u<=32767 ){ *pLen = 2; return 2; } + if( u<=8388607 ){ *pLen = 3; return 3; } + if( u<=2147483647 ){ *pLen = 4; return 4; } + if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } + *pLen = 8; return 6; } if( flags&MEM_Real ){ + *pLen = 8; return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); @@ -68854,26 +71174,46 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ if( flags & MEM_Zero ){ n += pMem->u.nZero; } + *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } /* -** The sizes for serial types less than 12 +** The sizes for serial types less than 128 */ static const u8 sqlite3SmallTypeSizes[] = { - 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 + /* 0 1 2 3 4 5 6 7 8 9 */ +/* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, +/* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, +/* 20 */ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, +/* 30 */ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, +/* 40 */ 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, +/* 50 */ 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, +/* 60 */ 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, +/* 70 */ 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, +/* 80 */ 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, +/* 90 */ 39, 39, 40, 40, 41, 41, 42, 42, 43, 43, +/* 100 */ 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, +/* 110 */ 49, 49, 50, 50, 51, 51, 52, 52, 53, 53, +/* 120 */ 54, 54, 55, 55, 56, 56, 57, 57 }; /* ** Return the length of the data corresponding to the supplied serial-type. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ - if( serial_type>=12 ){ + if( serial_type>=128 ){ return (serial_type-12)/2; }else{ + assert( serial_type<12 + || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 ); return sqlite3SmallTypeSizes[serial_type]; } } +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ + assert( serial_type<128 ); + return sqlite3SmallTypeSizes[serial_type]; +} /* ** If we are on an architecture with mixed-endian floating @@ -68969,7 +71309,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) == (int)sqlite3VdbeSerialTypeLen(serial_type) ); len = pMem->n; - memcpy(buf, pMem->z, len); + if( len>0 ) memcpy(buf, pMem->z, len); return len; } @@ -69072,6 +71412,10 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); +#ifdef __HP_cc + /* Work around a sign-extension bug in the HP compiler for HP/UX */ + if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL; +#endif pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; @@ -69369,9 +71713,9 @@ static int vdbeCompareMemString( v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM_BKPT; sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); - if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } @@ -69387,6 +71731,34 @@ static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ return pB1->n - pB2->n; } +/* +** Do a comparison between a 64-bit signed integer and a 64-bit floating-point +** number. Return negative, zero, or positive if the first (i64) is less than, +** equal to, or greater than the second (double). +*/ +static int sqlite3IntFloatCompare(i64 i, double r){ + if( sizeof(LONGDOUBLE_TYPE)>8 ){ + LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; + if( x<r ) return -1; + if( x>r ) return +1; + return 0; + }else{ + i64 y; + double s; + if( r<-9223372036854775808.0 ) return +1; + if( r>9223372036854775807.0 ) return -1; + y = (i64)r; + if( i<y ) return -1; + if( i>y ){ + if( y==SMALLEST_INT64 && r>0.0 ) return -1; + return +1; + } + s = (double)i; + if( s<r ) return -1; + if( s>r ) return +1; + return 0; + } +} /* ** Compare the values contained by the two memory cells, returning @@ -69413,34 +71785,34 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return (f2&MEM_Null) - (f1&MEM_Null); } - /* If one value is a number and the other is not, the number is less. - ** If both are numbers, compare as reals if one is a real, or as integers - ** if both values are integers. + /* At least one of the two values is a number */ if( combined_flags&(MEM_Int|MEM_Real) ){ - double r1, r2; if( (f1 & f2 & MEM_Int)!=0 ){ if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return 1; + if( pMem1->u.i > pMem2->u.i ) return +1; return 0; } - if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->u.r; - }else if( (f1&MEM_Int)!=0 ){ - r1 = (double)pMem1->u.i; - }else{ - return 1; + if( (f1 & f2 & MEM_Real)!=0 ){ + if( pMem1->u.r < pMem2->u.r ) return -1; + if( pMem1->u.r > pMem2->u.r ) return +1; + return 0; } - if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->u.r; - }else if( (f2&MEM_Int)!=0 ){ - r2 = (double)pMem2->u.i; - }else{ - return -1; + if( (f1&MEM_Int)!=0 ){ + if( (f2&MEM_Real)!=0 ){ + return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else{ + return -1; + } } - if( r1<r2 ) return -1; - if( r1>r2 ) return 1; - return 0; + if( (f1&MEM_Real)!=0 ){ + if( (f2&MEM_Int)!=0 ){ + return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); + }else{ + return -1; + } + } + return +1; } /* If one value is a string and the other is a blob, the string is less. @@ -69454,7 +71826,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return -1; } - assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed ); assert( pMem1->enc==SQLITE_UTF8 || pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); @@ -69591,13 +71963,8 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - if( mem1.u.r<rhs ){ - rc = -1; - }else if( mem1.u.r>rhs ){ - rc = +1; - } + rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); i64 rhs = pRhs->u.i; @@ -69621,18 +71988,15 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else{ - double rhs = pRhs->u.r; - double lhs; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - lhs = mem1.u.r; + if( mem1.u.r<pRhs->u.r ){ + rc = -1; + }else if( mem1.u.r>pRhs->u.r ){ + rc = +1; + } }else{ - lhs = (double)mem1.u.i; - } - if( lhs<rhs ){ - rc = -1; - }else if( lhs>rhs ){ - rc = +1; + rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } } @@ -69722,6 +72086,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) || pKeyInfo->db->mallocFailed ); + pPKey2->eqSeen = 1; return pPKey2->default_rc; } SQLITE_PRIVATE int sqlite3VdbeRecordCompare( @@ -69821,6 +72186,7 @@ static int vdbeRecordCompareInt( /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; + pPKey2->eqSeen = 1; } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); @@ -69841,6 +72207,7 @@ static int vdbeRecordCompareString( int serial_type; int res; + assert( pPKey2->aMem[0].flags & MEM_Str ); vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); getVarint32(&aKey1[1], serial_type); if( serial_type<12 ){ @@ -69867,6 +72234,7 @@ static int vdbeRecordCompareString( res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; + pPKey2->eqSeen = 1; } }else if( res>0 ){ res = pPKey2->r2; @@ -70024,9 +72392,11 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( ){ i64 nCellKey = 0; int rc; - BtCursor *pCur = pC->pCursor; + BtCursor *pCur; Mem m; + assert( pC->eCurType==CURTYPE_BTREE ); + pCur = pC->uc.pCursor; assert( sqlite3BtreeCursorIsValid(pCur) ); VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ @@ -70037,7 +72407,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( return SQLITE_CORRUPT_BKPT; } sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); + rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; } @@ -70133,11 +72503,13 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ ** in memory obtained from sqlite3DbMalloc). */ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ - sqlite3 *db = p->db; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; + if( pVtab->zErrMsg ){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -70332,6 +72704,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value *pVal){ SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value *pVal){ + Mem *pMem = (Mem*)pVal; + return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); +} SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } @@ -70510,6 +72886,12 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + pOut->eSubtype = eSubtype & 0xff; + pOut->flags |= MEM_Subtype; +} SQLITE_API void SQLITE_STDCALL sqlite3_result_text( sqlite3_context *pCtx, const char *z, @@ -70606,9 +72988,9 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context *pCtx SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); - pCtx->isError = SQLITE_NOMEM; + pCtx->isError = SQLITE_NOMEM_BKPT; pCtx->fErrorOrAux = 1; - pCtx->pOut->db->mallocFailed = 1; + sqlite3OomFault(pCtx->pOut->db); } /* @@ -70682,7 +73064,7 @@ static int sqlite3Step(Vdbe *p){ db = p->db; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( p->pc<=0 && p->expired ){ @@ -70745,7 +73127,7 @@ static int sqlite3Step(Vdbe *p){ db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ - p->rc = SQLITE_NOMEM; + p->rc = SQLITE_NOMEM_BKPT; } end_of_step: /* At this point local variable rc holds the value that should be @@ -70756,7 +73138,7 @@ end_of_step: ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ @@ -70812,7 +73194,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ v->rc = rc2; } else { v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM; + v->rc = rc = SQLITE_NOMEM_BKPT; } } rc = sqlite3ApiExit(db, rc); @@ -70841,7 +73223,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); + assert( p && p->pOut ); return p->pOut->db; } @@ -70917,7 +73299,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ ** same context that was returned on prior calls. */ SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - assert( p && p->pFunc && p->pFunc->xStep ); + assert( p && p->pFunc && p->pFunc->xFinalize ); assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); if( (p->pMem->flags & MEM_Agg)==0 ){ @@ -71008,7 +73390,7 @@ failed: ** context. */ SQLITE_API int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); + assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); return p->pMem->n; } #endif @@ -71050,18 +73432,19 @@ static const Mem *columnNullValue(void){ #endif = { /* .u = */ {0}, - /* .flags = */ MEM_Null, - /* .enc = */ 0, - /* .n = */ 0, - /* .z = */ 0, - /* .zMalloc = */ 0, - /* .szMalloc = */ 0, - /* .iPadding1 = */ 0, - /* .db = */ 0, - /* .xDel = */ 0, + /* .flags = */ (u16)MEM_Null, + /* .enc = */ (u8)0, + /* .eSubtype = */ (u8)0, + /* .n = */ (int)0, + /* .z = */ (char*)0, + /* .zMalloc = */ (char*)0, + /* .szMalloc = */ (int)0, + /* .uTemp = */ (u32)0, + /* .db = */ (sqlite3*)0, + /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG - /* .pScopyFrom = */ 0, - /* .pFiller = */ 0, + /* .pScopyFrom = */ (Mem*)0, + /* .pFiller = */ (void*)0, #endif }; return &nullMem; @@ -71235,7 +73618,7 @@ static const void *columnName( ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ - db->mallocFailed = 0; + sqlite3OomClear(db); ret = 0; } sqlite3_mutex_leave(db->mutex); @@ -71435,6 +73818,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob( int nData, void (*xDel)(void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( nData<0 ) return SQLITE_MISUSE_BKPT; +#endif return bindText(pStmt, i, zData, nData, xDel, 0); } SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64( @@ -71936,9 +74322,9 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( pVar->flags & MEM_Null ){ sqlite3StrAccumAppend(&out, "NULL", 4); }else if( pVar->flags & MEM_Int ){ - sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); + sqlite3XPrintf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, 0, "%!.15g", pVar->u.r); + sqlite3XPrintf(&out, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 @@ -71959,17 +74345,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( while( nOut<pVar->n && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } #endif - sqlite3XPrintf(&out, 0, "'%.*q'", nOut, pVar->z); + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOut<pVar->n ){ - sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); #endif }else if( pVar->flags & MEM_Zero ){ - sqlite3XPrintf(&out, 0, "zeroblob(%d)", pVar->u.nZero); + sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); @@ -71979,12 +74365,12 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif for(i=0; i<nOut; i++){ - sqlite3XPrintf(&out, 0, "%02x", pVar->z[i]&0xff); + sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOut<pVar->n ){ - sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif } @@ -72164,7 +74550,7 @@ SQLITE_API int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#define isSorter(x) ((x)->pSorter!=0) +#define isSorter(x) ((x)->eCurType==CURTYPE_SORTER) /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL @@ -72175,7 +74561,7 @@ static VdbeCursor *allocateCursor( int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ int iDb, /* Database the cursor belongs to, or -1 */ - int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ + u8 eCurType /* Type of the new cursor */ ){ /* Find the memory cell that will be used to store the blob of memory ** required for this VdbeCursor structure. It is convenient to use a @@ -72191,19 +74577,19 @@ static VdbeCursor *allocateCursor( ** be freed lazily via the sqlite3_release_memory() API. This ** minimizes the number of malloc calls made by the system. ** - ** Memory cells for cursors are allocated at the top of the address - ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for - ** cursor 1 is managed by memory cell (p->nMem-1), etc. + ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from + ** the top of the register space. Cursor 1 is at Mem[p->nMem-1]. + ** Cursor 2 is at Mem[p->nMem-2]. And so forth. */ - Mem *pMem = &p->aMem[p->nMem-iCur]; + Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; int nByte; VdbeCursor *pCx = 0; nByte = ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (isBtreeCursor?sqlite3BtreeCursorSize():0); + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); - assert( iCur<p->nCursor ); + assert( iCur>=0 && iCur<p->nCursor ); if( p->apCsr[iCur] ){ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; @@ -72211,13 +74597,14 @@ static VdbeCursor *allocateCursor( if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); + pCx->eCurType = eCurType; pCx->iDb = iDb; pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; - if( isBtreeCursor ){ - pCx->pCursor = (BtCursor*) + if( eCurType==CURTYPE_BTREE ){ + pCx->uc.pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; - sqlite3BtreeCursorZero(pCx->pCursor); + sqlite3BtreeCursorZero(pCx->uc.pCursor); } } return pCx; @@ -72469,6 +74856,7 @@ static void memTracePrint(Mem *p){ sqlite3VdbeMemPrettyPrint(p, zBuf); printf(" %s", zBuf); } + if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); } static void registerTrace(int iReg, Mem *p){ printf("REG[%d] = ", iReg); @@ -72606,16 +74994,24 @@ static int checkSavepointCount(sqlite3 *db){ /* ** Return the register of pOp->p2 after first preparing it to be ** overwritten with an integer value. -*/ +*/ +static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ + sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; +} static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ Mem *pOut; assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); - pOut->flags = MEM_Int; - return pOut; + if( VdbeMemDynamic(pOut) ){ + return out2PrereleaseWithClear(pOut); + }else{ + pOut->flags = MEM_Int; + return pOut; + } } @@ -72631,6 +75027,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) Op *pOrigOp; /* Value of pOp at the top of the loop */ #endif +#ifdef SQLITE_DEBUG + int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ +#endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ @@ -72659,7 +75058,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -72702,9 +75101,12 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } sqlite3EndBenignMalloc(); #endif - for(pOp=&aOp[p->pc]; rc==SQLITE_OK; pOp++){ + for(pOp=&aOp[p->pc]; 1; pOp++){ + /* Errors are detected by individual opcodes, with an immediate + ** jumps to abort_due_to_error. */ + assert( rc==SQLITE_OK ); + assert( pOp>=aOp && pOp<&aOp[p->nOp]); - if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE start = sqlite3Hwtime(); #endif @@ -72739,33 +75141,33 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); - assert( pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); memAboutToChange(p, &aMem[pOp->p3]); } #endif @@ -72850,7 +75252,7 @@ check_for_interrupt: nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); if( db->xProgress(db->pProgressArg) ){ rc = SQLITE_INTERRUPT; - goto vdbe_error_halt; + goto abort_due_to_error; } } #endif @@ -72864,7 +75266,7 @@ check_for_interrupt: ** and then jump to address P2. */ case OP_Gosub: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); @@ -72904,7 +75306,7 @@ case OP_Return: { /* in1 */ ** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2<p->nOp ); assert( pOp->p3>=0 && pOp->p3<p->nOp ); pOut = &aMem[pOp->p1]; @@ -73129,7 +75531,10 @@ case OP_String8: { /* same as TK_STRING, out2 */ #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); - if( rc==SQLITE_TOOBIG ) goto too_big; + if( rc ){ + assert( rc==SQLITE_TOOBIG ); /* This is the only possible error here */ + goto too_big; + } if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); @@ -73167,13 +75572,15 @@ case OP_String: { /* out2 */ pOut->n = pOp->p1; pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( pOp->p5 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; } +#endif break; } @@ -73194,7 +75601,7 @@ case OP_Null: { /* out2 */ u16 nullFlag; pOut = out2Prerelease(p, pOp); cnt = pOp->p3-pOp->p2; - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; while( cnt>0 ){ pOut++; @@ -73215,7 +75622,7 @@ case OP_Null: { /* out2 */ ** previously copied using OP_SCopy, the copies will continue to be valid. */ case OP_SoftNull: { - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pOut = &aMem[pOp->p1]; pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; break; @@ -73282,8 +75689,8 @@ case OP_Move: { pIn1 = &aMem[p1]; pOut = &aMem[p2]; do{ - assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); - assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); + assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); @@ -73353,6 +75760,22 @@ case OP_SCopy: { /* out2 */ break; } +/* Opcode: IntCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Transfer the integer value held in register P1 into register P2. +** +** This is an optimized version of SCopy that works only for integer +** values. +*/ +case OP_IntCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Int)!=0 ); + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetInt64(pOut, pIn1->u.i); + break; +} + /* Opcode: ResultRow P1 P2 * * * ** Synopsis: output=r[P1@P2] ** @@ -73367,7 +75790,7 @@ case OP_ResultRow: { int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Run the progress counter just before returning. @@ -73377,7 +75800,7 @@ case OP_ResultRow: { && db->xProgress(db->pProgressArg)!=0 ){ rc = SQLITE_INTERRUPT; - goto vdbe_error_halt; + goto abort_due_to_error; } #endif @@ -73387,7 +75810,7 @@ case OP_ResultRow: { if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ assert( db->flags&SQLITE_CountRows ); assert( p->usesStmtJournal ); - break; + goto abort_due_to_error; } /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then @@ -73407,9 +75830,7 @@ case OP_ResultRow: { */ assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); - if( NEVER(rc!=SQLITE_OK) ){ - break; - } + assert( rc==SQLITE_OK ); /* Invalidate all ephemeral cursor row caches */ p->cacheCtr = (p->cacheCtr + 2)|1; @@ -73681,10 +76102,10 @@ case OP_Function0: { assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; pCtx->pOut = 0; pCtx->pFunc = pOp->p4.pFunc; @@ -73723,8 +76144,8 @@ case OP_Function: { MemSetTypeFlag(pCtx->pOut, MEM_Null); pCtx->fErrorOrAux = 0; db->lastRowid = lastRowid; - (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ /* If the function returned an error, throw an exception */ if( pCtx->fErrorOrAux ){ @@ -73732,7 +76153,8 @@ case OP_Function: { sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); rc = pCtx->isError; } - sqlite3VdbeDeleteAuxData(p, pCtx->iOp, pOp->p1); + sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); + if( rc ) goto abort_due_to_error; } /* Copy the result of the function into register P3 */ @@ -73916,6 +76338,7 @@ case OP_Cast: { /* in1 */ rc = ExpandBlob(pIn1); sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); UPDATE_MAX_BLOBSIZE(pIn1); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_CAST */ @@ -74042,6 +76465,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Null); REGISTER_TRACE(pOp->p2, pOut); }else{ @@ -74056,21 +76480,21 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ - if( (pIn1->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); } - if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } }else if( affinity==SQLITE_AFF_TEXT ){ - if( (pIn1->flags & MEM_Str)==0 && (pIn1->flags & (MEM_Int|MEM_Real))!=0 ){ + if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); } - if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ + if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn3, encoding, 1); @@ -74079,15 +76503,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - if( pIn1->flags & MEM_Zero ){ + if( flags1 & MEM_Zero ){ sqlite3VdbeMemExpandBlob(pIn1); flags1 &= ~MEM_Zero; } - if( pIn3->flags & MEM_Zero ){ + if( flags3 & MEM_Zero ){ sqlite3VdbeMemExpandBlob(pIn3); flags3 &= ~MEM_Zero; } - if( db->mallocFailed ) goto no_mem; res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } switch( pOp->opcode ){ @@ -74128,11 +76551,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** The permutation is only valid until the next OP_Compare that has ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should ** occur immediately prior to the OP_Compare. +** +** The first integer in the P4 integer array is the length of the array +** and does not become part of the permutation. */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai; + aPermute = pOp->p4.ai + 1; break; } @@ -74177,11 +76603,11 @@ case OP_Compare: { if( aPermute ){ int k, mx = 0; for(k=0; k<n; k++) if( aPermute[k]>mx ) mx = aPermute[k]; - assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); }else{ - assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 ); } #endif /* SQLITE_DEBUG */ for(i=0; i<n; i++){ @@ -74432,36 +76858,35 @@ case OP_Column: { const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ - u32 szField; /* Number of bytes in the content of a field */ + u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ - u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ + pC = p->apCsr[pOp->p1]; p2 = pOp->p2; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + + /* If the cursor cache is stale, bring it up-to-date */ + rc = sqlite3VdbeCursorMoveto(&pC, &p2); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2<pC->nField ); aOffset = pC->aOffset; -#ifndef SQLITE_OMIT_VIRTUALTABLE - assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ -#endif - pCrsr = pC->pCursor; - assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */ - assert( pCrsr!=0 || pC->nullRow ); /* pC->nullRow on PseudoTables */ + assert( pC->eCurType!=CURTYPE_VTAB ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); + assert( pC->eCurType!=CURTYPE_SORTER ); + pCrsr = pC->uc.pCursor; - /* If the cursor cache is stale, bring it up-to-date */ - rc = sqlite3VdbeCursorMoveto(pC); if( rc ) goto abort_due_to_error; if( pC->cacheStatus!=p->cacheCtr ){ if( pC->nullRow ){ - if( pCrsr==0 ){ - assert( pC->pseudoTableReg>0 ); - pReg = &aMem[pC->pseudoTableReg]; + if( pC->eCurType==CURTYPE_PSEUDO ){ + assert( pC->uc.pseudoTableReg>0 ); + pReg = &aMem[pC->uc.pseudoTableReg]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = avail = pReg->n; @@ -74471,6 +76896,7 @@ case OP_Column: { goto op_column_out; } }else{ + assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); if( pC->isTable==0 ){ assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -74491,31 +76917,17 @@ case OP_Column: { assert( avail<=65536 ); /* Maximum page size is 64KiB */ if( pC->payloadSize <= (u32)avail ){ pC->szRow = pC->payloadSize; + }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; }else{ pC->szRow = avail; } - if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } } pC->cacheStatus = p->cacheCtr; pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; - /* Make sure a corrupt database has not given us an oversize header. - ** Do this now to avoid an oversize memory allocation. - ** - ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte - ** types use so much data space that there can only be 4096 and 32 of - ** them, respectively. So the maximum header length results from a - ** 3-byte type for each of the maximum of 32768 columns plus three - ** extra bytes for the header length itself. 32768*3 + 3 = 98307. - */ - if( offset > 98307 || offset > pC->payloadSize ){ - rc = SQLITE_CORRUPT_BKPT; - goto op_column_error; - } if( avail<offset ){ /* pC->aRow does not have to hold the entire row, but it does at least @@ -74524,6 +76936,20 @@ case OP_Column: { ** dynamically allocated. */ pC->aRow = 0; pC->szRow = 0; + + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( offset > 98307 || offset > pC->payloadSize ){ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } } /* The following goto is an optimization. It can be omitted and @@ -74546,11 +76972,8 @@ case OP_Column: { /* Make sure zData points to enough of the record to cover the header. */ if( pC->aRow==0 ){ memset(&sMem, 0, sizeof(sMem)); - rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], - !pC->isTable, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_error; - } + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem); + if( rc!=SQLITE_OK ) goto abort_due_to_error; zData = (u8*)sMem.z; }else{ zData = pC->aRow; @@ -74558,48 +76981,40 @@ case OP_Column: { /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ i = pC->nHdrParsed; - offset = aOffset[i]; + offset64 = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + aOffset[0]; assert( i<=p2 && zHdr<zEndHdr ); do{ - if( zHdr[0]<0x80 ){ - t = zHdr[0]; + if( (t = zHdr[0])<0x80 ){ zHdr++; + offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ zHdr += sqlite3GetVarint32(zHdr, &t); + offset64 += sqlite3VdbeSerialTypeLen(t); } - pC->aType[i] = t; - szField = sqlite3VdbeSerialTypeLen(t); - offset += szField; - if( offset<szField ){ /* True if offset overflows */ - zHdr = &zEndHdr[1]; /* Forces SQLITE_CORRUPT return below */ - break; - } - i++; - aOffset[i] = offset; + pC->aType[i++] = t; + aOffset[i] = (u32)(offset64 & 0xffffffff); }while( i<=p2 && zHdr<zEndHdr ); pC->nHdrParsed = i; pC->iHdrOffset = (u32)(zHdr - zData); - if( pC->aRow==0 ){ - sqlite3VdbeMemRelease(&sMem); - sMem.flags = MEM_Null; - } /* The record is corrupt if any of the following are true: ** (1) the bytes of the header extend past the declared header size - ** (zHdr>zEndHdr) ** (2) the entire header was used but not all data was used - ** (zHdr==zEndHdr && offset!=pC->payloadSize) ** (3) the end of the data extends beyond the end of the record. - ** (offset > pC->payloadSize) */ - if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize)) - || (offset > pC->payloadSize) + if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) + || (offset64 > pC->payloadSize) ){ + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); rc = SQLITE_CORRUPT_BKPT; - goto op_column_error; + goto abort_due_to_error; } + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + + }else{ + t = 0; } /* If after trying to extract new entries from the header, nHdrParsed is @@ -74614,6 +77029,8 @@ case OP_Column: { } goto op_column_out; } + }else{ + t = pC->aType[p2]; } /* Extract the content for the p2+1-th column. Control can only @@ -74624,11 +77041,32 @@ case OP_Column: { assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); - t = pC->aType[p2]; + assert( t==pC->aType[p2] ); + pDest->enc = encoding; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); + zData = pC->aRow + aOffset[p2]; + if( t<12 ){ + sqlite3VdbeSerialGet(zData, t, pDest); + }else{ + /* If the column value is a string, we need a persistent value, not + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). + */ + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; + pDest->n = len = (t-12)/2; + if( pDest->szMalloc < len+2 ){ + pDest->flags = MEM_Null; + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; + }else{ + pDest->z = pDest->zMalloc; + } + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = aFlag[t&1]; + } }else{ /* This branch happens only when content is on overflow pages */ if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 @@ -74640,39 +77078,19 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. NULL will work for the value for strings - ** and blobs and whatever is in the payloadSize64 variable - ** will work for everything else. */ - sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); + ** content from disk. */ + static u8 aZero[8]; /* This is the bogus content */ + sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, pDest); - if( rc!=SQLITE_OK ){ - goto op_column_error; - } + if( rc!=SQLITE_OK ) goto abort_due_to_error; sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); pDest->flags &= ~MEM_Ephem; } } - pDest->enc = encoding; op_column_out: - /* If the column value is an ephemeral string, go ahead and persist - ** that string in case the cursor moves before the column value is - ** used. The following code does the equivalent of Deephemeralize() - ** but does it faster. */ - if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ - fx = pDest->flags & (MEM_Str|MEM_Blob); - assert( fx!=0 ); - zData = (const u8*)pDest->z; - len = pDest->n; - if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; - memcpy(pDest->z, zData, len); - pDest->z[len] = 0; - pDest->z[len+1] = 0; - pDest->flags = fx|MEM_Term; - } -op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); break; @@ -74696,7 +77114,7 @@ case OP_Affinity: { assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (cAff = *(zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); + assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); applyAffinity(pIn1, cAff, encoding); pIn1++; @@ -74736,7 +77154,7 @@ case OP_MakeRecord: { int file_format; /* File format to use for encoding */ int i; /* Space used in zNewRecord[] header */ int j; /* Space used in zNewRecord[] content */ - int len; /* Length of a field */ + u32 len; /* Length of a field */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -74758,7 +77176,7 @@ case OP_MakeRecord: { nZero = 0; /* Number of zero bytes at the end of the record */ nField = pOp->p1; zAffinity = pOp->p4.z; - assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 ); pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; @@ -74786,8 +77204,7 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format); - len = sqlite3VdbeSerialTypeLen(serial_type); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); if( pRec->flags & MEM_Zero ){ if( nData ){ if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; @@ -74849,7 +77266,7 @@ case OP_MakeRecord: { assert( i==nHdr ); assert( j==nByte ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ @@ -74873,10 +77290,12 @@ case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; - pCrsr = p->apCsr[pOp->p1]->pCursor; + assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); + pCrsr = p->apCsr[pOp->p1]->uc.pCursor; assert( pCrsr ); nEntry = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3BtreeCount(pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; break; @@ -74933,7 +77352,7 @@ case OP_Savepoint: { #endif /* Create a new savepoint structure. */ - pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); + pNew = sqlite3DbMallocRawNN(db, sizeof(Savepoint)+nName+1); if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); @@ -75054,6 +77473,7 @@ case OP_Savepoint: { } } } + if( rc ) goto abort_due_to_error; break; } @@ -75070,38 +77490,37 @@ case OP_Savepoint: { case OP_AutoCommit: { int desiredAutoCommit; int iRollback; - int turnOnAC; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; - turnOnAC = desiredAutoCommit && !db->autoCommit; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); - if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ - /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. - */ - sqlite3VdbeError(p, "cannot commit transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else if( desiredAutoCommit!=db->autoCommit ){ + if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; + }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3VdbeError(p, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + goto abort_due_to_error; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -75118,6 +77537,7 @@ case OP_AutoCommit: { "cannot commit - no transaction is active")); rc = SQLITE_ERROR; + goto abort_due_to_error; } break; } @@ -75173,9 +77593,11 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); + if( (rc&0xff)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); - p->rc = rc = SQLITE_BUSY; + p->rc = rc; goto vdbe_return; } if( rc!=SQLITE_OK ){ @@ -75238,6 +77660,7 @@ case OP_Transaction: { p->expired = 1; rc = SQLITE_SCHEMA; } + if( rc ) goto abort_due_to_error; break; } @@ -75274,15 +77697,15 @@ case OP_ReadCookie: { /* out2 */ /* Opcode: SetCookie P1 P2 P3 * * ** -** Write the content of register P3 (interpreted as an integer) -** into cookie number P2 of database P1. P2==1 is the schema version. -** P2==2 is the database format. P2==3 is the recommended pager cache +** Write the integer value P3 into cookie number P2 of database P1. +** P2==1 is the schema version. P2==2 is the database format. +** P2==3 is the recommended pager cache ** size, and so forth. P1==0 is the main database file and P1==1 is the ** database file used to store temporary tables. ** ** A transaction must be started before executing this opcode. */ -case OP_SetCookie: { /* in3 */ +case OP_SetCookie: { Db *pDb; assert( pOp->p2<SQLITE_N_BTREE_META ); assert( pOp->p1>=0 && pOp->p1<db->nDb ); @@ -75291,17 +77714,15 @@ case OP_SetCookie: { /* in3 */ pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); - pIn3 = &aMem[pOp->p3]; - sqlite3VdbeMemIntegerify(pIn3); /* See note about index shifting on OP_ReadCookie */ - rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = (int)pIn3->u.i; + pDb->pSchema->schema_cookie = pOp->p3; db->flags |= SQLITE_InternChanges; }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ - pDb->pSchema->file_format = (u8)pIn3->u.i; + pDb->pSchema->file_format = pOp->p3; } if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database @@ -75309,6 +77730,7 @@ case OP_SetCookie: { /* in3 */ sqlite3ExpirePreparedStatements(db); p->expired = 0; } + if( rc ) goto abort_due_to_error; break; } @@ -75399,7 +77821,6 @@ case OP_ReopenIdx: { case OP_OpenRead: case OP_OpenWrite: - assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR|OPFLAG_SEEKEQ))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx @@ -75407,7 +77828,7 @@ case OP_OpenWrite: if( p->expired ){ rc = SQLITE_ABORT_ROLLBACK; - break; + goto abort_due_to_error; } nField = 0; @@ -75420,7 +77841,8 @@ case OP_OpenWrite: pX = pDb->pBt; assert( pX!=0 ); if( pOp->opcode==OP_OpenWrite ){ - wrFlag = 1; + assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); + wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; @@ -75430,7 +77852,7 @@ case OP_OpenWrite: } if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); - assert( p2<=(p->nMem-p->nCursor) ); + assert( p2<=(p->nMem+1 - p->nCursor) ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -75440,10 +77862,7 @@ case OP_OpenWrite: ** that opcode will always set the p2 value to 2 or more or else fail. ** If there were a failure, the prepared statement would have halted ** before reaching this instruction. */ - if( NEVER(p2<2) ) { - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; - } + assert( p2>=2 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; @@ -75456,12 +77875,15 @@ case OP_OpenWrite: assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ - pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); + pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; pCur->pgnoRoot = p2; - rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); +#ifdef SQLITE_DEBUG + pCur->wrFlag = wrFlag; +#endif + rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->uc.pCursor); pCur->pKeyInfo = pKeyInfo; /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point @@ -75472,8 +77894,13 @@ case OP_OpenWrite: open_cursor_set_hints: assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); - sqlite3BtreeCursorHints(pCur->pCursor, - (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); + testcase( pOp->p5 & OPFLAG_BULKCSR ); +#ifdef SQLITE_ENABLE_CURSOR_HINTS + testcase( pOp->p2 & OPFLAG_SEEKEQ ); +#endif + sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); + if( rc ) goto abort_due_to_error; break; } @@ -75516,7 +77943,7 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->isEphemeral = 1; @@ -75540,14 +77967,17 @@ case OP_OpenEphemeral: { assert( pKeyInfo->db==db ); assert( pKeyInfo->enc==ENC(db) ); pCx->pKeyInfo = pKeyInfo; - rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, pKeyInfo, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, + pKeyInfo, pCx->uc.pCursor); } pCx->isTable = 0; }else{ - rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, 1, 0, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, + 0, pCx->uc.pCursor); pCx->isTable = 1; } } + if( rc ) goto abort_due_to_error; pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); break; } @@ -75567,12 +77997,13 @@ case OP_SorterOpen: { assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); + if( rc ) goto abort_due_to_error; break; } @@ -75587,7 +78018,7 @@ case OP_SequenceTest: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC->pSorter ); + assert( isSorter(pC) ); if( (pC->seqCount++)==0 ){ goto jump_to_p2; } @@ -75615,10 +78046,10 @@ case OP_OpenPseudo: { assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); + pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - pCx->pseudoTableReg = pOp->p2; + pCx->uc.pseudoTableReg = pOp->p2; pCx->isTable = 1; assert( pOp->p5==0 ); break; @@ -75650,7 +78081,7 @@ case OP_Close: { case OP_ColumnsUsed: { VdbeCursor *pC; pC = p->apCsr[pOp->p1]; - assert( pC->pCursor ); + assert( pC->eCurType==CURTYPE_BTREE ); pC->maskUsed = *(u64*)pOp->p4.pI64; break; } @@ -75668,6 +78099,13 @@ case OP_ColumnsUsed: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will always land on a record that equally equals the key, or +** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this +** opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxLE opcode will be skipped if this opcode succeeds, but the +** IdxLE opcode will be used on subsequent loop iterations. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -75726,51 +78164,48 @@ case OP_ColumnsUsed: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. ** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will always land on a record that equally equals the key, or +** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this +** opcode must be followed by an IdxGE opcode with the same arguments. +** The IdxGE opcode will be skipped if this opcode succeeds, but the +** IdxGE opcode will be used on subsequent loop iterations. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump, in3 */ case OP_SeekLE: /* jump, in3 */ case OP_SeekGE: /* jump, in3 */ case OP_SeekGT: { /* jump, in3 */ - int res; - int oc; - VdbeCursor *pC; - UnpackedRecord r; - int nField; - i64 iKey; /* The rowid we are to seek to */ + int res; /* Comparison result */ + int oc; /* Opcode */ + VdbeCursor *pC; /* The cursor to seek */ + UnpackedRecord r; /* The key to seek for */ + int nField; /* Number of columns or fields in the key */ + i64 iKey; /* The rowid we are to seek to */ + int eqOnly; /* Only interested in == results */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); assert( pOp->p2!=0 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( OP_SeekLE == OP_SeekLT+1 ); assert( OP_SeekGE == OP_SeekLT+2 ); assert( OP_SeekGT == OP_SeekLT+3 ); assert( pC->isOrdered ); - assert( pC->pCursor!=0 ); + assert( pC->uc.pCursor!=0 ); oc = pOp->opcode; + eqOnly = 0; pC->nullRow = 0; #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif - /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and - ** OP_SeekLE opcodes are allowed, and these must be immediately followed - ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. - */ -#ifdef SQLITE_DEBUG - if( sqlite3BtreeCursorHasHint(pC->pCursor, BTREE_SEEK_EQ) ){ - assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); - assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); - assert( pOp[1].p1==pOp[0].p1 ); - assert( pOp[1].p2==pOp[0].p2 ); - assert( pOp[1].p3==pOp[0].p3 ); - assert( pOp[1].p4.i==pOp[0].p4.i ); - } -#endif - if( pC->isTable ){ + /* The BTREE_SEEK_EQ flag is only set on index cursors */ + assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 ); + /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ @@ -75813,12 +78248,26 @@ case OP_SeekGT: { /* jump, in3 */ if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } } - rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } }else{ + /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and + ** OP_SeekLE opcodes are allowed, and these must be immediately followed + ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + */ + if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ + eqOnly = 1; + assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp[1].p1==pOp[0].p1 ); + assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p3==pOp[0].p3 ); + assert( pOp[1].p4.i==pOp[0].p4.i ); + } + nField = pOp->p4.i; assert( pOp->p4type==P4_INT32 ); assert( nField>0 ); @@ -75843,10 +78292,15 @@ case OP_SeekGT: { /* jump, in3 */ { int i; for(i=0; i<r.nField; i++) assert( memIsValid(&r.aMem[i]) ); } #endif ExpandBlob(r.aMem); - rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, &r, 0, 0, &res); + r.eqSeen = 0; + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, &r, 0, 0, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } + if( eqOnly && r.eqSeen==0 ){ + assert( res!=0 ); + goto seek_not_found; + } } pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -75856,7 +78310,7 @@ case OP_SeekGT: { /* jump, in3 */ if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); if( res<0 || (res==0 && oc==OP_SeekGT) ){ res = 0; - rc = sqlite3BtreeNext(pC->pCursor, &res); + rc = sqlite3BtreeNext(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; }else{ res = 0; @@ -75865,47 +78319,26 @@ case OP_SeekGT: { /* jump, in3 */ assert( oc==OP_SeekLT || oc==OP_SeekLE ); if( res>0 || (res==0 && oc==OP_SeekLT) ){ res = 0; - rc = sqlite3BtreePrevious(pC->pCursor, &res); + rc = sqlite3BtreePrevious(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. */ - res = sqlite3BtreeEof(pC->pCursor); + res = sqlite3BtreeEof(pC->uc.pCursor); } } +seek_not_found: assert( pOp->p2>0 ); VdbeBranchTaken(res!=0,2); if( res ){ goto jump_to_p2; + }else if( eqOnly ){ + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */ } break; } - -/* Opcode: Seek P1 P2 * * * -** Synopsis: intkey=r[P2] -** -** P1 is an open table cursor and P2 is a rowid integer. Arrange -** for P1 to move so that it points to the rowid given by P2. -** -** This is actually a deferred seek. Nothing actually happens until -** the cursor is used to read a record. That way, if no reads -** occur, no unnecessary I/O happens. -*/ -case OP_Seek: { /* in2 */ - VdbeCursor *pC; - - assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); - assert( pC->pCursor!=0 ); - assert( pC->isTable ); - pC->nullRow = 0; - pIn2 = &aMem[pOp->p2]; - pC->movetoTarget = sqlite3VdbeIntValue(pIn2); - pC->deferredMoveto = 1; - break; -} /* Opcode: Found P1 P2 P3 P4 * @@ -75992,7 +78425,8 @@ case OP_Found: { /* jump, in3 */ pC->seekOp = pOp->opcode; #endif pIn3 = &aMem[pOp->p3]; - assert( pC->pCursor!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); pFree = 0; if( pOp->p4.i>0 ){ @@ -76029,10 +78463,10 @@ case OP_Found: { /* jump, in3 */ } } } - rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res); sqlite3DbFree(db, pFree); if( rc!=SQLITE_OK ){ - break; + goto abort_due_to_error; } pC->seekResult = res; alreadyExists = (res==0); @@ -76054,9 +78488,10 @@ case OP_Found: { /* jump, in3 */ ** ** P1 is the index of a cursor open on an SQL table btree (with integer ** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. If P1 does contain a record -** with rowid P3 then leave the cursor pointing at that record and fall -** through to the next instruction. +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. ** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). @@ -76082,19 +78517,28 @@ case OP_NotExists: { /* jump, in3 */ pC->seekOp = 0; #endif assert( pC->isTable ); - assert( pC->pseudoTableReg==0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); pC->seekResult = res; - if( res!=0 ) goto jump_to_p2; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + goto jump_to_p2; + } + } + if( rc ) goto abort_due_to_error; break; } @@ -76109,6 +78553,7 @@ case OP_NotExists: { /* jump, in3 */ case OP_Sequence: { /* out2 */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); assert( p->apCsr[pOp->p1]!=0 ); + assert( p->apCsr[pOp->p1]->eCurType!=CURTYPE_VTAB ); pOut = out2Prerelease(p, pOp); pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; @@ -76144,7 +78589,8 @@ case OP_NewRowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pCursor!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); { /* The next rowid or record number (different terms for the same ** thing) is obtained in a two-step algorithm. @@ -76172,15 +78618,15 @@ case OP_NewRowid: { /* out2 */ #endif if( !pC->useRandomRowid ){ - rc = sqlite3BtreeLast(pC->pCursor, &res); + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } if( res ){ v = 1; /* IMP: R-61914-48074 */ }else{ - assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); - rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) ); + rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v); assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; @@ -76201,7 +78647,7 @@ case OP_NewRowid: { /* out2 */ pMem = &pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p3]; memAboutToChange(p, pMem); } @@ -76231,11 +78677,12 @@ case OP_NewRowid: { /* out2 */ do{ sqlite3_randomness(sizeof(v), &v); v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ - }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) && (++cnt<100)); - if( rc==SQLITE_OK && res==0 ){ + if( rc ) goto abort_due_to_error; + if( res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } @@ -76311,8 +78758,8 @@ case OP_InsertInt: { assert( memIsValid(pData) ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pCursor!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable ); REGISTER_TRACE(pOp->p2, pData); @@ -76341,7 +78788,7 @@ case OP_InsertInt: { }else{ nZero = 0; } - rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey, + rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey, pData->z, pData->n, nZero, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult ); @@ -76349,7 +78796,8 @@ case OP_InsertInt: { pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ + if( rc ) goto abort_due_to_error; + if( db->xUpdateCallback && pOp->p4.z ){ zDb = db->aDb[pC->iDb].zName; zTbl = pOp->p4.z; op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); @@ -76360,17 +78808,26 @@ case OP_InsertInt: { break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Delete P1 P2 * P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. ** -** The cursor will be left pointing at either the next or the previous +** If the OPFLAG_SAVEPOSITION bit of the P5 parameter is set, then +** the cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within a Next loop. +** the next Next instruction will be a no-op. As a result, in this case +** it is ok to delete a record from within a Next loop. If +** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be +** left in an undefined state. ** -** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is -** incremented (otherwise not). +** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this +** delete one of several associated with deleting a table row and all its +** associated index entries. Exactly one of those deletes is the "primary" +** delete. The others are all on OPFLAG_FORDELETE cursors or else are +** marked with the AUXDELETE flag. +** +** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row +** change count is incremented (otherwise not). ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. @@ -76382,29 +78839,56 @@ case OP_InsertInt: { */ case OP_Delete: { VdbeCursor *pC; + u8 hasUpdateCallback; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->deferredMoveto==0 ); + hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable; + if( pOp->p5 && hasUpdateCallback ){ + sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget); + } + #ifdef SQLITE_DEBUG /* The seek operation that positioned the cursor prior to OP_Delete will ** have also set the pC->movetoTarget field to the rowid of the row that ** is being deleted */ - if( pOp->p4.z && pC->isTable ){ + if( pOp->p4.z && pC->isTable && pOp->p5==0 ){ i64 iKey = 0; - sqlite3BtreeKeySize(pC->pCursor, &iKey); + sqlite3BtreeKeySize(pC->uc.pCursor, &iKey); assert( pC->movetoTarget==iKey ); } #endif - - rc = sqlite3BtreeDelete(pC->pCursor); + + /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ + assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 ); + assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION ); + assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE ); + +#ifdef SQLITE_DEBUG + if( p->pFrame==0 ){ + if( pC->isEphemeral==0 + && (pOp->p5 & OPFLAG_AUXDELETE)==0 + && (pC->wrFlag & OPFLAG_FORDELETE)==0 + ){ + nExtraDelete++; + } + if( pOp->p2 & OPFLAG_NCHANGE ){ + nExtraDelete--; + } + } +#endif + + rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + if( rc ) goto abort_due_to_error; + if( hasUpdateCallback ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); assert( pC->iDb>=0 ); @@ -76453,6 +78937,7 @@ case OP_SorterCompare: { res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); + if( rc ) goto abort_due_to_error; if( res ) goto jump_to_p2; break; }; @@ -76478,6 +78963,7 @@ case OP_SorterData: { rc = sqlite3VdbeSorterRowkey(pC, pOut); assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); + if( rc ) goto abort_due_to_error; p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; break; } @@ -76517,14 +79003,14 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( isSorter(pC)==0 ); assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isTable==0 || pOp->opcode==OP_RowData ); - assert( pC!=0 ); assert( pC->nullRow==0 ); - assert( pC->pseudoTableReg==0 ); - assert( pC->pCursor!=0 ); - pCrsr = pC->pCursor; + assert( pC->uc.pCursor!=0 ); + pCrsr = pC->uc.pCursor; /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or ** OP_Rewind/Op_Next with no intervening instructions that might invalidate @@ -76566,6 +79052,7 @@ case OP_RowData: { }else{ rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z); } + if( rc ) goto abort_due_to_error; pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); @@ -76592,29 +79079,32 @@ case OP_Rowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 || pC->nullRow ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); if( pC->nullRow ){ pOut->flags = MEM_Null; break; }else if( pC->deferredMoveto ){ v = pC->movetoTarget; #ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( pC->pVtabCursor ){ - pVtab = pC->pVtabCursor->pVtab; + }else if( pC->eCurType==CURTYPE_VTAB ){ + assert( pC->uc.pVCur!=0 ); + pVtab = pC->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xRowid ); - rc = pModule->xRowid(pC->pVtabCursor, &v); + rc = pModule->xRowid(pC->uc.pVCur, &v); sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ - assert( pC->pCursor!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; if( pC->nullRow ){ pOut->flags = MEM_Null; break; } - rc = sqlite3BtreeKeySize(pC->pCursor, &v); + rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v); assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } pOut->u.i = v; @@ -76635,8 +79125,9 @@ case OP_NullRow: { assert( pC!=0 ); pC->nullRow = 1; pC->cacheStatus = CACHE_STALE; - if( pC->pCursor ){ - sqlite3BtreeClearCursor(pC->pCursor); + if( pC->eCurType==CURTYPE_BTREE ){ + assert( pC->uc.pCursor!=0 ); + sqlite3BtreeClearCursor(pC->uc.pCursor); } break; } @@ -76661,7 +79152,8 @@ case OP_Last: { /* jump */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); @@ -76672,6 +79164,7 @@ case OP_Last: { /* jump */ #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; #endif + if( rc ) goto abort_due_to_error; if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; @@ -76729,12 +79222,14 @@ case OP_Rewind: { /* jump */ if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(pC, &res); }else{ - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } + if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2<p->nOp ); VdbeBranchTaken(res!=0,2); @@ -76826,7 +79321,7 @@ case OP_Next: /* jump */ res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); - assert( pC->pCursor ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( res==0 || (res==1 && pC->isTable==0) ); testcase( res==1 ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); @@ -76843,10 +79338,11 @@ case OP_Next: /* jump */ || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE || pC->seekOp==OP_Last ); - rc = pOp->p4.xAdvance(pC->pCursor, &res); + rc = pOp->p4.xAdvance(pC->uc.pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); + if( rc ) goto abort_due_to_error; if( res==0 ){ pC->nullRow = 0; p->aCounter[pOp->p5]++; @@ -76894,22 +79390,22 @@ case OP_IdxInsert: { /* in2 */ pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pC->pCursor!=0 ); + assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); - if( rc==SQLITE_OK ){ - if( pOp->opcode==OP_SorterInsert ){ - rc = sqlite3VdbeSorterWrite(pC, pIn2); - }else{ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3BtreeInsert(pC->pCursor, zKey, nKey, "", 0, 0, pOp->p3, - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); - assert( pC->deferredMoveto==0 ); - pC->cacheStatus = CACHE_STALE; - } + if( rc ) goto abort_due_to_error; + if( pOp->opcode==OP_SorterInsert ){ + rc = sqlite3VdbeSorterWrite(pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; + rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; } + if( rc) goto abort_due_to_error; break; } @@ -76927,29 +79423,48 @@ case OP_IdxDelete: { UnpackedRecord r; assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 ); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.default_rc = 0; r.aMem = &aMem[pOp->p2]; -#ifdef SQLITE_DEBUG - { int i; for(i=0; i<r.nField; i++) assert( memIsValid(&r.aMem[i]) ); } -#endif rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); - if( rc==SQLITE_OK && res==0 ){ - rc = sqlite3BtreeDelete(pCrsr); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); + if( rc ) goto abort_due_to_error; } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; break; } +/* Opcode: Seek P1 * P3 P4 * +** Synopsis: Move P3 to P1.rowid +** +** P1 is an open index cursor and P3 is a cursor on the corresponding +** table. This opcode does a deferred seek of the P3 table cursor +** to the row that corresponds to the current row of P1. +** +** This is a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. +** +** P4 may be an array of integers (type P4_INTARRAY) containing +** one entry for each column in the P3 table. If array entry a(i) +** is non-zero, then reading column a(i)-1 from cursor P3 is +** equivalent to performing the deferred seek and then reading column i +** from P1. This information is stored in P3 and used to redirect +** reads against P3 over to P1, thus possibly avoiding the need to +** seek and read cursor P3. +*/ /* Opcode: IdxRowid P1 P2 * * * ** Synopsis: r[P2]=rowid ** @@ -76959,36 +79474,57 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ +case OP_Seek: case OP_IdxRowid: { /* out2 */ - BtCursor *pCrsr; - VdbeCursor *pC; - i64 rowid; + VdbeCursor *pC; /* The P1 index cursor */ + VdbeCursor *pTabCur; /* The P2 table cursor (OP_Seek only) */ + i64 rowid; /* Rowid that P1 current points to */ - pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - pCrsr = pC->pCursor; - assert( pCrsr!=0 ); - pOut->flags = MEM_Null; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); assert( pC->deferredMoveto==0 ); + assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); - /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happend for an IdxRowid - ** opcode, hence the NEVER() arround the check of the return value. - */ + /* The IdxRowid and Seek opcodes are combined because of the commonality + ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ rc = sqlite3VdbeCursorRestore(pC); + + /* sqlite3VbeCursorRestore() can only fail if the record has been deleted + ** out from under the cursor. That will never happens for an IdxRowid + ** or Seek opcode */ if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); + rc = sqlite3VdbeIdxRowid(db, pC->uc.pCursor, &rowid); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pOut->u.i = rowid; - pOut->flags = MEM_Int; + if( pOp->opcode==OP_Seek ){ + assert( pOp->p3>=0 && pOp->p3<p->nCursor ); + pTabCur = p->apCsr[pOp->p3]; + assert( pTabCur!=0 ); + assert( pTabCur->eCurType==CURTYPE_BTREE ); + assert( pTabCur->uc.pCursor!=0 ); + assert( pTabCur->isTable ); + pTabCur->nullRow = 0; + pTabCur->movetoTarget = rowid; + pTabCur->deferredMoveto = 1; + assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); + pTabCur->aAltMap = pOp->p4.ai; + pTabCur->pAltCursor = pC; + }else{ + pOut = out2Prerelease(p, pOp); + pOut->u.i = rowid; + pOut->flags = MEM_Int; + } + }else{ + assert( pOp->opcode==OP_IdxRowid ); + sqlite3VdbeMemSetNull(&aMem[pOp->p2]); } break; } @@ -77049,7 +79585,8 @@ case OP_IdxGE: { /* jump */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->isOrdered ); - assert( pC->pCursor!=0); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0); assert( pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); assert( pOp->p4type==P4_INT32 ); @@ -77077,6 +79614,7 @@ case OP_IdxGE: { /* jump */ res++; } VdbeBranchTaken(res>0,2); + if( rc ) goto abort_due_to_error; if( res>0 ) goto jump_to_p2; break; } @@ -77106,11 +79644,13 @@ case OP_Destroy: { /* out2 */ int iDb; assert( p->readOnly==0 ); + assert( pOp->p1>1 ); pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; if( db->nVdbeRead > db->nVDestroy+1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; + goto abort_due_to_error; }else{ iDb = pOp->p3; assert( DbMaskTest(p->btreeMask, iDb) ); @@ -77118,8 +79658,9 @@ case OP_Destroy: { /* out2 */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; pOut->u.i = iMoved; + if( rc ) goto abort_due_to_error; #ifndef SQLITE_OMIT_AUTOVACUUM - if( rc==SQLITE_OK && iMoved!=0 ){ + if( iMoved!=0 ){ sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1); /* All OP_Destroy operations occur on the same btree */ assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 ); @@ -77165,6 +79706,7 @@ case OP_Clear: { aMem[pOp->p3].u.i += nChange; } } + if( rc ) goto abort_due_to_error; break; } @@ -77182,11 +79724,13 @@ case OP_ResetSorter: { assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( pC->pSorter ){ - sqlite3VdbeSorterReset(db, pC->pSorter); + if( isSorter(pC) ){ + sqlite3VdbeSorterReset(db, pC->uc.pSorter); }else{ + assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->isEphemeral ); - rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor); + if( rc ) goto abort_due_to_error; } break; } @@ -77235,6 +79779,7 @@ case OP_CreateTable: { /* out2 */ flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); + if( rc ) goto abort_due_to_error; pOut->u.i = pgno; break; } @@ -77275,7 +79820,7 @@ case OP_ParseSchema: { "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", db->aDb[iDb].zName, zMaster, pOp->p4.z); if( zSql==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ assert( db->init.busy==0 ); db->init.busy = 1; @@ -77287,9 +79832,12 @@ case OP_ParseSchema: { db->init.busy = 0; } } - if( rc ) sqlite3ResetAllSchemasOfConnection(db); - if( rc==SQLITE_NOMEM ){ - goto no_mem; + if( rc ){ + sqlite3ResetAllSchemasOfConnection(db); + if( rc==SQLITE_NOMEM ){ + goto no_mem; + } + goto abort_due_to_error; } break; } @@ -77304,6 +79852,7 @@ case OP_ParseSchema: { case OP_LoadAnalysis: { assert( pOp->p1>=0 && pOp->p1<db->nDb ); rc = sqlite3AnalysisLoad(db, pOp->p1); + if( rc ) goto abort_due_to_error; break; } #endif /* !defined(SQLITE_OMIT_ANALYZE) */ @@ -77349,7 +79898,7 @@ case OP_DropTrigger: { #ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* Opcode: IntegrityCk P1 P2 P3 * P5 +/* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in ** register P1 the text of an error message describing any problems. @@ -77360,9 +79909,8 @@ case OP_DropTrigger: { ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** -** The root page numbers of all tables in the database are integer -** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables -** total. +** The root page numbers of all tables in the database are integers +** stored in P4_INTARRAY argument. ** ** If P5 is not zero, the check is done on the auxiliary database ** file, not the main database file. @@ -77372,30 +79920,24 @@ case OP_DropTrigger: { case OP_IntegrityCk: { int nRoot; /* Number of tables to check. (Number of root pages.) */ int *aRoot; /* Array of rootpage numbers for tables to be checked */ - int j; /* Loop counter */ int nErr; /* Number of errors reported */ char *z; /* Text of the error report */ Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); nRoot = pOp->p2; + aRoot = pOp->p4.ai; assert( nRoot>0 ); - aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); - if( aRoot==0 ) goto no_mem; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( aRoot[nRoot]==0 ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); pIn1 = &aMem[pOp->p1]; - for(j=0; j<nRoot; j++){ - aRoot[j] = (int)sqlite3VdbeIntValue(&pIn1[j]); - } - aRoot[j] = 0; assert( pOp->p5<db->nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); - sqlite3DbFree(db, aRoot); pnErr->u.i -= nErr; sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ @@ -77563,7 +80105,7 @@ case OP_Program: { /* jump */ if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ rc = SQLITE_ERROR; sqlite3VdbeError(p, "too many levels of trigger recursion"); - break; + goto abort_due_to_error; } /* Register pRt is used to store the memory required to save the state @@ -77577,6 +80119,8 @@ case OP_Program: { /* jump */ ** variable nMem (and later, VdbeFrame.nChildMem) to this value. */ nMem = pProgram->nMem + pProgram->nCsr; + assert( nMem>0 ); + if( pProgram->nCsr==0 ) nMem++; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) + pProgram->nCsr * sizeof(VdbeCursor *) @@ -77613,7 +80157,8 @@ case OP_Program: { /* jump */ } }else{ pFrame = pRt->u.pFrame; - assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem + || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); assert( (int)(pOp - aOp)==pFrame->pc ); } @@ -77623,12 +80168,15 @@ case OP_Program: { /* jump */ pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; pFrame->nDbChange = p->db->nChange; + assert( pFrame->pAuxData==0 ); + pFrame->pAuxData = p->pAuxData; + p->pAuxData = 0; p->nChange = 0; p->pFrame = pFrame; - p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; + p->aMem = aMem = VdbeFrameMem(pFrame); p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->apCsr = (VdbeCursor **)&aMem[p->nMem]; p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; @@ -77741,12 +80289,12 @@ case OP_MemMax: { /* in2 */ } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 * * * -** Synopsis: if r[P1]>0 goto P2 +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. -** If the value of register P1 is 1 or greater, jump to P2 and -** add the literal value P3 to register P1. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. ** ** If the initial value of register P1 is less than 1, then the ** value is unchanged and control passes through to the next instruction. @@ -77755,38 +80303,55 @@ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); - if( pIn1->u.i>0 ) goto jump_to_p2; + if( pIn1->u.i>0 ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; + } break; } -/* Opcode: IfNeg P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +/* Opcode: OffsetLimit P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) +** +** This opcode performs a commonly used computation associated with +** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** holds the offset counter. The opcode computes the combined value +** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] +** value computed is the total number of rows that will need to be +** visited in order to complete the query. +** +** If r[P3] is zero or negative, that means there is no OFFSET +** and r[P2] is set to be the value of the LIMIT, r[P1]. ** -** Register P1 must contain an integer. Add literal P3 to the value in -** register P1 then if the value of register P1 is less than zero, jump to P2. +** if r[P1] is zero or negative, that means there is no LIMIT +** and r[P2] is set to -1. +** +** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. */ -case OP_IfNeg: { /* jump, in1 */ +case OP_OffsetLimit: { /* in1, out2, in3 */ pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; - VdbeBranchTaken(pIn1->u.i<0, 2); - if( pIn1->u.i<0 ) goto jump_to_p2; + pIn3 = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + assert( pIn1->flags & MEM_Int ); + assert( pIn3->flags & MEM_Int ); + pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0); break; } /* Opcode: IfNotZero P1 P2 P3 * * -** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. If the content of register P1 is -** initially nonzero, then add P3 to P1 and jump to P2. If register P1 is -** initially zero, leave it unchanged and fall through. +** initially nonzero, then subtract P3 from the value in register P1 and +** jump to P2. If register P1 is initially zero, leave it unchanged +** and fall through. */ case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i ){ - pIn1->u.i += pOp->p3; + pIn1->u.i -= pOp->p3; goto jump_to_p2; } break; @@ -77857,10 +80422,10 @@ case OP_AggStep0: { assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; pCtx->pMem = 0; pCtx->pFunc = pOp->p4.pFunc; @@ -77903,13 +80468,14 @@ case OP_AggStep: { pCtx->pOut = &t; pCtx->fErrorOrAux = 0; pCtx->skipFlag = 0; - (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ if( pCtx->fErrorOrAux ){ if( pCtx->isError ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); rc = pCtx->isError; } sqlite3VdbeMemRelease(&t); + if( rc ) goto abort_due_to_error; }else{ assert( t.flags==MEM_Null ); } @@ -77936,12 +80502,13 @@ case OP_AggStep: { */ case OP_AggFinal: { Mem *pMem; - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); + goto abort_due_to_error; } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); @@ -77977,7 +80544,8 @@ case OP_Checkpoint: { || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); - if( rc==SQLITE_BUSY ){ + if( rc ){ + if( rc!=SQLITE_BUSY ) goto abort_due_to_error; rc = SQLITE_OK; aRes[0] = 1; } @@ -78050,7 +80618,7 @@ case OP_JournalMode: { /* out2 */ "cannot change %s wal mode from within a transaction", (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); - break; + goto abort_due_to_error; }else{ if( eOld==PAGER_JOURNALMODE_WAL ){ @@ -78080,9 +80648,7 @@ case OP_JournalMode: { /* out2 */ } #endif /* ifndef SQLITE_OMIT_WAL */ - if( rc ){ - eNew = eOld; - } + if( rc ) eNew = eOld; eNew = sqlite3PagerSetJournalMode(pPager, eNew); pOut->flags = MEM_Str|MEM_Static|MEM_Term; @@ -78090,6 +80656,7 @@ case OP_JournalMode: { /* out2 */ pOut->n = sqlite3Strlen30(pOut->z); pOut->enc = SQLITE_UTF8; sqlite3VdbeChangeEncoding(pOut, encoding); + if( rc ) goto abort_due_to_error; break; }; #endif /* SQLITE_OMIT_PRAGMA */ @@ -78104,6 +80671,7 @@ case OP_JournalMode: { /* out2 */ case OP_Vacuum: { assert( p->readOnly==0 ); rc = sqlite3RunVacuum(&p->zErrMsg, db); + if( rc ) goto abort_due_to_error; break; } #endif @@ -78124,7 +80692,8 @@ case OP_IncrVacuum: { /* jump */ pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); VdbeBranchTaken(rc==SQLITE_DONE,2); - if( rc==SQLITE_DONE ){ + if( rc ){ + if( rc!=SQLITE_DONE ) goto abort_due_to_error; rc = SQLITE_OK; goto jump_to_p2; } @@ -78175,9 +80744,12 @@ case OP_TableLock: { assert( DbMaskTest(p->btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); - if( (rc&0xFF)==SQLITE_LOCKED ){ - const char *z = pOp->p4.z; - sqlite3VdbeError(p, "database table is locked: %s", z); + if( rc ){ + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3VdbeError(p, "database table is locked: %s", z); + } + goto abort_due_to_error; } } break; @@ -78199,6 +80771,7 @@ case OP_VBegin: { pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, pVTab); if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -78227,6 +80800,7 @@ case OP_VCreate: { rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg); } sqlite3VdbeMemRelease(&sMem); + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -78241,6 +80815,7 @@ case OP_VDestroy: { db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); db->nVDestroy--; + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -78254,35 +80829,35 @@ case OP_VDestroy: { */ case OP_VOpen: { VdbeCursor *pCur; - sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; const sqlite3_module *pModule; assert( p->bIsReader ); pCur = 0; - pVtabCursor = 0; + pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ rc = SQLITE_LOCKED; - break; + goto abort_due_to_error; } pModule = pVtab->pModule; - rc = pModule->xOpen(pVtab, &pVtabCursor); + rc = pModule->xOpen(pVtab, &pVCur); sqlite3VtabImportErrmsg(p, pVtab); - if( SQLITE_OK==rc ){ - /* Initialize sqlite3_vtab_cursor base class */ - pVtabCursor->pVtab = pVtab; + if( rc ) goto abort_due_to_error; - /* Initialize vdbe cursor object */ - pCur = allocateCursor(p, pOp->p1, 0, -1, 0); - if( pCur ){ - pCur->pVtabCursor = pVtabCursor; - pVtab->nRef++; - }else{ - assert( db->mallocFailed ); - pModule->xClose(pVtabCursor); - goto no_mem; - } + /* Initialize sqlite3_vtab_cursor base class */ + pVCur->pVtab = pVtab; + + /* Initialize vdbe cursor object */ + pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); + if( pCur ){ + pCur->uc.pVCur = pVCur; + pVtab->nRef++; + }else{ + assert( db->mallocFailed ); + pModule->xClose(pVCur); + goto no_mem; } break; } @@ -78314,7 +80889,7 @@ case OP_VFilter: { /* jump */ const sqlite3_module *pModule; Mem *pQuery; Mem *pArgc; - sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; VdbeCursor *pCur; int res; @@ -78326,9 +80901,9 @@ case OP_VFilter: { /* jump */ pCur = p->apCsr[pOp->p1]; assert( memIsValid(pQuery) ); REGISTER_TRACE(pOp->p3, pQuery); - assert( pCur->pVtabCursor ); - pVtabCursor = pCur->pVtabCursor; - pVtab = pVtabCursor->pVtab; + assert( pCur->eCurType==CURTYPE_VTAB ); + pVCur = pCur->uc.pVCur; + pVtab = pVCur->pVtab; pModule = pVtab->pModule; /* Grab the index number and argc parameters */ @@ -78342,11 +80917,10 @@ case OP_VFilter: { /* jump */ for(i = 0; i<nArg; i++){ apArg[i] = &pArgc[i+1]; } - rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); + rc = pModule->xFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg); sqlite3VtabImportErrmsg(p, pVtab); - if( rc==SQLITE_OK ){ - res = pModule->xEof(pVtabCursor); - } + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pVCur); pCur->nullRow = 0; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; @@ -78369,21 +80943,21 @@ case OP_VColumn: { sqlite3_context sContext; VdbeCursor *pCur = p->apCsr[pOp->p1]; - assert( pCur->pVtabCursor ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pCur->eCurType==CURTYPE_VTAB ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(pDest); break; } - pVtab = pCur->pVtabCursor->pVtab; + pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; MemSetTypeFlag(pDest, MEM_Null); - rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); + rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; @@ -78395,6 +80969,7 @@ case OP_VColumn: { if( sqlite3VdbeMemTooBig(pDest) ){ goto too_big; } + if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -78414,11 +80989,11 @@ case OP_VNext: { /* jump */ res = 0; pCur = p->apCsr[pOp->p1]; - assert( pCur->pVtabCursor ); + assert( pCur->eCurType==CURTYPE_VTAB ); if( pCur->nullRow ){ break; } - pVtab = pCur->pVtabCursor->pVtab; + pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xNext ); @@ -78428,11 +81003,10 @@ case OP_VNext: { /* jump */ ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ - rc = pModule->xNext(pCur->pVtabCursor); + rc = pModule->xNext(pCur->uc.pVCur); sqlite3VtabImportErrmsg(p, pVtab); - if( rc==SQLITE_OK ){ - res = pModule->xEof(pCur->pVtabCursor); - } + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pCur->uc.pVCur); VdbeBranchTaken(!res,2); if( !res ){ /* If there is data, jump to P2 */ @@ -78464,11 +81038,11 @@ case OP_VRename: { testcase( pName->enc==SQLITE_UTF16BE ); testcase( pName->enc==SQLITE_UTF16LE ); rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); - if( rc==SQLITE_OK ){ - rc = pVtab->pModule->xRename(pVtab, pName->z); - sqlite3VtabImportErrmsg(p, pVtab); - p->expired = 0; - } + if( rc ) goto abort_due_to_error; + rc = pVtab->pModule->xRename(pVtab, pName->z); + sqlite3VtabImportErrmsg(p, pVtab); + p->expired = 0; + if( rc ) goto abort_due_to_error; break; } #endif @@ -78517,7 +81091,7 @@ case OP_VUpdate: { pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ rc = SQLITE_LOCKED; - break; + goto abort_due_to_error; } pModule = pVtab->pModule; nArg = pOp->p2; @@ -78549,6 +81123,7 @@ case OP_VUpdate: { }else{ p->nChange++; } + if( rc ) goto abort_due_to_error; } break; } @@ -78640,6 +81215,28 @@ case OP_Init: { /* jump */ break; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* Opcode: CursorHint P1 * * P4 * +** +** Provide a hint to cursor P1 that it only needs to return rows that +** satisfy the Expr in P4. TK_REGISTER terms in the P4 expression refer +** to values currently held in registers. TK_COLUMN terms in the P4 +** expression refer to columns in the b-tree to which cursor P1 is pointing. +*/ +case OP_CursorHint: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1<p->nCursor ); + assert( pOp->p4type==P4_EXPR ); + pC = p->apCsr[pOp->p1]; + if( pC ){ + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorHint(pC->uc.pCursor, BTREE_HINT_RANGE, + pOp->p4.pExpr, aMem); + } + break; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* Opcode: Noop * * * * * ** @@ -78698,14 +81295,19 @@ default: { /* This is really OP_Noop and OP_Explain */ /* If we reach this point, it means that execution is finished with ** an error of some kind. */ -vdbe_error_halt: +abort_due_to_error: + if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT; assert( rc ); + if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){ + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); + } p->rc = rc; + sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); - if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ sqlite3ResetOneSchema(db, resetSchemaOnFault-1); @@ -78719,6 +81321,9 @@ vdbe_return: testcase( nVmStep>0 ); p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); + assert( rc!=SQLITE_OK || nExtraDelete==0 + || sqlite3_strlike("DELETE%",p->zSql,0)!=0 + ); return rc; /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH @@ -78727,36 +81332,25 @@ vdbe_return: too_big: sqlite3VdbeError(p, "string or blob too big"); rc = SQLITE_TOOBIG; - goto vdbe_error_halt; + goto abort_due_to_error; /* Jump to here if a malloc() fails. */ no_mem: - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3VdbeError(p, "out of memory"); - rc = SQLITE_NOMEM; - goto vdbe_error_halt; - - /* Jump to here for any other kind of fatal error. The "rc" variable - ** should hold the error number. - */ -abort_due_to_error: - assert( p->zErrMsg==0 ); - if( db->mallocFailed ) rc = SQLITE_NOMEM; - if( rc!=SQLITE_IOERR_NOMEM ){ - sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); - } - goto vdbe_error_halt; + rc = SQLITE_NOMEM_BKPT; + goto abort_due_to_error; /* Jump to here if the sqlite3_interrupt() API sets the interrupt ** flag. */ abort_due_to_interrupt: assert( db->u1.isInterrupted ); - rc = SQLITE_INTERRUPT; + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; p->rc = rc; sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); - goto vdbe_error_halt; + goto abort_due_to_error; } @@ -78840,7 +81434,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ }else{ p->iOffset = pC->aType[p->iCol + pC->nField]; p->nByte = sqlite3VdbeSerialTypeLen(type); - p->pCsr = pC->pCursor; + p->pCsr = pC->uc.pCursor; sqlite3BtreeIncrblobCursor(p->pCsr); } } @@ -78879,38 +81473,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( ){ int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ - - /* This VDBE program seeks a btree cursor to the identified - ** db/table/row entry. The reason for using a vdbe program instead - ** of writing code to use the b-tree layer directly is that the - ** vdbe program will take advantage of the various transaction, - ** locking and error handling infrastructure built into the vdbe. - ** - ** After seeking the cursor, the vdbe executes an OP_ResultRow. - ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and - ** blob_bytes() functions. - ** - ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the - ** transaction. - */ - static const int iLn = VDBE_OFFSET_LINENO(4); - static const VdbeOpList openBlob[] = { - /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ - {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ - /* One of the following two instructions is replaced by an OP_Noop. */ - {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ - {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ - {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ - {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 6 */ - {OP_ResultRow, 1, 0, 0}, /* 7 */ - {OP_Goto, 0, 4, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ - }; - int rc = SQLITE_OK; char *zErr = 0; Table *pTab; @@ -79011,7 +81573,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; j<pIdx->nKeyCol; j++){ - if( pIdx->aiColumn[j]==iCol ){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ zFault = "indexed"; } } @@ -79028,45 +81591,78 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList openBlob[] = { + {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ + {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ + {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ + {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 4 */ + {OP_ResultRow, 1, 0, 0}, /* 5 */ + {OP_Goto, 0, 2, 0}, /* 6 */ + {OP_Close, 0, 0, 0}, /* 7 */ + {OP_Halt, 0, 0, 0}, /* 8 */ + }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - + VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); - sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ sqlite3VdbeUsesBtree(v, iDb); - /* Configure the OP_TableLock instruction */ + if( db->mallocFailed==0 ){ + assert( aOp!=0 ); + /* Configure the OP_TableLock instruction */ #ifdef SQLITE_OMIT_SHARED_CACHE - sqlite3VdbeChangeToNoop(v, 1); + aOp[0].opcode = OP_Noop; #else - sqlite3VdbeChangeP1(v, 1, iDb); - sqlite3VdbeChangeP2(v, 1, pTab->tnum); - sqlite3VdbeChangeP3(v, 1, flags); - sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); -#endif - - /* Remove either the OP_OpenWrite or OpenRead. Set the P2 - ** parameter of the other to pTab->tnum. */ - sqlite3VdbeChangeToNoop(v, 3 - flags); - sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); - sqlite3VdbeChangeP3(v, 2 + flags, iDb); - - /* Configure the number of columns. Configure the cursor to - ** think that the table has one more column than it really - ** does. An OP_Column to retrieve this imaginary column will - ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type - ** and offset cache without causing any IO. - */ - sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); - sqlite3VdbeChangeP2(v, 6, pTab->nCol); - if( !db->mallocFailed ){ + aOp[0].p1 = iDb; + aOp[0].p2 = pTab->tnum; + aOp[0].p3 = flags; + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + } + if( db->mallocFailed==0 ){ +#endif + + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + if( flags ) aOp[1].opcode = OP_OpenWrite; + aOp[1].p2 = pTab->tnum; + aOp[1].p3 = iDb; + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + aOp[1].p4type = P4_INT32; + aOp[1].p4.i = pTab->nCol+1; + aOp[4].p2 = pTab->nCol; + pParse->nVar = 1; pParse->nMem = 1; pParse->nTab = 1; @@ -79782,7 +82378,7 @@ static int vdbePmaReadBlob( int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); - if( !aNew ) return SQLITE_NOMEM; + if( !aNew ) return SQLITE_NOMEM_BKPT; p->nAlloc = nNew; p->aAlloc = aNew; } @@ -79894,7 +82490,7 @@ static int vdbePmaReaderSeek( int iBuf = pReadr->iReadOff % pgsz; if( pReadr->aBuffer==0 ){ pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); - if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM; + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM_BKPT; pReadr->nBuffer = pgsz; } if( rc==SQLITE_OK && iBuf ){ @@ -79979,7 +82575,7 @@ static int vdbePmaReaderInit( rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ + u64 nByte = 0; /* Size of PMA in bytes */ rc = vdbePmaReadVarint(pReadr, &nByte); pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; @@ -80203,13 +82799,14 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); + assert( pCsr->eCurType==CURTYPE_SORTER ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); - pCsr->pSorter = pSorter; + pCsr->uc.pSorter = pSorter; if( pSorter==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); @@ -80220,7 +82817,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; - pSorter->iPrev = nWorker-1; + pSorter->iPrev = (u8)(nWorker - 1); pSorter->bUseThreads = (pSorter->nTask>1); pSorter->db = db; for(i=0; i<pSorter->nTask; i++){ @@ -80243,7 +82840,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); - if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT; } } @@ -80491,12 +83088,14 @@ SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); - pCsr->pSorter = 0; + pCsr->uc.pSorter = 0; } } @@ -80563,7 +83162,7 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ pTask->pSorter->pKeyInfo, 0, 0, &pFree ); assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ) return SQLITE_NOMEM; + if( pFree==0 ) return SQLITE_NOMEM_BKPT; pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; pTask->pUnpacked->errCode = 0; } @@ -80638,7 +83237,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } while( p ){ @@ -80688,7 +83287,7 @@ static void vdbePmaWriterInit( memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ - p->eFWErr = SQLITE_NOMEM; + p->eFWErr = SQLITE_NOMEM_BKPT; }else{ p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; @@ -80976,7 +83575,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ pSorter->nMemory = sqlite3MallocSize(aMem); }else if( pSorter->list.aMemory ){ pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); - if( !pSorter->list.aMemory ) return SQLITE_NOMEM; + if( !pSorter->list.aMemory ) return SQLITE_NOMEM_BKPT; } rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); @@ -80994,15 +83593,16 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ - int bFlush; /* True to flush contents of memory to PMA */ int nReq; /* Bytes of memory required */ int nPMA; /* Bytes of PMA space required */ int t; /* serial type of first record field */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; getVarint32((const u8*)&pVal->z[1], t); if( t>0 && t<10 && t!=7 ){ pSorter->typeMask &= SORTER_TYPE_INTEGER; @@ -81059,27 +83659,28 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( if( nMin>pSorter->nMemory ){ u8 *aNew; + int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; int nNew = pSorter->nMemory * 2; while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); - if( !aNew ) return SQLITE_NOMEM; - pSorter->list.pList = (SorterRecord*)( - aNew + ((u8*)pSorter->list.pList - pSorter->list.aMemory) - ); + if( !aNew ) return SQLITE_NOMEM_BKPT; + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + if( pSorter->list.pList ){ + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + } }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pNew->u.pNext = pSorter->list.pList; } @@ -81226,7 +83827,7 @@ static int vdbeIncrMergerNew( pTask->file2.iEof += pIncr->mxSz; }else{ vdbeMergeEngineFree(pMerger); - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } return rc; } @@ -81531,10 +84132,10 @@ static int vdbeMergeEngineLevel0( int rc = SQLITE_OK; *ppOut = pNew = vdbeMergeEngineNew(nPMA); - if( pNew==0 ) rc = SQLITE_NOMEM; + if( pNew==0 ) rc = SQLITE_NOMEM_BKPT; for(i=0; i<nPMA && rc==SQLITE_OK; i++){ - i64 nDummy; + i64 nDummy = 0; PmaReader *pReadr = &pNew->aReadr[i]; rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); iOff = pReadr->iEof; @@ -81602,7 +84203,7 @@ static int vdbeSorterAddToTree( if( pReadr->pIncr==0 ){ MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); } @@ -81647,7 +84248,7 @@ static int vdbeSorterMergeTreeBuild( assert( pSorter->bUseThreads || pSorter->nTask==1 ); if( pSorter->nTask>1 ){ pMain = vdbeMergeEngineNew(pSorter->nTask); - if( pMain==0 ) rc = SQLITE_NOMEM; + if( pMain==0 ) rc = SQLITE_NOMEM_BKPT; } #endif @@ -81665,7 +84266,7 @@ static int vdbeSorterMergeTreeBuild( int i; int iSeq = 0; pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); - if( pRoot==0 ) rc = SQLITE_NOMEM; + if( pRoot==0 ) rc = SQLITE_NOMEM_BKPT; for(i=0; i<pTask->nPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ MergeEngine *pMerger = 0; /* New level-0 PMA merger */ int nReader; /* Number of level-0 PMAs to merge */ @@ -81736,7 +84337,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); pSorter->pReader = pReadr; - if( pReadr==0 ) rc = SQLITE_NOMEM; + if( pReadr==0 ) rc = SQLITE_NOMEM_BKPT; } if( rc==SQLITE_OK ){ rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); @@ -81794,9 +84395,11 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ ** in sorted order. */ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; int rc = SQLITE_OK; /* Return code */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, @@ -81840,9 +84443,11 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ ** Advance to the next element in the sorter. */ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; int rc; /* Return code */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); if( pSorter->bUsePMA ){ assert( pSorter->pReader==0 || pSorter->pMerger==0 ); @@ -81902,12 +84507,14 @@ static void *vdbeSorterRowkey( ** Copy the current sorter key into the memory cell pOut. */ SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ - VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorter *pSorter; void *pKey; int nKey; /* Sorter key to copy into pOut */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; pKey = vdbeSorterRowkey(pSorter, &nKey); if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); @@ -81938,17 +84545,21 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( int nKeyCol, /* Compare this many columns */ int *pRes /* OUT: Result of comparison */ ){ - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; - KeyInfo *pKeyInfo = pCsr->pKeyInfo; + VdbeSorter *pSorter; + UnpackedRecord *r2; + KeyInfo *pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + r2 = pSorter->pUnpacked; + pKeyInfo = pCsr->pKeyInfo; if( r2==0 ){ char *p; r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); assert( pSorter->pUnpacked==(UnpackedRecord*)p ); - if( r2==0 ) return SQLITE_NOMEM; + if( r2==0 ) return SQLITE_NOMEM_BKPT; r2->nField = nKeyCol; } assert( r2->nField==nKeyCol ); @@ -81967,265 +84578,6 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( } /************** End of vdbesort.c ********************************************/ -/************** Begin file journal.c *****************************************/ -/* -** 2007 August 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a special kind of sqlite3_file object used -** by SQLite to create journal files if the atomic-write optimization -** is enabled. -** -** The distinctive characteristic of this sqlite3_file is that the -** actual on disk file is created lazily. When the file is created, -** the caller specifies a buffer size for an in-memory buffer to -** be used to service read() and write() requests. The actual file -** on disk is not created or populated until either: -** -** 1) The in-memory representation grows too large for the allocated -** buffer, or -** 2) The sqlite3JournalCreate() function is called. -*/ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE -/* #include "sqliteInt.h" */ - - -/* -** A JournalFile object is a subclass of sqlite3_file used by -** as an open file handle for journal files. -*/ -struct JournalFile { - sqlite3_io_methods *pMethod; /* I/O methods on journal files */ - int nBuf; /* Size of zBuf[] in bytes */ - char *zBuf; /* Space to buffer journal writes */ - int iSize; /* Amount of zBuf[] currently used */ - int flags; /* xOpen flags */ - sqlite3_vfs *pVfs; /* The "real" underlying VFS */ - sqlite3_file *pReal; /* The "real" underlying file descriptor */ - const char *zJournal; /* Name of the journal file */ -}; -typedef struct JournalFile JournalFile; - -/* -** If it does not already exists, create and populate the on-disk file -** for JournalFile p. -*/ -static int createFile(JournalFile *p){ - int rc = SQLITE_OK; - if( !p->pReal ){ - sqlite3_file *pReal = (sqlite3_file *)&p[1]; - rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); - if( rc==SQLITE_OK ){ - p->pReal = pReal; - if( p->iSize>0 ){ - assert(p->iSize<=p->nBuf); - rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); - } - if( rc!=SQLITE_OK ){ - /* If an error occurred while writing to the file, close it before - ** returning. This way, SQLite uses the in-memory journal data to - ** roll back changes made to the internal page-cache before this - ** function was called. */ - sqlite3OsClose(pReal); - p->pReal = 0; - } - } - } - return rc; -} - -/* -** Close the file. -*/ -static int jrnlClose(sqlite3_file *pJfd){ - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - sqlite3OsClose(p->pReal); - } - sqlite3_free(p->zBuf); - return SQLITE_OK; -} - -/* -** Read data from the file. -*/ -static int jrnlRead( - sqlite3_file *pJfd, /* The journal file from which to read */ - void *zBuf, /* Put the results here */ - int iAmt, /* Number of bytes to read */ - sqlite_int64 iOfst /* Begin reading at this offset */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); - }else if( (iAmt+iOfst)>p->iSize ){ - rc = SQLITE_IOERR_SHORT_READ; - }else{ - memcpy(zBuf, &p->zBuf[iOfst], iAmt); - } - return rc; -} - -/* -** Write data to the file. -*/ -static int jrnlWrite( - sqlite3_file *pJfd, /* The journal file into which to write */ - const void *zBuf, /* Take data to be written from here */ - int iAmt, /* Number of bytes to write */ - sqlite_int64 iOfst /* Begin writing at this offset into the file */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ - rc = createFile(p); - } - if( rc==SQLITE_OK ){ - if( p->pReal ){ - rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); - }else{ - memcpy(&p->zBuf[iOfst], zBuf, iAmt); - if( p->iSize<(iOfst+iAmt) ){ - p->iSize = (iOfst+iAmt); - } - } - } - return rc; -} - -/* -** Truncate the file. -*/ -static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsTruncate(p->pReal, size); - }else if( size<p->iSize ){ - p->iSize = size; - } - return rc; -} - -/* -** Sync the file. -*/ -static int jrnlSync(sqlite3_file *pJfd, int flags){ - int rc; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsSync(p->pReal, flags); - }else{ - rc = SQLITE_OK; - } - return rc; -} - -/* -** Query the size of the file in bytes. -*/ -static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsFileSize(p->pReal, pSize); - }else{ - *pSize = (sqlite_int64) p->iSize; - } - return rc; -} - -/* -** Table of methods for JournalFile sqlite3_file object. -*/ -static struct sqlite3_io_methods JournalFileMethods = { - 1, /* iVersion */ - jrnlClose, /* xClose */ - jrnlRead, /* xRead */ - jrnlWrite, /* xWrite */ - jrnlTruncate, /* xTruncate */ - jrnlSync, /* xSync */ - jrnlFileSize, /* xFileSize */ - 0, /* xLock */ - 0, /* xUnlock */ - 0, /* xCheckReservedLock */ - 0, /* xFileControl */ - 0, /* xSectorSize */ - 0, /* xDeviceCharacteristics */ - 0, /* xShmMap */ - 0, /* xShmLock */ - 0, /* xShmBarrier */ - 0 /* xShmUnmap */ -}; - -/* -** Open a journal file. -*/ -SQLITE_PRIVATE int sqlite3JournalOpen( - sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ - const char *zName, /* Name of the journal file */ - sqlite3_file *pJfd, /* Preallocated, blank file handle */ - int flags, /* Opening flags */ - int nBuf /* Bytes buffered before opening the file */ -){ - JournalFile *p = (JournalFile *)pJfd; - memset(p, 0, sqlite3JournalSize(pVfs)); - if( nBuf>0 ){ - p->zBuf = sqlite3MallocZero(nBuf); - if( !p->zBuf ){ - return SQLITE_NOMEM; - } - }else{ - return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); - } - p->pMethod = &JournalFileMethods; - p->nBuf = nBuf; - p->flags = flags; - p->zJournal = zName; - p->pVfs = pVfs; - return SQLITE_OK; -} - -/* -** If the argument p points to a JournalFile structure, and the underlying -** file has not yet been created, create it now. -*/ -SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ - if( p->pMethods!=&JournalFileMethods ){ - return SQLITE_OK; - } - return createFile((JournalFile *)p); -} - -/* -** The file-handle passed as the only argument is guaranteed to be an open -** file. It may or may not be of class JournalFile. If the file is a -** JournalFile, and the underlying file on disk has not yet been opened, -** return 0. Otherwise, return 1. -*/ -SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p){ - return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); -} - -/* -** Return the number of bytes required to store a JournalFile that uses vfs -** pVfs to create the underlying on-disk files. -*/ -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return (pVfs->szOsFile+sizeof(JournalFile)); -} -#endif - -/************** End of journal.c *********************************************/ /************** Begin file memjournal.c **************************************/ /* ** 2008 October 7 @@ -82250,25 +84602,29 @@ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; -/* Space to hold the rollback journal is allocated in increments of -** this many bytes. -** -** The size chosen is a little less than a power of two. That way, -** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This minimizes wasted space in power-of-two -** memory allocators. -*/ -#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) - /* ** The rollback journal is composed of a linked list of these structures. +** +** The zChunk array is always at least 8 bytes in size - usually much more. +** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ + u8 zChunk[8]; /* Content of this chunk */ }; /* +** By default, allocate this many bytes of memory for each FileChunk object. +*/ +#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 + +/* +** For chunk size nChunkSize, return the number of bytes that should +** be allocated for each FileChunk structure. +*/ +#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) + +/* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. */ @@ -82278,14 +84634,22 @@ struct FilePoint { }; /* -** This subclass is a subclass of sqlite3_file. Each open memory-journal +** This structure is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { - sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + int nChunkSize; /* In-memory chunk-size */ + + int nSpill; /* Bytes of data before flushing */ + int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ + + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + const char *zJournal; /* Name of the journal file */ }; /* @@ -82304,37 +84668,95 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; - /* SQLite never tries to read past the end of a rollback journal file */ - assert( iOfst+iAmt<=p->endpoint.iOffset ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( (iAmt+iOfst)>p->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; + } +#endif + assert( (iAmt+iOfst)<=p->endpoint.iOffset ); + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; pChunk=pChunk->pNext ){ - iOff += JOURNAL_CHUNKSIZE; + iOff += p->nChunkSize; } }else{ pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); } - iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + iChunkOffset = (int)(iOfst%p->nChunkSize); do { - int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; - int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); - memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); zOut += nCopy; nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; p->readpoint.pChunk = pChunk; return SQLITE_OK; } /* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(MemJournal *p){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=p->pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } + p->pFirst = 0; +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int memjrnlCreateFile(MemJournal *p){ + int rc; + sqlite3_file *pReal = (sqlite3_file*)p; + MemJournal copy = *p; + + memset(p, 0, sizeof(MemJournal)); + rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = copy.nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ + if( iOff + nChunk > copy.endpoint.iOffset ){ + nChunk = copy.endpoint.iOffset - iOff; + } + rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); + if( rc ) break; + iOff += nChunk; + } + if( rc==SQLITE_OK ){ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(©); + } + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while creating or writing to the file, restore + ** the original before returning. This way, SQLite uses the in-memory + ** journal data to roll back changes made to the internal page-cache + ** before this function was called. */ + sqlite3OsClose(pReal); + *p = copy; + } + return rc; +} + + +/* ** Write data to the file. */ static int memjrnlWrite( @@ -82347,38 +84769,62 @@ static int memjrnlWrite( int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required by sqlite. - */ - assert( iOfst==p->endpoint.iOffset ); - UNUSED_PARAMETER(iOfst); + /* If the file should be created now, create it and write the new data + ** into the file on disk. */ + if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ + int rc = memjrnlCreateFile(p); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); + } + return rc; + } - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); - int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); + }else +#else + assert( iOfst>0 || p->pFirst==0 ); +#endif + { + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM; + memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; + p->nSize = iAmt + iOfst; } - - memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; } return SQLITE_OK; @@ -82386,19 +84832,21 @@ static int memjrnlWrite( /* ** Truncate the file. +** +** If the journal file is already on disk, truncate it there. Or, if it +** is still in main memory but is being truncated to zero bytes in size, +** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - FileChunk *pChunk; - assert(size==0); - UNUSED_PARAMETER(size); - pChunk = p->pFirst; - while( pChunk ){ - FileChunk *pTmp = pChunk; - pChunk = pChunk->pNext; - sqlite3_free(pTmp); - } - sqlite3MemJournalOpen(pJfd); + if( ALWAYS(size==0) ){ + memjrnlFreeChunks(p); + p->nSize = 0; + p->endpoint.pChunk = 0; + p->endpoint.iOffset = 0; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; + } return SQLITE_OK; } @@ -82406,21 +84854,19 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ - memjrnlTruncate(pJfd, 0); + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p); return SQLITE_OK; } - /* ** Sync the file. ** -** Syncing an in-memory journal is a no-op. And, in fact, this routine -** is never called in a working implementation. This implementation -** exists purely as a contingency, in case some malfunction in some other -** part of SQLite causes Sync to be called by mistake. +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. */ -static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + UNUSED_PARAMETER2(pJfd, flags); return SQLITE_OK; } @@ -82459,28 +84905,88 @@ static const struct sqlite3_io_methods MemJournalMethods = { }; /* -** Open a journal file. +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and +** accessed using the underlying VFS. If nSpill is less than zero, then +** all content is always stored in main-memory. Finally, if nSpill is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nSpill bytes in size, +** or when sqlite3JournalCreate() is called. +*/ +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nSpill /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nSpill was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nSpill==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nSpill>0 ){ + p->nChunkSize = nSpill; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; + p->nSpill = nSpill; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** Open an in-memory journal file. */ SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - assert( EIGHT_BYTE_ALIGNMENT(p) ); - memset(p, 0, sqlite3MemJournalSize()); - p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nSpill parameter), and the underlying file has not yet been created, +** create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ + int rc = SQLITE_OK; + if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ + rc = memjrnlCreateFile((MemJournal*)p); + } + return rc; } +#endif /* -** Return true if the file-handle passed as an argument is -** an in-memory journal +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, +** or false otherwise. */ -SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods; +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods; } /* -** Return the number of bytes required to store a MemJournal file descriptor. +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. */ -SQLITE_PRIVATE int sqlite3MemJournalSize(void){ - return sizeof(MemJournal); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); } /************** End of memjournal.c ******************************************/ @@ -82523,9 +85029,8 @@ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; - if( pExpr==0 ) return WRC_Continue; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); @@ -82541,6 +85046,9 @@ SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ } return rc & WRC_Abort; } +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; +} /* ** Call sqlite3WalkExpr() for every expression in list p or until @@ -82592,6 +85100,11 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; @@ -82689,30 +85202,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -82746,19 +85235,11 @@ static void resolveAlias( db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -82950,7 +85431,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->jointype & JT_NATURAL ) continue; + if( pItem->fg.jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -82965,8 +85446,8 @@ static int lookupName( pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->jointype & JT_RIGHT)==0 ); - if( (pMatch->jointype & JT_LEFT)!=0 ){ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } pSchema = pExpr->pTab->pSchema; @@ -83004,7 +85485,6 @@ static int lookupName( } if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ /* IMP: R-51414-32910 */ - /* IMP: R-44911-55124 */ iCol = -1; } if( iCol<pTab->nCol ){ @@ -83031,10 +85511,15 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) - && VisibleRowid(pMatch->pTab) ){ + if( cnt==0 + && cntTab==1 + && pMatch + && (pNC->ncFlags & NC_IdxExpr)==0 + && sqlite3IsRowid(zCol) + && VisibleRowid(pMatch->pTab) + ){ cnt = 1; - pExpr->iColumn = -1; /* IMP: R-44911-55124 */ + pExpr->iColumn = -1; pExpr->affinity = SQLITE_AFF_INTEGER; } @@ -83051,9 +85536,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** or HAVING clauses, or as part of a larger expression in the ORDER BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. TO DO: Issue a warning + ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 @@ -83150,7 +85635,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -83191,36 +85676,25 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask /* Set of contexts for which prohibited */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -83305,6 +85779,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -83334,19 +85810,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); + notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ wrong_num_args = 1; } }else{ - is_agg = pDef->xFunc==0; + is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); if( n==2 ){ @@ -83382,9 +85858,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ ExprSetProperty(pExpr,EP_ConstFunc); } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); @@ -83430,8 +85915,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -83441,8 +85925,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } } @@ -83786,7 +86269,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -83859,7 +86341,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->isCorrelated flag if this is the case. */ + ** pItem->fg.isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -83868,8 +86350,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->isCorrelated==0 && nRef<=0 ); - pItem->isCorrelated = (nRef!=0); + assert( pItem->fg.isCorrelated==0 && nRef<=0 ); + pItem->fg.isCorrelated = (nRef!=0); } } @@ -83881,14 +86363,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; i<pEList->nExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -83921,6 +86396,16 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + /* Resolve names in table-valued-function arguments */ + for(i=0; i<p->pSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ @@ -84065,10 +86550,12 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); - memset(&w, 0, sizeof(w)); + w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; - w.pParse = pNC->pParse; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.eCode = 0; w.u.pNC = pNC; sqlite3WalkExpr(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 @@ -84084,6 +86571,23 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return ExprHasProperty(pExpr, EP_Error); } +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + if( pList ){ + for(i=0; i<pList->nExpr; i++){ + if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + } + } + return WRC_Continue; +} /* ** Resolve all names in all expressions of a SELECT and in all @@ -84127,15 +86631,14 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( SQLITE_PRIVATE void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - int i; /* Loop counter */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; @@ -84146,13 +86649,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ){ - for(i=0; i<pList->nExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ - return; - } - } - } + if( pList ) sqlite3ResolveExprListNames(&sNC, pList); } /************** End of resolve.c *********************************************/ @@ -84244,13 +86741,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ Token s; assert( zC!=0 ); - s.z = zC; - s.n = sqlite3Strlen30(s.z); + sqlite3TokenInit(&s, (char*)zC); return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -84261,7 +86757,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -84592,7 +87088,7 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote +** If dequote is false, no dequoting is performed. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. @@ -84613,6 +87109,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( int nExtra = 0; int iValue = 0; + assert( db!=0 ); if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ @@ -84620,8 +87117,9 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( assert( iValue>=0 ); } } - pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); + pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ + memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; if( pToken ){ @@ -84707,11 +87205,11 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( const Token *pToken /* Argument token */ ){ Expr *p; - if( op==TK_AND && pLeft && pRight && pParse->nErr==0 ){ + if( op==TK_AND && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ - p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -84858,7 +87356,10 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( x>pParse->nzVar ){ char **a; a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ) return; /* Error reported through db->mallocFailed */ + if( a==0 ){ + assert( db->mallocFailed ); /* Error reported through mallocFailed */ + return; + } pParse->azVar = a; memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); pParse->nzVar = x; @@ -85012,6 +87513,8 @@ static int dupedExprSize(Expr *p, int flags){ */ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ Expr *pNew = 0; /* Value to return */ + assert( flags==0 || flags==EXPRDUP_REDUCE ); + assert( db!=0 ); if( p ){ const int isReduced = (flags&EXPRDUP_REDUCE); u8 *zAlloc; @@ -85024,7 +87527,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ zAlloc = *pzBuffer; staticFlag = EP_Static; }else{ - zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, flags)); } pNew = (Expr *)zAlloc; @@ -85046,9 +87549,11 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ assert( ExprHasProperty(p, EP_Reduced)==0 ); memcpy(zAlloc, p, nNewSize); }else{ - int nSize = exprStructSize(p); + u32 nSize = (u32)exprStructSize(p); memcpy(zAlloc, p, nSize); - memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); + if( nSize<EXPR_FULLSIZE ){ + memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); + } } /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ @@ -85138,18 +87643,20 @@ static With *withDup(sqlite3 *db, With *p){ ** part of the in-memory representation of the database schema. */ SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ + assert( flags==0 || flags==EXPRDUP_REDUCE ); return exprDup(db, p, flags, 0); } SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; struct ExprList_item *pItem, *pOldItem; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; i<p->nExpr; i+=i){} - pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); + pNew->a = pItem = sqlite3DbMallocRawNN(db, i*sizeof(p->a[0]) ); if( pItem==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -85180,9 +87687,10 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SrcList *pNew; int i; int nByte; + assert( db!=0 ); if( p==0 ) return 0; nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRaw(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, nByte ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; i<p->nSrc; i++){ @@ -85193,16 +87701,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; + pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - pNewItem->isCorrelated = pOldItem->isCorrelated; - pNewItem->viaCoroutine = pOldItem->viaCoroutine; - pNewItem->isRecursive = pOldItem->isRecursive; - pNewItem->zIndexedBy = sqlite3DbStrDup(db, pOldItem->zIndexedBy); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->pIBIndex = pOldItem->pIBIndex; + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; @@ -85217,11 +87727,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ IdList *pNew; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); if( pNew->a==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -85239,8 +87750,9 @@ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ Select *pNew, *pPrior; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); if( pNew==0 ) return 0; pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); @@ -85286,12 +87798,14 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( Expr *pExpr /* Expression to be appended. Might be NULL */ ){ sqlite3 *db = pParse->db; + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); if( pList==0 ){ goto no_mem; } - pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); + pList->nExpr = 0; + pList->a = sqlite3DbMallocRawNN(db, sizeof(pList->a[0])); if( pList->a==0 ) goto no_mem; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; @@ -85318,6 +87832,20 @@ no_mem: } /* +** Set the sort order for the last element on the given ExprList. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ + if( p==0 ) return; + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); + assert( p->nExpr>0 ); + if( iSortOrder<0 ){ + assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); + return; + } + p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; +} + +/* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. ** @@ -85411,7 +87939,8 @@ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ if( pList ){ for(i=0; i<pList->nExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( ALWAYS(pExpr) ) m |= pExpr->flags; + assert( pExpr!=0 ); + m |= pExpr->flags; } } return m; @@ -85558,6 +88087,22 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ return exprIsConst(p, 4+isInit, 0); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Walk an expression tree. Return 1 if the expression contains a +** subquery of some kind. Return 0 if there are no subqueries. +*/ +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.eCode==0; +} +#endif + /* ** If the expression p codes a constant integer that is small enough ** to fit in a 32-bit integer, return 1 and put the value of the integer @@ -85680,23 +88225,22 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ } /* -** Return true if we are able to the IN operator optimization on a -** query of the form -** -** x IN (SELECT ...) -** -** Where the SELECT... clause is as specified by the parameter to this -** routine. -** -** The Select object passed in has already been preprocessed and no -** errors have been found. +** pX is the RHS of an IN operator. If pX is a SELECT statement +** that can be simplified to a direct table access, then return +** a pointer to the SELECT statement. If pX is not a SELECT statement, +** or if the SELECT statement needs to be manifested into a transient +** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY -static int isCandidateForInOpt(Select *p){ +static Select *isCandidateForInOpt(Expr *pX){ + Select *p; SrcList *pSrc; ExprList *pEList; + Expr *pRes; Table *pTab; - if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ + p = pX->x.pSelect; if( p->pPrior ) return 0; /* Not a compound SELECT */ if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); @@ -85712,13 +88256,15 @@ static int isCandidateForInOpt(Select *p){ if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ pTab = pSrc->a[0].pTab; - if( NEVER(pTab==0) ) return 0; + assert( pTab!=0 ); assert( pTab->pSelect==0 ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ - return 1; + pRes = pEList->a[0].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; /* Result is a column */ + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + return p; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -85738,13 +88284,13 @@ SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int j1; + int addr1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } @@ -85850,15 +88396,13 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int ** satisfy the query. This is preferable to generating a new ** ephemeral table. */ - p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); - if( pParse->nErr==0 && isCandidateForInOpt(p) ){ + if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table <table>. */ Expr *pExpr; /* Expression <column> */ i16 iCol; /* Index of column <column> */ i16 iDb; /* Database idx for pTab */ - assert( p ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ @@ -86016,9 +88560,10 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ - char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", - pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d", + jmpIfDynamic>=0?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", + pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } @@ -86288,7 +88833,7 @@ static void sqlite3ExprCodeIN( } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeGoto(v, destIfFalse); } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); @@ -86306,7 +88851,7 @@ static void sqlite3ExprCodeIN( int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeGoto(v, destIfNull); sqlite3VdbeJumpHere(v, addr1); } } @@ -86344,7 +88889,7 @@ static void sqlite3ExprCodeIN( ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1; + int addr1; /* First check to see if the LHS is contained in the RHS. If so, ** then the answer is TRUE the presence of NULLs in the RHS does @@ -86352,12 +88897,12 @@ static void sqlite3ExprCodeIN( ** answer is NULL if the RHS contains NULLs and the answer is ** FALSE if the RHS is NULL-free. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeGoto(v, destIfFalse); + sqlite3VdbeJumpHere(v, addr1); } } } @@ -86575,6 +89120,28 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -86602,9 +89169,12 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. An effort -** is made to store the column value in register iReg, but this is -** not guaranteed. The location of the column value is returned. +** table pTab and store the column value in a register. +** +** An effort is made to store the column value in register iReg. This +** is not garanteeed for GetColumn() - the result can be stored in +** any register. But the result is guaranteed to land in register iReg +** for GetColumnToReg(). ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -86615,7 +89185,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ int iReg, /* Store results here */ - u8 p5 /* P5 value for OP_Column */ + u8 p5 /* P5 value for OP_Column + FLAGS */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -86637,6 +89207,17 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( } return iReg; } +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg /* Store results here */ +){ + int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); + if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); +} + /* ** Clear all column cache entries. @@ -86760,8 +89341,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iColumn + pParse->ckBase; break; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, @@ -86782,7 +89364,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); break; } case TK_NULL: { @@ -86821,10 +89403,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iTable; break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; - } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -86966,7 +89544,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ExprList *pFarg; /* List of function arguments */ int nFarg; /* Number of function arguments */ FuncDef *pDef; /* The function definition object */ - int nId; /* Length of the function name in bytes */ const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ @@ -86982,10 +89559,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 || pDef->xFunc==0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); + pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); + if( pDef==0 || pDef->xFinalize!=0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); break; } @@ -87055,7 +89631,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ @@ -87150,6 +89726,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ReleaseTempReg(pParse, r4); break; } + case TK_SPAN: case TK_COLLATE: case TK_UPLUS: { inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); @@ -87279,7 +89856,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3VdbeGoto(v, endLabel); sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } @@ -87410,7 +89987,7 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); }else{ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); if( inReg!=target && pParse->pVdbe ){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } @@ -87418,6 +89995,18 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ } /* +** Make a transient copy of expression pExpr and then code it using +** sqlite3ExprCode(). This routine works just like sqlite3ExprCode() +** except that the input expression is guaranteed to be unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ + sqlite3 *db = pParse->db; + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ) sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprDelete(db, pExpr); +} + +/* ** Generate code that will evaluate expression pExpr and store the ** results in register target. The results are guaranteed to appear ** in register target. If the expression is constant, then this routine @@ -87466,16 +90055,22 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ ** ** The SQLITE_ECEL_FACTOR argument allows constant arguments to be ** factored out into initialization code. +** +** The SQLITE_ECEL_REF flag means that expressions in the list with +** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored +** in registers at srcReg, and so the value can be copied from there. */ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, n; + int i, j, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -87483,13 +90078,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; i<n; i++, pItem++){ Expr *pExpr = pItem->pExpr; - if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; - Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg @@ -87609,6 +90205,13 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: @@ -87624,23 +90227,12 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -87666,14 +90258,14 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { if( exprAlwaysTrue(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ /* No-op */ }else{ @@ -87765,6 +90357,13 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: @@ -87780,23 +90379,12 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -87829,7 +90417,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int #endif default: { if( exprAlwaysFalse(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ /* no-op */ }else{ @@ -87904,8 +90492,10 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ } return 2; } - if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } @@ -88189,7 +90779,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pItem->iMem = ++pParse->nMem; assert( !ExprHasProperty(pExpr, EP_IntValue) ); pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), + pExpr->u.zToken, pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); if( pExpr->flags & EP_Distinct ){ pItem->iDistinct = pParse->nTab++; @@ -88318,6 +90908,29 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeReg<iLast + && pParse->iRangeReg>=iFirst + ){ + return 0; + } + for(i=0; i<pParse->nTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ + /************** End of expr.c ************************************************/ /************** Begin file alter.c *******************************************/ /* @@ -88551,7 +91164,7 @@ static void renameTriggerFunc( ** Register built-in functions used to help implement ALTER TABLE */ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static SQLITE_WSD FuncDef aAlterTableFuncs[] = { + static FuncDef aAlterTableFuncs[] = { FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), #ifndef SQLITE_OMIT_TRIGGER FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), @@ -88560,13 +91173,7 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); - - for(i=0; i<ArraySize(aAlterTableFuncs); i++){ - sqlite3FuncDefInsert(pHash, &aFunc[i]); - } + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } /* @@ -88813,7 +91420,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( #ifndef SQLITE_OMIT_VIRTUALTABLE if( pVTab ){ int i = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); sqlite3MayAbort(pParse); } @@ -88910,33 +91517,6 @@ exit_rename_table: db->flags = savedDbFlags; } - -/* -** Generate code to make sure the file format number is at least minFormat. -** The generated code will increase the file format number if necessary. -*/ -SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ - Vdbe *v; - v = sqlite3GetVdbe(pParse); - /* The VDBE should have been allocated before this routine is called. - ** If that allocation failed, we would have quit before reaching this - ** point */ - if( ALWAYS(v) ){ - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3GetTempReg(pParse); - int j1; - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); - j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); - sqlite3VdbeJumpHere(v, j1); - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); - } -} - /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -88955,9 +91535,11 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; + assert( v!=0 ); pNew = pParse->pNewTable; assert( pNew ); @@ -88981,7 +91563,8 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** literal NULL, then set pDflt to 0. This simplifies checking ** for an SQL NULL default below. */ - if( pDflt && pDflt->op==TK_NULL ){ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ pDflt = 0; } @@ -89017,7 +91600,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc!=SQLITE_OK ){ - db->mallocFailed = 1; + assert( db->mallocFailed == 1 ); return; } if( !pVal ){ @@ -89047,11 +91630,16 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then set the file + /* If the default value of the new column is NULL, then the file ** format to 2. If the default value of the new column is not NULL, - ** the file format becomes 3. + ** the file format be 3. Back when this feature was first added + ** in 2006, we went to the trouble to upgrade the file format to the + ** minimum support values. But 10-years on, we can assume that all + ** extent versions of SQLite support file-format 4, so we always and + ** unconditionally upgrade to 4. */ - sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, + SQLITE_MAX_FILE_FORMAT); /* Reload the schema of the modified table. */ reloadTableSchema(pParse, pTab, pTab->zName); @@ -89125,7 +91713,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ - db->mallocFailed = 1; + assert( db->mallocFailed ); goto exit_begin_add_column; } memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); @@ -89133,9 +91721,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Column *pCol = &pNew->aCol[i]; pCol->zName = sqlite3DbStrDup(db, pCol->zName); pCol->zColl = 0; - pCol->zType = 0; pCol->pDflt = 0; - pCol->zDflt = 0; } pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; @@ -89470,7 +92056,7 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); - p->u.aRowid = sqlite3DbMallocRaw(db, n); + p->u.aRowid = sqlite3DbMallocRawNN(db, n); if( p->u.aRowid ){ p->nRowid = n; memcpy(p->u.aRowid, pData, n); @@ -89635,12 +92221,10 @@ static const FuncDef statInitFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statInit, /* xFunc */ - 0, /* xStep */ + statInit, /* xSFunc */ 0, /* xFinalize */ "stat_init", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #ifdef SQLITE_ENABLE_STAT4 @@ -89936,12 +92520,10 @@ static const FuncDef statPushFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statPush, /* xFunc */ - 0, /* xStep */ + statPush, /* xSFunc */ 0, /* xFinalize */ "stat_push", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ @@ -90083,12 +92665,10 @@ static const FuncDef statGetFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statGet, /* xFunc */ - 0, /* xStep */ + statGet, /* xSFunc */ 0, /* xFinalize */ "stat_get", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ @@ -90100,8 +92680,8 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut); - sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, + (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } @@ -90147,7 +92727,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strlike("sqlite_%", pTab->zName, 0)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -90171,7 +92751,7 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ @@ -90193,7 +92773,7 @@ static void analyzeOneTable( } /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + sqlite3VdbeLoadString(v, regIdxname, zIdxName); VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* @@ -90255,8 +92835,8 @@ static void analyzeOneTable( #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4); - sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, + (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: @@ -90275,7 +92855,7 @@ static void analyzeOneTable( if( nColTest>0 ){ int endDistinctTest = sqlite3VdbeMakeLabel(v); int *aGotoChng; /* Array of jump instruction addresses */ - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; /* @@ -90307,7 +92887,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + sqlite3VdbeGoto(v, endDistinctTest); /* @@ -90343,6 +92923,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; j<pPk->nKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && k<pTab->nCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -90351,8 +92932,8 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp); - sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, + (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); @@ -90392,12 +92973,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; i<nCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif @@ -90684,7 +93263,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** the old data with the new instead of allocating a new array. */ if( pIndex->aiRowEst==0 ){ pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); - if( pIndex->aiRowEst==0 ) pInfo->db->mallocFailed = 1; + if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); } aiRowEst = pIndex->aiRowEst; #endif @@ -90831,10 +93410,10 @@ static int loadStatTbl( Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -90874,7 +93453,7 @@ static int loadStatTbl( pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += nIdxCol; @@ -90890,7 +93469,7 @@ static int loadStatTbl( zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -90928,7 +93507,7 @@ static int loadStatTbl( pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); if( pSample->p==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); pIdx->nSample++; @@ -90945,7 +93524,7 @@ static int loadStatTbl( static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ rc = loadStatTbl(db, 0, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", @@ -91017,7 +93596,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ zSql = sqlite3MPrintf(db, "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); sqlite3DbFree(db, zSql); @@ -91027,10 +93606,9 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ - int lookasideEnabled = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bEnabled = lookasideEnabled; + db->lookaside.bDisable--; } for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); @@ -91040,7 +93618,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ #endif if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } @@ -91161,7 +93739,7 @@ static void attachFunc( ** hash tables. */ if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ @@ -91179,7 +93757,7 @@ static void attachFunc( flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; @@ -91196,7 +93774,7 @@ static void attachFunc( Pager *pPager; aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); if( !aNew->pSchema ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ zErrDyn = sqlite3MPrintf(db, "attached databases must use the same text encoding as main database"); @@ -91208,14 +93786,15 @@ static void attachFunc( sqlite3BtreeSecureDelete(aNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); #ifndef SQLITE_OMIT_PAGER_PRAGMAS - sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); + sqlite3BtreeSetPagerFlags(aNew->pBt, + PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); #endif sqlite3BtreeLeave(aNew->pBt); } - aNew->safety_level = 3; + aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; aNew->zName = sqlite3DbStrDup(db, zName); if( rc==SQLITE_OK && aNew->zName==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } @@ -91281,7 +93860,7 @@ static void attachFunc( sqlite3ResetAllSchemasOfConnection(db); db->nDb = iDb; if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zErrDyn); zErrDyn = sqlite3MPrintf(db, "out of memory"); }else if( zErrDyn==0 ){ @@ -91411,11 +93990,11 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, + (char *)pFunc, P4_FUNCDEF); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); - + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). @@ -91440,12 +94019,10 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - detachFunc, /* xFunc */ - 0, /* xStep */ + detachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -91461,12 +94038,10 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - attachFunc, /* xFunc */ - 0, /* xStep */ + attachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -91926,15 +94501,6 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ */ /* #include "sqliteInt.h" */ -/* -** This routine is called when a new SQL statement is beginning to -** be parsed. Initialize the pParse structure as needed. -*/ -SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ - pParse->explain = (u8)explainFlag; - pParse->nVar = 0; -} - #ifndef SQLITE_OMIT_SHARED_CACHE /* ** The TableLock structure is only used by the sqlite3TableLock() and @@ -91989,7 +94555,7 @@ SQLITE_PRIVATE void sqlite3TableLock( p->zName = zName; }else{ pToplevel->nTableLock = 0; - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); } } @@ -92094,6 +94660,8 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; i<pParse->nVtabLock; i++){ @@ -92123,7 +94691,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); + sqlite3VdbeGoto(v, 1); } } @@ -92137,15 +94705,19 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; - pParse->colNamesSet = 0; }else{ pParse->rc = SQLITE_ERROR; } + + /* We are done with this Parse object. There is no need to de-initialize it */ +#if 0 + pParse->colNamesSet = 0; pParse->nTab = 0; pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; DbMaskZero(pParse->cookieMask); +#endif } /* @@ -92258,6 +94830,17 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } + } +#endif if( zDbase ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ @@ -92265,12 +94848,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( } pParse->checkSchema = 1; } -#if SQLITE_USER_AUTHENICATION - else if( pParse->db->auth.authLevel<UAUTH_User ){ - sqlite3ErrorMsg(pParse, "user not authenticated"); - p = 0; - } -#endif + return p; } @@ -92336,8 +94914,9 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3DeleteIndexSamples(db, p); #endif sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); - if( p->isResized ) sqlite3DbFree(db, p->azColl); + if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3_free(p->aiRowEst); #endif @@ -92397,7 +94976,6 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ } j++; } - memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); db->nDb = j; if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); @@ -92462,7 +95040,7 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -92470,8 +95048,6 @@ static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ for(i=0; i<pTable->nCol; i++, pCol++){ sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); - sqlite3DbFree(db, pCol->zDflt); - sqlite3DbFree(db, pCol->zType); sqlite3DbFree(db, pCol->zColl); } sqlite3DbFree(db, pTable->aCol); @@ -92529,13 +95105,11 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqliteDeleteColumnNames(db, pTable); + sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -92611,12 +95185,8 @@ SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ int i = -1; /* Database number */ if( zName ){ Db *pDb; - int n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && - 0==sqlite3StrICmp(pDb->zName, zName) ){ - break; - } + if( 0==sqlite3StrICmp(pDb->zName, zName) ) break; } } return i; @@ -92662,7 +95232,8 @@ SQLITE_PRIVATE int sqlite3TwoPartName( int iDb; /* Database holding the object */ sqlite3 *db = pParse->db; - if( ALWAYS(pName2!=0) && pName2->n>0 ){ + assert( pName2!=0 ); + if( pName2->n>0 ){ if( db->init.busy ) { sqlite3ErrorMsg(pParse, "corrupt database"); return -1; @@ -92751,62 +95322,46 @@ SQLITE_PRIVATE void sqlite3StartTable( int iDb; /* Database number to create the table in */ Token *pName; /* Unqualified name of the table to create */ - /* The table or view name to create is passed to this routine via tokens - ** pName1 and pName2. If the table name was fully qualified, for example: - ** - ** CREATE TABLE xxx.yyy (...); - ** - ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if - ** the table name is not fully qualified, i.e.: - ** - ** CREATE TABLE yyy(...); - ** - ** Then pName1 is set to "yyy" and pName2 is "". - ** - ** The call below sets the pName pointer to point at the token (pName1 or - ** pName2) that stores the unqualified table name. The variable iDb is - ** set to the index of the database that the table or view is to be - ** created in. - */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless - ** the database name is "temp" anyway. */ - sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); - return; + if( db->init.busy && db->init.newTnum==1 ){ + /* Special case: Parsing the sqlite_master or sqlite_temp_master schema */ + iDb = db->init.iDb; + zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); + pName = pName1; + }else{ + /* The common case */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + zName = sqlite3NameFromToken(db, pName); } - if( !OMIT_TEMPDB && isTemp ) iDb = 1; - pParse->sNameToken = *pName; - zName = sqlite3NameFromToken(db, pName); if( zName==0 ) return; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; #ifndef SQLITE_OMIT_AUTHORIZATION - assert( (isTemp & 1)==isTemp ); + assert( isTemp==0 || isTemp==1 ); + assert( isView==0 || isView==1 ); { - int code; + static const u8 aCode[] = { + SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_VIEW, + SQLITE_CREATE_TEMP_VIEW + }; char *zDb = db->aDb[iDb].zName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } - if( isView ){ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_VIEW; - }else{ - code = SQLITE_CREATE_VIEW; - } - }else{ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_TABLE; - }else{ - code = SQLITE_CREATE_TABLE; - } - } - if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ + if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], + zName, 0, zDb) ){ goto begin_table_error; } } @@ -92842,8 +95397,8 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ - db->mallocFailed = 1; - pParse->rc = SQLITE_NOMEM; + assert( db->mallocFailed ); + pParse->rc = SQLITE_NOMEM_BKPT; pParse->nErr++; goto begin_table_error; } @@ -92875,9 +95430,11 @@ SQLITE_PRIVATE void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; + int addr1; int fileFormat; int reg1, reg2, reg3; + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -92894,14 +95451,12 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; - sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); - sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, fileFormat); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); + sqlite3VdbeJumpHere(v, addr1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -92922,7 +95477,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -92937,18 +95492,19 @@ begin_table_error: return; } -/* -** This macro is used to compare two strings in a case-insensitive manner. -** It is slightly faster than calling sqlite3StrICmp() directly, but -** produces larger code. -** -** WARNING: This macro is not compatible with the strcmp() family. It -** returns true if the two strings are equal, otherwise false. +/* Set properties of a table column based on the (magical) +** name of the column. */ -#define STRICMP(x, y) (\ -sqlite3UpperToLower[*(unsigned char *)(x)]== \ -sqlite3UpperToLower[*(unsigned char *)(y)] \ -&& sqlite3StrICmp((x)+1,(y)+1)==0 ) +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ + if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ + pCol->colFlags |= COLFLAG_HIDDEN; + }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ + pTab->tabFlags |= TF_OOOHidden; + } +} +#endif + /* ** Add a new column to the table currently being constructed. @@ -92958,10 +95514,11 @@ sqlite3UpperToLower[*(unsigned char *)(y)] \ ** first to get things going. Then this routine is called for each ** column. */ -SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ Table *p; int i; char *z; + char *zType; Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; @@ -92971,10 +95528,13 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ return; } #endif - z = sqlite3NameFromToken(db, pName); + z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; + memcpy(z, pName->z, pName->n); + z[pName->n] = 0; + sqlite3Dequote(z); for(i=0; i<p->nCol; i++){ - if( STRICMP(z, p->aCol[i].zName) ){ + if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); sqlite3DbFree(db, z); return; @@ -92992,14 +95552,22 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zName = z; + sqlite3ColumnPropertiesFromName(p, pCol); - /* If there is no type specified, columns have the default affinity - ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will - ** be called next to set pCol->affinity correctly. - */ - pCol->affinity = SQLITE_AFF_BLOB; - pCol->szEst = 1; + if( pType->n==0 ){ + /* If there is no type specified, columns have the default affinity + ** 'BLOB'. */ + pCol->affinity = SQLITE_AFF_BLOB; + pCol->szEst = 1; + }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, pType->z, pType->n); + zType[pType->n] = 0; + pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->colFlags |= COLFLAG_HASTYPE; + } p->nCol++; + pParse->constraintName.n = 0; } /* @@ -93045,7 +95613,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; - if( zIn==0 ) return aff; + assert( zIn!=0 ); while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; @@ -93103,28 +95671,6 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ } /* -** This routine is called by the parser while in the middle of -** parsing a CREATE TABLE statement. The pFirst token is the first -** token in the sequence of tokens that describe the type of the -** column currently under construction. pLast is the last token -** in the sequence. Use this information to construct a string -** that contains the typename of the column and store that string -** in zType. -*/ -SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ - Table *p; - Column *pCol; - - p = pParse->pNewTable; - if( p==0 || NEVER(p->nCol<1) ) return; - pCol = &p->aCol[p->nCol-1]; - assert( pCol->zType==0 || CORRUPT_DB ); - sqlite3DbFree(pParse->db, pCol->zType); - pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); -} - -/* ** The expression is the default value for the most recently added column ** of the table currently under construction. ** @@ -93149,17 +95695,46 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ ** tokens that point to volatile memory. The 'span' of the expression ** is required by pragma table_info. */ + Expr x; sqlite3ExprDelete(db, pCol->pDflt); - pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); - sqlite3DbFree(db, pCol->zDflt); - pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); + memset(&x, 0, sizeof(x)); + x.op = TK_SPAN; + x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + x.pLeft = pSpan->pExpr; + x.flags = EP_Skip; + pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + sqlite3DbFree(db, x.u.zToken); } } sqlite3ExprDelete(db, pSpan->pExpr); } /* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + +/* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the ** most recently added column of the table is the primary key. @@ -93185,7 +95760,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - char *zType = 0; + Column *pCol = 0; int iCol = -1, i; int nTerm; if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; @@ -93197,24 +95772,31 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; nTerm = 1; }else{ nTerm = pList->nExpr; for(i=0; i<nTerm; i++){ - for(iCol=0; iCol<pTab->nCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; - break; + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iCol<pTab->nCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; + break; + } } } } } if( nTerm==1 - && zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC + && pCol + && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 + && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -93353,13 +95935,11 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** 1 chance in 2^32. So we're safe enough. */ SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ - int r1 = sqlite3GetTempReg(pParse); sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); - sqlite3ReleaseTempReg(pParse, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + db->aDb[iDb].pSchema->schema_cookie+1); } /* @@ -93441,7 +96021,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ n += 35 + 6*p->nCol; zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } sqlite3_snprintf(n, zStmt, "CREATE TABLE "); @@ -93494,9 +96074,9 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ assert( pIdx->isResized==0 ); nByte = (sizeof(char*) + sizeof(i16) + 1)*N; zExtra = sqlite3DbMallocZero(db, nByte); - if( zExtra==0 ) return SQLITE_NOMEM; + if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); - pIdx->azColl = (char**)zExtra; + pIdx->azColl = (const char**)zExtra; zExtra += sizeof(char*)*N; memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); pIdx->aiColumn = (i16*)zExtra; @@ -93581,7 +96161,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; + sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -93589,10 +96169,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token ipkToken; + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); @@ -93608,7 +96189,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** a database schema). */ if( v ){ assert( db->init.busy==0 ); - sqlite3VdbeGetOp(v, pPk->tnum)->opcode = OP_Goto; + sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); } /* @@ -93633,7 +96214,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** do not enforce this for imposter tables.) */ if( !db->init.imposterTable ){ for(i=0; i<nPk; i++){ - pTab->aCol[pPk->aiColumn[i]].notNull = 1; + pTab->aCol[pPk->aiColumn[i]].notNull = OE_Abort; } pPk->uniqNotNull = 1; } @@ -93675,7 +96256,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) ){ assert( j<pPk->nColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = "BINARY"; + pPk->azColl[j] = sqlite3StrBINARY; j++; } } @@ -93718,9 +96299,10 @@ SQLITE_PRIVATE void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + if( pEnd==0 && pSelect==0 ){ return; } + assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -93731,9 +96313,13 @@ SQLITE_PRIVATE void sqlite3EndTable( ** So do not write to the disk again. Extract the root page number ** for the table from the db->init.newTnum field. (The page number ** should have been put there by the sqliteOpenCb routine.) + ** + ** If the root page number is 1, that means this is the sqlite_master + ** table itself. So mark it read-only. */ if( db->init.busy ){ p->tnum = db->init.newTnum; + if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } /* Special processing for WITHOUT ROWID Tables */ @@ -93834,7 +96420,7 @@ SQLITE_PRIVATE void sqlite3EndTable( sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); if( pParse->nErr ) return; pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); @@ -93851,7 +96437,7 @@ SQLITE_PRIVATE void sqlite3EndTable( sqlite3TableAffinity(v, p, 0); sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrInsLoop); + sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); sqlite3VdbeAddOp1(v, OP_Close, 1); } @@ -93918,7 +96504,7 @@ SQLITE_PRIVATE void sqlite3EndTable( pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } pParse->pNewTable = 0; @@ -93948,6 +96534,7 @@ SQLITE_PRIVATE void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -93963,22 +96550,15 @@ SQLITE_PRIVATE void sqlite3CreateView( if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); - return; + goto create_view_fail; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -93986,30 +96566,31 @@ SQLITE_PRIVATE void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ - return; - } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + assert( sEnd.z[0]!=0 ); + if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + while( sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -94074,11 +96655,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable->pSelect ); pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; @@ -94087,25 +96667,43 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #else pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bEnabled = enableLookaside; pParse->nTab = n; - if( pSelTab ){ + if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); + } + }else if( pSelTab ){ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ assert( pTable->aCol==0 ); pTable->nCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; } + if( pSelTab ) sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); + db->lookaside.bDisable--; } else { nErr++; } + pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -94122,7 +96720,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); + sqlite3DeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -94184,6 +96782,7 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iT static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); + assert( iTable>1 ); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM @@ -94569,7 +97168,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ - db->mallocFailed = 1; + sqlite3OomFault(db); goto fk_end; } if( pNextTo ){ @@ -94677,7 +97276,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ assert( pKey!=0 || db->mallocFailed || pParse->nErr ); if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); @@ -94722,7 +97321,7 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->azColl = (const char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; @@ -94774,7 +97373,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ @@ -94929,11 +97527,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token prevCol; + sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->nCol-1].zName); - pList->a[0].sortOrder = (u8)sortOrder; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); } /* Figure out how many bytes of space are required to store explicitly @@ -94941,8 +97543,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ for(i=0; i<pList->nExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -94983,35 +97585,54 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; i<pList->nExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; - int requestedSortOrder; - char *zColl; /* Collation sequence name */ - - for(j=0, pTabCol=pTab->aCol; j<pTab->nCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ + const char *zColl; /* Collation sequence name */ + + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - assert( j<=0x7fff ); - pIndex->aiColumn[i] = (i16)j; - if( pListItem->pExpr ){ + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; - assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -95019,21 +97640,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = sqlite3StrBINARY; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; j<pPk->nKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -95045,12 +97671,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = -1; - pIndex->azColl[i] = "BINARY"; + pIndex->aiColumn[i] = XN_ROWID; + pIndex->azColl[i] = sqlite3StrBINARY; } sqlite3DefaultRowEst(pIndex); if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + /* If this index contains every column of its table, then mark + ** it as a covering index */ + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ + pIndex->isCovering = 1; + for(j=0; j<pTab->nCol; j++){ + if( j==pTab->iPKey ) continue; + if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + pIndex->isCovering = 0; + break; + } + } + if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or @@ -95084,10 +97724,11 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(k=0; k<pIdx->nKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; + if( sqlite3StrICmp(z1, z2) ) break; } if( k==pIdx->nKeyCol ){ if( pIdx->onError!=pIndex->onError ){ @@ -95115,6 +97756,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); @@ -95122,7 +97764,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ - db->mallocFailed = 1; + sqlite3OomFault(db); goto exit_create_index; } db->flags |= SQLITE_InternChanges; @@ -95144,7 +97786,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -95551,10 +98193,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( ){ struct SrcList_item *pItem; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; + pList->nSrc = 0; } pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); if( db->mallocFailed ){ @@ -95604,7 +98248,8 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndexedBy); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -95677,18 +98322,38 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndexedBy==0 ); + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; + pItem->fg.notIndexed = 1; }else{ - pItem->zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); } } } /* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + +/* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator ** expects the join operator to be on the right operand. This routine @@ -95707,14 +98372,14 @@ SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; + p->a[i].fg.jointype = p->a[i-1].fg.jointype; } - p->a[0].jointype = 0; + p->a[0].fg.jointype = 0; } } /* -** Begin a transaction +** Generate VDBE code for a BEGIN statement. */ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3 *db; @@ -95724,7 +98389,6 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ assert( pParse!=0 ); db = pParse->db; assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ return; } @@ -95736,11 +98400,11 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); + sqlite3VdbeAddOp0(v, OP_AutoCommit); } /* -** Commit a transaction +** Generate VDBE code for a COMMIT statement. */ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ Vdbe *v; @@ -95752,12 +98416,12 @@ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ } v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); + sqlite3VdbeAddOp1(v, OP_AutoCommit, 1); } } /* -** Rollback a transaction +** Generate VDBE code for a ROLLBACK statement. */ SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ Vdbe *v; @@ -95819,7 +98483,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 1; } } @@ -95936,7 +98600,7 @@ SQLITE_PRIVATE void sqlite3HaltConstraint( sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); - if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); + sqlite3VdbeChangeP5(v, p5Errmsg); } /* @@ -95953,12 +98617,16 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( Table *pTab = pIdx->pTable; sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); - for(j=0; j<pIdx->nKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, "index '%q'", pIdx->zName); + }else{ + for(j=0; j<pIdx->nKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, @@ -96143,9 +98811,8 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ if( pKey ){ assert( sqlite3KeyInfoIsWriteable(pKey) ); for(i=0; i<nCol; i++){ - char *zColl = pIdx->azColl[i]; - assert( zColl!=0 ); - pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + const char *zColl = pIdx->azColl[i]; + pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : sqlite3LocateCollSeq(pParse, zColl); pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } @@ -96191,10 +98858,9 @@ SQLITE_PRIVATE With *sqlite3WithAdd( }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } - assert( zName!=0 || pNew==0 ); - assert( db->mallocFailed==0 || pNew==0 ); + assert( (pNew!=0 && zName!=0) || db->mallocFailed ); - if( pNew==0 ){ + if( db->mallocFailed ){ sqlite3ExprListDelete(db, pArglist); sqlite3SelectDelete(db, pQuery); sqlite3DbFree(db, zName); @@ -96203,7 +98869,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zErr = 0; + pNew->a[pNew->nCte].zCteErr = 0; pNew->nCte++; } @@ -96408,7 +99074,7 @@ static CollSeq *findCollSeqEntry( */ assert( pDel==0 || pDel==pColl ); if( pDel!=0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); pColl = 0; } @@ -96474,8 +99140,8 @@ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( ** 5: UTF16 byte order conversion required - argument count matches exactly ** 6: Perfect match: encoding and argument count match exactly. ** -** If nArg==(-2) then any function with a non-null xStep or xFunc is -** a perfect match and any function with both xStep and xFunc NULL is +** If nArg==(-2) then any function with a non-null xSFunc is +** a perfect match and any function with xSFunc NULL is ** a non-match. */ #define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ @@ -96487,7 +99153,7 @@ static int matchQuality( int match; /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg && p->nArg>=0 ) return 0; @@ -96515,14 +99181,12 @@ static int matchQuality( ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ static FuncDef *functionSearch( - FuncDefHash *pHash, /* Hash table to search */ int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Number of bytes in zFunc */ + const char *zFunc /* Name of function */ ){ FuncDef *p; - for(p=pHash->a[h]; p; p=p->pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + if( sqlite3StrICmp(p->zName, zFunc)==0 ){ return p; } } @@ -96532,23 +99196,26 @@ static FuncDef *functionSearch( /* ** Insert a new FuncDef into a FuncDefHash hash table. */ -SQLITE_PRIVATE void sqlite3FuncDefInsert( - FuncDefHash *pHash, /* The hash table into which to insert */ - FuncDef *pDef /* The function definition to insert */ +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( + FuncDef *aDef, /* List of global functions to be inserted */ + int nDef /* Length of the apDef[] list */ ){ - FuncDef *pOther; - int nName = sqlite3Strlen30(pDef->zName); - u8 c1 = (u8)pDef->zName[0]; - int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); - pOther = functionSearch(pHash, h, pDef->zName, nName); - if( pOther ){ - assert( pOther!=pDef && pOther->pNext!=pDef ); - pDef->pNext = pOther->pNext; - pOther->pNext = pDef; - }else{ - pDef->pNext = 0; - pDef->pHash = pHash->a[h]; - pHash->a[h] = pDef; + int i; + for(i=0; i<nDef; i++){ + FuncDef *pOther; + const char *zName = aDef[i].zName; + int nName = sqlite3Strlen30(zName); + int h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; + pOther = functionSearch(h, zName); + if( pOther ){ + assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); + aDef[i].pNext = pOther->pNext; + pOther->pNext = &aDef[i]; + }else{ + aDef[i].pNext = 0; + aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; + sqlite3BuiltinFunctions.a[h] = &aDef[i]; + } } } @@ -96565,7 +99232,7 @@ SQLITE_PRIVATE void sqlite3FuncDefInsert( ** no matching function previously existed. ** ** If nArg is -2, then the first valid function found is returned. A -** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** function is valid if xSFunc is non-zero. The nArg==(-2) ** case is used to see if zName is a valid function name for some number ** of arguments. If nArg is -2, then createFlag must be 0. ** @@ -96575,8 +99242,7 @@ SQLITE_PRIVATE void sqlite3FuncDefInsert( */ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( sqlite3 *db, /* An open database */ - const char *zName, /* Name of the function. Not null-terminated */ - int nName, /* Number of characters in the name */ + const char *zName, /* Name of the function. zero-terminated */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ u8 createFlag /* Create new entry if true and does not otherwise exist */ @@ -96585,14 +99251,15 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ + int nName; /* Length of the name */ assert( nArg>=(-2) ); assert( nArg>=(-1) || createFlag==0 ); - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + nName = sqlite3Strlen30(zName); /* First search for a match amongst the application-defined functions. */ - p = functionSearch(&db->aFunc, h, zName, nName); + p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -96615,9 +99282,9 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); bestScore = 0; - p = functionSearch(pHash, h, zName, nName); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; + p = functionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -96634,15 +99301,22 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( */ if( createFlag && bestScore<FUNC_PERFECT_MATCH && (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ - pBest->zName = (char *)&pBest[1]; + FuncDef *pOther; + pBest->zName = (const char*)&pBest[1]; pBest->nArg = (u16)nArg; pBest->funcFlags = enc; - memcpy(pBest->zName, zName, nName); - pBest->zName[nName] = 0; - sqlite3FuncDefInsert(&db->aFunc, pBest); + memcpy((char*)&pBest[1], zName, nName+1); + pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); + if( pOther==pBest ){ + sqlite3DbFree(db, pBest); + sqlite3OomFault(db); + return 0; + }else{ + pBest->pNext = pOther; + } } - if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ + if( pBest && (pBest->xSFunc || createFlag) ){ return pBest; } return 0; @@ -96696,7 +99370,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if ( 0==p->file_format ){ sqlite3HashInit(&p->tblHash); sqlite3HashInit(&p->idxHash); @@ -96817,7 +99491,8 @@ SQLITE_PRIVATE void sqlite3MaterializeView( assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, + SF_IncludeHidden, 0, 0); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -96946,7 +99621,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -96958,12 +99633,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ - int addrDelete = 0; /* Jump directly to the delete logic */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ + int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -96987,9 +99662,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 +# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -97070,8 +99747,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -97086,6 +99765,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -97106,13 +99787,18 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -97122,6 +99808,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; i<nPk; i++){ + assert( pPk->aiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -97132,13 +99819,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ - aToOpen = sqlite3DbMallocRaw(db, nIdx+2); + aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); if( aToOpen==0 ){ sqlite3WhereEnd(pWInfo); goto delete_from_cleanup; @@ -97148,27 +99834,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass!=ONEPASS_OFF ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -97177,21 +99863,24 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } testcase( IsVirtual(pTab) ); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, - &iDataCur, &iIdxCur); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, + iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ + if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - assert( !IsVirtual(pTab) ); - if( aToOpen[iDataCur-iTabCur] ){ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); @@ -97213,23 +99902,32 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } }else #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } @@ -97296,6 +99994,25 @@ delete_from_cleanup: ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -97307,7 +100024,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -97324,7 +100042,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -97384,11 +100102,20 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** a view (in which case the only effect of the DELETE statement is to ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + u8 p5 = 0; + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } + if( eMode!=ONEPASS_OFF ){ + sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); + } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; + sqlite3VdbeChangeP5(v, p5); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -97431,7 +100158,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -97447,11 +100175,12 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -97500,14 +100229,13 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); @@ -97519,9 +100247,14 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; j<nCol; j++){ - if( pPrior && pPrior->aiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -97792,7 +100525,8 @@ static void printfFunc( x.nUsed = 0; x.apArg = argv+1; sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); - sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); + str.printfFlags = SQLITE_PRINTF_SQLFUNC; + sqlite3XPrintf(&str, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, SQLITE_DYNAMIC); @@ -98120,10 +100854,10 @@ static void total_changes( ** A structure defining how to do GLOB-style comparisons. */ struct compareInfo { - u8 matchAll; - u8 matchOne; - u8 matchSet; - u8 noCase; + u8 matchAll; /* "*" or "%" */ + u8 matchOne; /* "?" or "_" */ + u8 matchSet; /* "[" or 0 */ + u8 noCase; /* true to ignore case differences */ }; /* @@ -98186,22 +100920,14 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - u32 esc /* The escape character */ + u32 matchOther /* The escape char (LIKE) or '[' (GLOB) */ ){ u32 c, c2; /* Next pattern and input string chars */ u32 matchOne = pInfo->matchOne; /* "?" or "_" */ u32 matchAll = pInfo->matchAll; /* "*" or "%" */ - u32 matchOther; /* "[" or the escape character */ u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ const u8 *zEscaped = 0; /* One past the last escaped input char */ - /* The GLOB operator does not have an ESCAPE clause. And LIKE does not - ** have the matchSet operator. So we either have to look for one or - ** the other, never both. Hence the single variable matchOther is used - ** to store the one we have to look for. - */ - matchOther = esc ? esc : pInfo->matchSet; - while( (c = Utf8Read(zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there @@ -98215,7 +100941,7 @@ static int patternCompare( if( c==0 ){ return 1; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ - if( esc ){ + if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; }else{ @@ -98223,7 +100949,7 @@ static int patternCompare( ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ while( *zString - && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + && patternCompare(&zPattern[-1],zString,pInfo,matchOther)==0 ){ SQLITE_SKIP_UTF8(zString); } return *zString!=0; @@ -98249,18 +100975,18 @@ static int patternCompare( } while( (c2 = *(zString++))!=0 ){ if( c2!=c && c2!=cx ) continue; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; } }else{ while( (c2 = Utf8Read(zString))!=0 ){ if( c2!=c ) continue; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; } } return 0; } if( c==matchOther ){ - if( esc ){ + if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; zEscaped = zPattern; @@ -98313,7 +101039,14 @@ static int patternCompare( ** The sqlite3_strglob() interface. */ SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[')==0; +} + +/* +** The sqlite3_strlike() interface. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc)==0; } /* @@ -98344,10 +101077,22 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - u32 escape = 0; + u32 escape; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); + struct compareInfo *pInfo = sqlite3_user_data(context); +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + || sqlite3_value_type(argv[1])==SQLITE_BLOB + ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, 0); + return; + } +#endif zB = sqlite3_value_text(argv[0]); zA = sqlite3_value_text(argv[1]); @@ -98375,13 +101120,13 @@ static void likeFunc( return; } escape = sqlite3Utf8Read(&zEsc); + }else{ + escape = pInfo->matchSet; } if( zA && zB ){ - struct compareInfo *pInfo = sqlite3_user_data(context); #ifdef SQLITE_TEST sqlite3_like_count++; #endif - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); } } @@ -99152,11 +101897,11 @@ static void groupConcatFinalize(sqlite3_context *context){ ** of the built-in functions above are part of the global function set. ** This routine only deals with those that are not global. */ -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } @@ -99165,8 +101910,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ */ static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); if( ALWAYS(pDef) ){ pDef->funcFlags |= flagVal; } @@ -99214,9 +101958,7 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas return 0; } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, - sqlite3Strlen30(pExpr->u.zToken), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -99240,7 +101982,7 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas ** ** After this routine runs */ -SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ /* ** The following array holds FuncDef structures for all of the functions ** defined in this file. @@ -99248,8 +101990,27 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** The array cannot be constant since changes are made to the ** FuncDef.pHash elements at start-time. The elements of this array ** are read-only after initialization is complete. + ** + ** For peak efficiency, put the most frequently used function last. */ - static SQLITE_WSD FuncDef aBuiltinFunc[] = { + static FuncDef aBuiltinFunc[] = { +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -99267,8 +102028,6 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), - FUNCTION(substr, 2, 0, 0, substrFunc ), - FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), @@ -99279,40 +102038,22 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ #endif FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), - FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), - FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), -#if SQLITE_USER_AUTHENTICATION - FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), -#endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), VFUNCTION(changes, 0, 0, 0, changes ), VFUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), - #ifdef SQLITE_SOUNDEX - FUNCTION(soundex, 1, 0, 0, soundexFunc ), - #endif - #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), - #endif + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), @@ -99330,22 +102071,34 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), #endif + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), }; - - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); - - for(i=0; i<ArraySize(aBuiltinFunc); i++){ - sqlite3FuncDefInsert(pHash, &aFunc[i]); - } - sqlite3RegisterDateTimeFunctions(); #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(); #endif #if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) sqlite3AnalyzeFunctions(); #endif + sqlite3RegisterDateTimeFunctions(); + sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); + +#if 0 /* Enable to print out how the built-in functions are hashed */ + { + int i; + FuncDef *p; + for(i=0; i<SQLITE_FUNC_HASH_SZ; i++){ + printf("FUNC-HASH %02d:", i); + for(p=sqlite3BuiltinFunctions.a[i]; p; p=p->u.pHash){ + int n = sqlite3Strlen30(p->zName); + int h = p->zName[0] + n; + printf(" %s(%d)", p->zName, h); + } + printf("\n"); + } + } +#endif } /************** End of func.c ************************************************/ @@ -99571,7 +102324,7 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( } }else if( paiCol ){ assert( nCol>1 ); - aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); + aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); if( !aiCol ) return 1; *paiCol = aiCol; } @@ -99601,16 +102354,16 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( int i, j; for(i=0; i<nCol; i++){ i16 iCol = pIdx->aiColumn[i]; /* Index of column in parent tbl */ - char *zDfltColl; /* Def. collation for column */ + const char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; - if( !zDfltColl ){ - zDfltColl = "BINARY"; - } + if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; zIdxCol = pParent->aCol[iCol].zName; @@ -99726,7 +102479,7 @@ static void fkLookupParent( sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); sqlite3ReleaseTempReg(pParse, regTemp); @@ -99756,6 +102509,7 @@ static void fkLookupParent( for(i=0; i<nCol; i++){ int iChild = aiCol[i]+1+regData; int iParent = pIdx->aiColumn[i]+1+regData; + assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ @@ -99764,11 +102518,11 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); @@ -99964,6 +102718,7 @@ static void fkScanChildren( assert( pIdx!=0 ); for(i=0; i<pPk->nKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); @@ -100283,6 +103038,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any @@ -100414,7 +103170,10 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; i<pIdx->nKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; i<pIdx->nKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } } } @@ -100508,10 +103267,13 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ + return 0; + } + pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ - u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ char const *zFrom; /* Name of child table */ int nFrom; /* Length in bytes of zFrom */ Index *pIdx = 0; /* Parent key index for this FK */ @@ -100537,11 +103299,10 @@ static Trigger *fkActionTrigger( iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKey<pTab->nCol) ); - tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; - tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; - - tToCol.n = sqlite3Strlen30(tToCol.z); - tFromCol.n = sqlite3Strlen30(tFromCol.z); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); + sqlite3TokenInit(&tToCol, + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); /* Create the expression "OLD.zToCol = zFromCol". It is important ** that the "OLD.zToCol" term is on the LHS of the = operator, so @@ -100621,8 +103382,7 @@ static Trigger *fkActionTrigger( } /* Disable lookaside memory allocation */ - enableLookaside = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ @@ -100644,7 +103404,7 @@ static Trigger *fkActionTrigger( } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bEnabled = enableLookaside; + db->lookaside.bDisable--; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); @@ -100825,7 +103585,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -100837,15 +103597,25 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } for(n=0; n<pIdx->nColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + char aff; + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; + } } pIdx->zColAff[n] = 0; } @@ -100880,7 +103650,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ sqlite3 *db = sqlite3VdbeDb(v); zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } @@ -100976,7 +103746,7 @@ static int autoIncBegin( pInfo = pToplevel->pAinc; while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } if( pInfo==0 ){ - pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); + pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); if( pInfo==0 ) return 0; pInfo->pNext = pToplevel->pAinc; pToplevel->pAinc = pInfo; @@ -101000,43 +103770,55 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* Database only autoinc table */ int memId; /* Register holding max rowid */ - int addr; /* A VDBE address */ Vdbe *v = pParse->pVdbe; /* VDBE under construction */ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + assert( sqlite3IsToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoInc[] = { + /* 0 */ {OP_Null, 0, 0, 0}, + /* 1 */ {OP_Rewind, 0, 9, 0}, + /* 2 */ {OP_Column, 0, 0, 0}, + /* 3 */ {OP_Ne, 0, 7, 0}, + /* 4 */ {OP_Rowid, 0, 0, 0}, + /* 5 */ {OP_Column, 0, 1, 0}, + /* 6 */ {OP_Goto, 0, 9, 0}, + /* 7 */ {OP_Next, 0, 2, 0}, + /* 8 */ {OP_Integer, 0, 0, 0}, + /* 9 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; pDb = &db->aDb[p->iDb]; memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); - sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); - sqlite3VdbeAddOp0(v, OP_Close); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); + if( aOp==0 ) break; + aOp[0].p2 = memId; + aOp[0].p3 = memId+1; + aOp[2].p3 = memId; + aOp[3].p1 = memId-1; + aOp[3].p3 = memId; + aOp[3].p5 = SQLITE_JUMPIFNULL; + aOp[4].p2 = memId+1; + aOp[5].p3 = memId; + aOp[8].p2 = memId; } } /* ** Update the maximum rowid for an autoincrement calculation. ** -** This routine should be called when the top of the stack holds a +** This routine should be called when the regRowid register holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the -** memory cell is updated. The stack is unchanged. +** memory cell is updated. */ static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ @@ -101051,31 +103833,44 @@ static void autoIncStep(Parse *pParse, int memId, int regRowid){ ** table (either directly or through triggers) needs to call this ** routine just before the "exit" code. */ -SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ +static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ AutoincInfo *p; Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoIncEnd[] = { + /* 0 */ {OP_NotNull, 0, 2, 0}, + /* 1 */ {OP_NewRowid, 0, 0, 0}, + /* 2 */ {OP_MakeRecord, 0, 2, 0}, + /* 3 */ {OP_Insert, 0, 0, 0}, + /* 4 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; Db *pDb = &db->aDb[p->iDb]; - int j1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); - sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3VdbeAddOp0(v, OP_Close); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); + if( aOp==0 ) break; + aOp[0].p1 = memId+1; + aOp[1].p2 = memId+1; + aOp[2].p1 = memId-1; + aOp[2].p3 = iRec; + aOp[3].p2 = iRec; + aOp[3].p3 = memId+1; + aOp[3].p5 = OPFLAG_APPEND; sqlite3ReleaseTempReg(pParse, iRec); } } +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ + if( pParse->pAinc ) autoIncrementEnd(pParse); +} #else /* ** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines @@ -101406,7 +104201,7 @@ SQLITE_PRIVATE void sqlite3Insert( rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -101447,7 +104242,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeGoto(v, addrL); sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); @@ -101461,11 +104256,13 @@ SQLITE_PRIVATE void sqlite3Insert( sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; i<nColumn; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ goto insert_cleanup; } + }else{ + nColumn = 0; } } @@ -101480,10 +104277,8 @@ SQLITE_PRIVATE void sqlite3Insert( /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ - if( IsVirtual(pTab) ){ - for(i=0; i<pTab->nCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); - } + for(i=0; i<pTab->nCol; i++){ + nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); } if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ sqlite3ErrorMsg(pParse, @@ -101506,9 +104301,9 @@ SQLITE_PRIVATE void sqlite3Insert( /* If this is not a view, open the table and and all indices */ if( !isView ){ int nIdx; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ goto insert_cleanup; } @@ -101558,7 +104353,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int j1; + int addr1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); @@ -101566,9 +104361,9 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } @@ -101579,15 +104374,14 @@ SQLITE_PRIVATE void sqlite3Insert( /* Create the new column data */ - for(i=0; i<pTab->nCol; i++){ - if( pColumn==0 ){ - j = i; - }else{ + for(i=j=0; i<pTab->nCol; i++){ + if( pColumn ){ for(j=0; j<pColumn->nId; j++){ if( pColumn->a[j].idx==i ) break; } } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ + if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) + || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); }else if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); @@ -101595,6 +104389,7 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); } + if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; } /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, @@ -101642,14 +104437,14 @@ SQLITE_PRIVATE void sqlite3Insert( ** to generate a unique primary key value. */ if( !appendFlag ){ - int j1; + int addr1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } @@ -101678,7 +104473,6 @@ SQLITE_PRIVATE void sqlite3Insert( } if( pColumn==0 ){ if( IsHiddenColumn(&pTab->aCol[i]) ){ - assert( IsVirtual(pTab) ); j = -1; nHidden++; }else{ @@ -101717,7 +104511,7 @@ SQLITE_PRIVATE void sqlite3Insert( { int isReplace; /* Set to true if constraints may cause a replace */ sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0 ); sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, @@ -101746,7 +104540,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeGoto(v, addrCont); sqlite3VdbeJumpHere(v, addrInsTop); } @@ -101800,6 +104594,59 @@ insert_cleanup: #endif /* +** Meanings of bits in of pWalker->eCode for checkConstraintUnchanged() +*/ +#define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ +#define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ + +/* This is the Walker callback from checkConstraintUnchanged(). Set +** bit 0x01 of pWalker->eCode if +** pWalker->eCode to 0 if this expression node references any of the +** columns that are being modifed by an UPDATE statement. +*/ +static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + assert( pExpr->iColumn>=0 || pExpr->iColumn==-1 ); + if( pExpr->iColumn>=0 ){ + if( pWalker->u.aiCol[pExpr->iColumn]>=0 ){ + pWalker->eCode |= CKCNSTRNT_COLUMN; + } + }else{ + pWalker->eCode |= CKCNSTRNT_ROWID; + } + } + return WRC_Continue; +} + +/* +** pExpr is a CHECK constraint on a row that is being UPDATE-ed. The +** only columns that are modified by the UPDATE are those for which +** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. +** +** Return true if CHECK constraint pExpr does not use any of the +** changing columns (or the rowid if it is changing). In other words, +** return true if this CHECK constraint can be skipped when validating +** the new row in the UPDATE statement. +*/ +static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 0; + w.xExprCallback = checkConstraintExprNode; + w.u.aiCol = aiChng; + sqlite3WalkExpr(&w, pExpr); + if( !chngRowid ){ + testcase( (w.eCode & CKCNSTRNT_ROWID)!=0 ); + w.eCode &= ~CKCNSTRNT_ROWID; + } + testcase( w.eCode==0 ); + testcase( w.eCode==CKCNSTRNT_COLUMN ); + testcase( w.eCode==CKCNSTRNT_ROWID ); + testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); + return !w.eCode; +} + +/* ** Generate code to do constraint checks prior to an INSERT or an UPDATE ** on table pTab. ** @@ -101893,7 +104740,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ u8 overrideError, /* Override onError to this if not OE_Default */ int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ - int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ + int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ + int *aiChng /* column i is unchanged if aiChng[i]<0 */ ){ Vdbe *v; /* VDBE under constrution */ Index *pIdx; /* Pointer to one of the indices */ @@ -101903,7 +104751,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Address of jump instruction */ + int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -101939,10 +104787,14 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( */ for(i=0; i<nCol; i++){ if( i==pTab->iPKey ){ + continue; /* ROWID is never NULL */ + } + if( aiChng && aiChng[i]<0 ){ + /* Don't bother checking for NOT NULL on columns that do not change */ continue; } onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; + if( onError==OE_None ) continue; /* This column is allowed to be NULL */ if( overrideError!=OE_Default ){ onError = overrideError; }else if( onError==OE_Default ){ @@ -101974,9 +104826,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); break; } } @@ -101990,10 +104843,13 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pParse->ckBase = regNewData+1; onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; i<pCheck->nExpr; i++){ - int allOk = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + int allOk; + Expr *pExpr = pCheck->a[i].pExpr; + if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; + allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pExpr, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; @@ -102091,17 +104947,20 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + regNewData, 1, 0, OE_Replace, + ONEPASS_SINGLE, -1); + }else{ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } } seenReplace = 1; break; } case OE_Ignore: { /*assert( seenReplace==0 );*/ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } } @@ -102149,15 +105008,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; i<pIdx->nColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==XN_EXPR ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==XN_ROWID || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -102207,6 +105073,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; i<pPk->nKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, @@ -102228,6 +105095,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; i<pPk->nKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; + assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; @@ -102254,7 +105122,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } default: { @@ -102265,7 +105133,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); seenReplace = 1; break; } @@ -102275,7 +105144,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeGoto(v, ipkTop+1); sqlite3VdbeJumpHere(v, ipkBottom); } @@ -102328,7 +105197,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; } - if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); + sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; @@ -102380,6 +105249,7 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ int op, /* OP_OpenRead or OP_OpenWrite */ + u8 p5, /* P5 value for OP_Open* opcodes (except on WITHOUT ROWID) */ int iBase, /* Use this for the table cursor, if there is one */ u8 *aToOpen, /* If not NULL: boolean for each table and index */ int *piDataCur, /* Write the database source cursor number here */ @@ -102392,6 +105262,7 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Vdbe *v; assert( op==OP_OpenRead || op==OP_OpenWrite ); + assert( op==OP_OpenWrite || p5==0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur uninitialized so that valgrind @@ -102413,14 +105284,16 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ - *piDataCur = iIdxCur; - } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); } + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + }else{ + sqlite3VdbeChangeP5(v, p5); + } } if( iBase>pParse->nTab ) pParse->nTab = iBase; return i; @@ -102440,20 +105313,6 @@ SQLITE_API int sqlite3_xferopt_count; #ifndef SQLITE_OMIT_XFER_OPT /* -** Check to collation names to see if they are compatible. -*/ -static int xferCompatibleCollation(const char *z1, const char *z2){ - if( z1==0 ){ - return z2==0; - } - if( z2==0 ){ - return 0; - } - return sqlite3StrICmp(z1, z2)==0; -} - - -/* ** Check to see if index pSrc is compatible as a source of data ** for index pDest in an insert transfer optimization. The rules ** for a compatible index: @@ -102478,10 +105337,17 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } - if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ + if( sqlite3_stricmp(pSrc->azColl[i],pDest->azColl[i])!=0 ){ return 0; /* Different collating sequences */ } } @@ -102596,7 +105462,7 @@ static int xferOptimization( return 0; /* The result set must have exactly one column */ } assert( pEList->a[0].pExpr ); - if( pEList->a[0].pExpr->op!=TK_ALL ){ + if( pEList->a[0].pExpr->op!=TK_ASTERISK ){ return 0; /* The result set must be the special operator "*" */ } @@ -102632,21 +105498,32 @@ static int xferOptimization( for(i=0; i<pDest->nCol; i++){ Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + if( (db->flags & SQLITE_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + ){ + return 0; /* Neither table may have __hidden__ columns */ + } +#endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ + if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ return 0; /* Collating sequence must be the same on all columns */ } if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 - && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) - || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) - ){ - return 0; /* Default values must be the same for all columns */ + if( i>0 ){ + assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); + assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); + if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) + || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, + pSrcCol->pDflt->u.zToken)!=0) + ){ + return 0; /* Default values must be the same for all columns */ + } } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ @@ -102721,7 +105598,7 @@ static int xferOptimization( ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ @@ -102741,9 +105618,9 @@ static int xferOptimization( assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, + pDest->zName, 0); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); - sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -102782,9 +105659,10 @@ static int xferOptimization( ** a VACUUM command. In that case keys may not be written in strictly ** sorted order. */ for(i=0; i<pSrcIdx->nColumn; i++){ - char *zColl = pSrcIdx->azColl[i]; - assert( zColl!=0 ); - if( sqlite3_stricmp("BINARY", zColl) ) break; + const char *zColl = pSrcIdx->azColl[i]; + assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 + || sqlite3StrBINARY==zColl ); + if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ idxInsFlags = OPFLAG_USESEEKRESULT; @@ -102909,7 +105787,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( for(i=0; i<nCol; i++){ azVals[i] = (char *)sqlite3_column_text(pStmt, i); if( !azVals[i] && sqlite3_column_type(pStmt, i)!=SQLITE_NULL ){ - db->mallocFailed = 1; + sqlite3OomFault(db); goto exec_out; } } @@ -102950,7 +105828,7 @@ exec_out: if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ @@ -103258,6 +106136,15 @@ struct sqlite3_api_routines { void (*value_free)(sqlite3_value*); int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); + /* Version 3.10.0 and later */ + int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); + int (*strlike)(const char*,const char*,unsigned int); + int (*db_cacheflush)(sqlite3*); + /* Version 3.12.0 and later */ + int (*system_errno)(sqlite3*); }; /* @@ -103271,7 +106158,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -103398,6 +106285,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -103493,9 +106381,18 @@ struct sqlite3_api_routines { #define sqlite3_value_free sqlite3_api->value_free #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 -#endif /* SQLITE_CORE */ - -#ifndef SQLITE_CORE +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +/* Version 3.10.0 and later */ +#define sqlite3_status64 sqlite3_api->status64 +#define sqlite3_strlike sqlite3_api->strlike +#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush +/* Version 3.12.0 and later */ +#define sqlite3_system_errno sqlite3_api->system_errno +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; @@ -103904,7 +106801,16 @@ static const sqlite3_api_routines sqlite3Apis = { (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, sqlite3_value_free, sqlite3_result_zeroblob64, - sqlite3_bind_zeroblob64 + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype, + /* Version 3.10.0 and later */ + sqlite3_status64, + sqlite3_strlike, + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno }; /* @@ -103968,7 +106874,7 @@ static int sqlite3LoadExtension( #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){ char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]); - if( zAltFile==0 ) return SQLITE_NOMEM; + if( zAltFile==0 ) return SQLITE_NOMEM_BKPT; handle = sqlite3OsDlOpen(pVfs, zAltFile); sqlite3_free(zAltFile); } @@ -104004,7 +106910,7 @@ static int sqlite3LoadExtension( zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} @@ -104047,7 +106953,7 @@ static int sqlite3LoadExtension( /* Append the new shared library handle to the db->aExtension array. */ aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); if( aHandle==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( db->nExtension>0 ){ memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); @@ -104169,7 +107075,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ void (**aNew)(void); aNew = sqlite3_realloc64(wsdAutoext.aExt, nByte); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ wsdAutoext.aExt = aNew; wsdAutoext.aExt[wsdAutoext.nExt] = xInit; @@ -104315,43 +107221,44 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #define PragTyp_FLAG 2 #define PragTyp_BUSY_TIMEOUT 3 #define PragTyp_CACHE_SIZE 4 -#define PragTyp_CASE_SENSITIVE_LIKE 5 -#define PragTyp_COLLATION_LIST 6 -#define PragTyp_COMPILE_OPTIONS 7 -#define PragTyp_DATA_STORE_DIRECTORY 8 -#define PragTyp_DATABASE_LIST 9 -#define PragTyp_DEFAULT_CACHE_SIZE 10 -#define PragTyp_ENCODING 11 -#define PragTyp_FOREIGN_KEY_CHECK 12 -#define PragTyp_FOREIGN_KEY_LIST 13 -#define PragTyp_INCREMENTAL_VACUUM 14 -#define PragTyp_INDEX_INFO 15 -#define PragTyp_INDEX_LIST 16 -#define PragTyp_INTEGRITY_CHECK 17 -#define PragTyp_JOURNAL_MODE 18 -#define PragTyp_JOURNAL_SIZE_LIMIT 19 -#define PragTyp_LOCK_PROXY_FILE 20 -#define PragTyp_LOCKING_MODE 21 -#define PragTyp_PAGE_COUNT 22 -#define PragTyp_MMAP_SIZE 23 -#define PragTyp_PAGE_SIZE 24 -#define PragTyp_SECURE_DELETE 25 -#define PragTyp_SHRINK_MEMORY 26 -#define PragTyp_SOFT_HEAP_LIMIT 27 -#define PragTyp_STATS 28 -#define PragTyp_SYNCHRONOUS 29 -#define PragTyp_TABLE_INFO 30 -#define PragTyp_TEMP_STORE 31 -#define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_THREADS 33 -#define PragTyp_WAL_AUTOCHECKPOINT 34 -#define PragTyp_WAL_CHECKPOINT 35 -#define PragTyp_ACTIVATE_EXTENSIONS 36 -#define PragTyp_HEXKEY 37 -#define PragTyp_KEY 38 -#define PragTyp_REKEY 39 -#define PragTyp_LOCK_STATUS 40 -#define PragTyp_PARSER_TRACE 41 +#define PragTyp_CACHE_SPILL 5 +#define PragTyp_CASE_SENSITIVE_LIKE 6 +#define PragTyp_COLLATION_LIST 7 +#define PragTyp_COMPILE_OPTIONS 8 +#define PragTyp_DATA_STORE_DIRECTORY 9 +#define PragTyp_DATABASE_LIST 10 +#define PragTyp_DEFAULT_CACHE_SIZE 11 +#define PragTyp_ENCODING 12 +#define PragTyp_FOREIGN_KEY_CHECK 13 +#define PragTyp_FOREIGN_KEY_LIST 14 +#define PragTyp_INCREMENTAL_VACUUM 15 +#define PragTyp_INDEX_INFO 16 +#define PragTyp_INDEX_LIST 17 +#define PragTyp_INTEGRITY_CHECK 18 +#define PragTyp_JOURNAL_MODE 19 +#define PragTyp_JOURNAL_SIZE_LIMIT 20 +#define PragTyp_LOCK_PROXY_FILE 21 +#define PragTyp_LOCKING_MODE 22 +#define PragTyp_PAGE_COUNT 23 +#define PragTyp_MMAP_SIZE 24 +#define PragTyp_PAGE_SIZE 25 +#define PragTyp_SECURE_DELETE 26 +#define PragTyp_SHRINK_MEMORY 27 +#define PragTyp_SOFT_HEAP_LIMIT 28 +#define PragTyp_STATS 29 +#define PragTyp_SYNCHRONOUS 30 +#define PragTyp_TABLE_INFO 31 +#define PragTyp_TEMP_STORE 32 +#define PragTyp_TEMP_STORE_DIRECTORY 33 +#define PragTyp_THREADS 34 +#define PragTyp_WAL_AUTOCHECKPOINT 35 +#define PragTyp_WAL_CHECKPOINT 36 +#define PragTyp_ACTIVATE_EXTENSIONS 37 +#define PragTyp_HEXKEY 38 +#define PragTyp_KEY 39 +#define PragTyp_REKEY 40 +#define PragTyp_LOCK_STATUS 41 +#define PragTyp_PARSER_TRACE 42 #define PragFlag_NeedSchema 0x01 #define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { @@ -104393,14 +107300,14 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ 0, + /* ePragFlag: */ PragFlag_NeedSchema, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_FLAG, + /* ePragTyp: */ PragTyp_CACHE_SPILL, /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CacheSpill }, + /* iArg: */ 0 }, #endif { /* zName: */ "case_sensitive_like", /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, @@ -104614,7 +107521,7 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, /* ePragFlag: */ 0, @@ -104774,8 +107681,8 @@ static const struct sPragmaNames { /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. The FULL option is disallowed +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** unrecognized string argument. The FULL and EXTRA option is disallowed ** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that @@ -104784,18 +107691,21 @@ static const struct sPragmaNames { ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ - /* 123456789 123456789 */ - static const char zText[] = "onoffalseyestruefull"; - static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; - static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; - static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; + /* 123456789 123456789 123 */ + static const char zText[] = "onoffalseyestruextrafull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; + /* on no off false yes true extra full */ int i, n; if( sqlite3Isdigit(*z) ){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; i<ArraySize(iLength)-omitFull; i++){ - if( iLength[i]==n && sqlite3StrNICmp(&zText[iOffset[i]],z,n)==0 ){ + for(i=0; i<ArraySize(iLength); i++){ + if( iLength[i]==n && sqlite3StrNICmp(&zText[iOffset[i]],z,n)==0 + && (!omitFull || iValue[i]<=1) + ){ return iValue[i]; } } @@ -104902,19 +107812,45 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ #endif /* SQLITE_PAGER_PRAGMAS */ /* +** Set the names of the first N columns to the values in azCol[] +*/ +static void setAllColumnNames( + Vdbe *v, /* The query under construction */ + int N, /* Number of columns */ + const char **azCol /* Names of columns */ +){ + int i; + sqlite3VdbeSetNumCols(v, N); + for(i=0; i<N; i++){ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, azCol[i], SQLITE_STATIC); + } +} +static void setOneColumnName(Vdbe *v, const char *z){ + setAllColumnNames(v, 1, &z); +} + +/* ** Generate code to return a single integer value. */ -static void returnSingleInt(Parse *pParse, const char *zLabel, i64 value){ - Vdbe *v = sqlite3GetVdbe(pParse); - int nMem = ++pParse->nMem; - i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); - if( pI64 ){ - memcpy(pI64, &value, sizeof(value)); +static void returnSingleInt(Vdbe *v, const char *zLabel, i64 value){ + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, 1, 0, (const u8*)&value, P4_INT64); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + +/* +** Generate code to return a single text value. +*/ +static void returnSingleText( + Vdbe *v, /* Prepared statement under construction */ + const char *zLabel, /* Name of the result column */ + const char *zValue /* Value to be returned */ +){ + if( zValue ){ + sqlite3VdbeLoadString(v, 1, (const char*)zValue); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - sqlite3VdbeAddOp4(v, OP_Int64, 0, nMem, 0, (char*)pI64, P4_INT64); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); } @@ -104995,7 +107931,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ ** ** Pragmas are of this form: ** -** PRAGMA [database.]id [= value] +** PRAGMA [schema.]id [= value] ** ** The identifier might also be a string. The value is a string, and ** identifier, or a number. If minusFlag is true, then the value is @@ -105007,8 +107943,8 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ */ SQLITE_PRIVATE void sqlite3Pragma( Parse *pParse, - Token *pId1, /* First part of [database.]id field */ - Token *pId2, /* Second part of [database.]id field, or NULL */ + Token *pId1, /* First part of [schema.]id field */ + Token *pId2, /* Second part of [schema.]id field, or NULL */ Token *pValue, /* Token for <value>, or NULL */ int minusFlag /* True if a '-' sign preceded <value> */ ){ @@ -105029,7 +107965,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; - /* Interpret the [database.] part of the pragma statement. iDb is the + /* Interpret the [schema.] part of the pragma statement. iDb is the ** index of the database this pragma is being applied to in db.aDb[]. */ iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); if( iDb<0 ) return; @@ -105078,14 +108014,8 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - if( aFcntl[0] ){ - int nMem = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, nMem, 0, aFcntl[0], 0); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); - sqlite3_free(aFcntl[0]); - } + returnSingleText(v, "result", aFcntl[0]); + sqlite3_free(aFcntl[0]); goto pragma_out; } if( rc!=SQLITE_NOTFOUND ){ @@ -105124,8 +108054,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* - ** PRAGMA [database.]default_cache_size - ** PRAGMA [database.]default_cache_size=N + ** PRAGMA [schema.]default_cache_size + ** PRAGMA [schema.]default_cache_size=N ** ** The first form reports the current persistent setting for the ** page cache size. The value returned is the maximum number of @@ -105152,21 +108082,21 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; - int addr; + VdbeOp *aOp; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + setOneColumnName(v, "cache_size"); pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; }else{ int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, size, 1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, size); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); @@ -105177,8 +108107,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* - ** PRAGMA [database.]page_size - ** PRAGMA [database.]page_size=N + ** PRAGMA [schema.]page_size + ** PRAGMA [schema.]page_size=N ** ** The first form reports the current setting for the ** database page size in bytes. The second form sets the @@ -105190,22 +108120,22 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(pParse, "page_size", size); + returnSingleInt(v, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } break; } /* - ** PRAGMA [database.]secure_delete - ** PRAGMA [database.]secure_delete=ON/OFF + ** PRAGMA [schema.]secure_delete + ** PRAGMA [schema.]secure_delete=ON/OFF ** ** The first form reports the current setting for the ** secure_delete flag. The second form changes the secure_delete @@ -105225,13 +108155,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); + returnSingleInt(v, "secure_delete", b); break; } /* - ** PRAGMA [database.]max_page_count - ** PRAGMA [database.]max_page_count=N + ** PRAGMA [schema.]max_page_count + ** PRAGMA [schema.]max_page_count=N ** ** The first form reports the current setting for the ** maximum number of pages in the database file. The @@ -105242,7 +108172,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** change. The only purpose is to provide an easy way to test ** the sqlite3AbsInt32() function. ** - ** PRAGMA [database.]page_count + ** PRAGMA [schema.]page_count ** ** Return the number of pages in the specified database. */ @@ -105263,8 +108193,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [database.]locking_mode - ** PRAGMA [database.]locking_mode = (normal|exclusive) + ** PRAGMA [schema.]locking_mode + ** PRAGMA [schema.]locking_mode = (normal|exclusive) */ case PragTyp_LOCKING_MODE: { const char *zRet = "normal"; @@ -105304,25 +108234,20 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "locking_mode", zRet); break; } /* - ** PRAGMA [database.]journal_mode - ** PRAGMA [database.]journal_mode = + ** PRAGMA [schema.]journal_mode + ** PRAGMA [schema.]journal_mode = ** (delete|persist|off|truncate|memory|wal|off) */ case PragTyp_JOURNAL_MODE: { int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); - + setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -105355,8 +108280,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [database.]journal_size_limit - ** PRAGMA [database.]journal_size_limit=N + ** PRAGMA [schema.]journal_size_limit + ** PRAGMA [schema.]journal_size_limit=N ** ** Get or set the size limit on rollback journal files. */ @@ -105368,15 +108293,15 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(pParse, "journal_size_limit", iLimit); + returnSingleInt(v, "journal_size_limit", iLimit); break; } #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ /* - ** PRAGMA [database.]auto_vacuum - ** PRAGMA [database.]auto_vacuum=N + ** PRAGMA [schema.]auto_vacuum + ** PRAGMA [schema.]auto_vacuum=N ** ** Get or set the value of the database 'auto-vacuum' parameter. ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL @@ -105386,7 +108311,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -105409,16 +108334,18 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, { OP_If, 1, 0, 0}, /* 2 */ { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_Integer, 0, 1, 0}, /* 4 */ - { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 0}, /* 4 */ }; - int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); - sqlite3VdbeChangeP1(v, iAddr, iDb); - sqlite3VdbeChangeP1(v, iAddr+1, iDb); - sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); - sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); - sqlite3VdbeChangeP1(v, iAddr+5, iDb); + VdbeOp *aOp; + int iAddr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[2].p2 = iAddr+4; + aOp[4].p1 = iDb; + aOp[4].p3 = eAuto - 1; sqlite3VdbeUsesBtree(v, iDb); } } @@ -105427,7 +108354,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* - ** PRAGMA [database.]incremental_vacuum(N) + ** PRAGMA [schema.]incremental_vacuum(N) ** ** Do N steps of incremental vacuuming on a database. */ @@ -105450,8 +108377,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_PAGER_PRAGMAS /* - ** PRAGMA [database.]cache_size - ** PRAGMA [database.]cache_size=N + ** PRAGMA [schema.]cache_size + ** PRAGMA [schema.]cache_size=N ** ** The first form reports the current local setting for the ** page cache size. The second form sets the local @@ -105463,19 +108390,60 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); - if( sqlite3ReadSchema(pParse) ) goto pragma_out; } break; } /* - ** PRAGMA [database.]mmap_size(N) + ** PRAGMA [schema.]cache_spill + ** PRAGMA cache_spill=BOOLEAN + ** PRAGMA [schema.]cache_spill=N + ** + ** The first form reports the current local setting for the + ** page cache spill size. The second form turns cache spill on + ** or off. When turnning cache spill on, the size is set to the + ** current cache_size. The third form sets a spill size that + ** may be different form the cache size. + ** If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + ** + ** If the number of cache_spill pages is less then the number of + ** cache_size pages, no spilling occurs until the page count exceeds + ** the number of cache_size pages. + ** + ** The cache_spill=BOOLEAN setting applies to all attached schemas, + ** not just the schema specified. + */ + case PragTyp_CACHE_SPILL: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, "cache_spill", + (db->flags & SQLITE_CacheSpill)==0 ? 0 : + sqlite3BtreeSetSpillSize(pDb->pBt,0)); + }else{ + int size = 1; + if( sqlite3GetInt32(zRight, &size) ){ + sqlite3BtreeSetSpillSize(pDb->pBt, size); + } + if( sqlite3GetBoolean(zRight, size!=0) ){ + db->flags |= SQLITE_CacheSpill; + }else{ + db->flags &= ~SQLITE_CacheSpill; + } + setAllPagerFlags(db); + } + break; + } + + /* + ** PRAGMA [schema.]mmap_size(N) ** ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the @@ -105510,7 +108478,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(pParse, "mmap_size", sz); + returnSingleInt(v, "mmap_size", sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -105531,7 +108499,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(pParse, "temp_store", db->temp_store); + returnSingleInt(v, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -105550,13 +108518,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_temp_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -105600,13 +108562,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_data_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "data_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "data_store_directory", sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -105631,8 +108587,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if SQLITE_ENABLE_LOCKING_STYLE /* - ** PRAGMA [database.]lock_proxy_file - ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** PRAGMA [schema.]lock_proxy_file + ** PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path" ** ** Return or set the value of the lock_proxy_file flag. Changing ** the value sets a specific file to be used for database access locks. @@ -105645,14 +108601,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - - if( proxy_file_path ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "lock_proxy_file", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "lock_proxy_file", proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -105674,8 +108623,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* - ** PRAGMA [database.]synchronous - ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL + ** PRAGMA [schema.]synchronous + ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA ** ** Return or set the local value of the synchronous flag. Changing ** the local value does not make changes to the disk file and the @@ -105684,7 +108633,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + returnSingleInt(v, "synchronous", pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, @@ -105693,6 +108642,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; + pDb->bSyncSet = 1; setAllPagerFlags(db); } } @@ -105703,7 +108653,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(pParse, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); + returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); }else{ int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ @@ -105753,35 +108703,22 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "cid", "name", "type", "notnull", "dflt_value", "pk" + }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){ if( IsHiddenColumn(pCol) ){ nHidden++; continue; } - sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); - if( pCol->zDflt ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, 5); - } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ @@ -105789,7 +108726,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - sqlite3VdbeAddOp2(v, OP_Integer, k, 6); + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); + sqlite3VdbeMultiLoad(v, 1, "issisi", + i-nHidden, + pCol->zName, + sqlite3ColumnType(pCol,""), + pCol->notNull ? 1 : 0, + pCol->pDflt ? pCol->pDflt->u.zToken : 0, + k); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -105797,31 +108741,26 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_STATS: { + static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Null, 0, 2); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeMultiLoad(v, 1, "ssii", + pTab->zName, + 0, + pTab->szTabRow, + pTab->nRowLogEst); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeMultiLoad(v, 2, "sii", + pIdx->zName, + pIdx->szIdxRow, + pIdx->aiRowLogEst[0]); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -105833,6 +108772,9 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ + static const char *azCol[] = { + "seqno", "cid", "name", "desc", "coll", "key" + }; int i; int mx; if( pPragma->iArg ){ @@ -105845,29 +108787,18 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem = 3; } pTab = pIdx->pTable; - sqlite3VdbeSetNumCols(v, pParse->nMem); sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - if( pPragma->iArg ){ - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "desc", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "coll", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "key", SQLITE_STATIC); - } + assert( pParse->nMem<=ArraySize(azCol) ); + setAllColumnNames(v, pParse->nMem, azCol); for(i=0; i<mx; i++){ i16 cnum = pIdx->aiColumn[i]; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); - if( cnum<0 ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, 3); - }else{ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); - } + sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zName); if( pPragma->iArg ){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->aSortOrder[i], 4); - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, pIdx->azColl[i], 0); - sqlite3VdbeAddOp2(v, OP_Integer, i<pIdx->nKeyCol, 6); + sqlite3VdbeMultiLoad(v, 4, "isi", + pIdx->aSortOrder[i], + pIdx->azColl[i], + i<pIdx->nKeyCol); } sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); } @@ -105881,22 +108812,21 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "seq", "name", "unique", "origin", "partial" + }; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 5); pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "origin", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "partial", SQLITE_STATIC); + setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, azOrigin[pIdx->idxType], 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->pPartIdxWhere!=0, 5); + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } @@ -105904,35 +108834,31 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_DATABASE_LIST: { + static const char *azCol[] = { "seq", "name", "file" }; int i; - sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; i<db->nDb; i++){ if( db->aDb[i].pBt==0 ) continue; assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { + static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; - sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } @@ -105948,33 +108874,26 @@ SQLITE_PRIVATE void sqlite3Pragma( v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ + static const char *azCol[] = { + "id", "seq", "table", "from", "to", "on_update", "on_delete", + "match" + }; int i = 0; - sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); while(pFK){ int j; for(j=0; j<pFK->nCol; j++){ - char *zCol = pFK->aCol[j].zCol; - char *zOnDelete = (char *)actionName(pFK->aAction[0]); - char *zOnUpdate = (char *)actionName(pFK->aAction[1]); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, j, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; @@ -106003,17 +108922,14 @@ SQLITE_PRIVATE void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ + static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -106028,8 +108944,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, - P4_TRANSIENT); + sqlite3VdbeLoadString(v, regResult, pTab->zName); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; @@ -106074,7 +108989,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; j<pFK->nCol; j++){ @@ -106084,15 +108999,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, - pFK->zTo, P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -106109,7 +109022,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_PARSER_TRACE: { if( zRight ){ if( sqlite3GetBoolean(zRight, 0) ){ - sqlite3ParserTrace(stderr, "parser: "); + sqlite3ParserTrace(stdout, "parser: "); }else{ sqlite3ParserTrace(0, 0); } @@ -106140,17 +109053,6 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; - /* Code that appears at the end of the integrity check. If no error - ** messages have been generated, output OK. Otherwise output the - ** error message - */ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList endCode[] = { - { OP_IfNeg, 1, 0, 0}, /* 0 */ - { OP_String8, 0, 3, 0}, /* 1 */ - { OP_ResultRow, 3, 1, 0}, - }; - int isQuick = (sqlite3Tolower(zLeft[0])=='q'); /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check", @@ -106168,8 +109070,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -106185,7 +109086,10 @@ SQLITE_PRIVATE void sqlite3Pragma( for(i=0; i<db->nDb; i++){ HashElem *x; Hash *pTbls; + int *aRoot; int cnt = 0; + int mxIdx = 0; + int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -106198,31 +109102,35 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Do an integrity check of the B-Tree ** - ** Begin by filling registers 2, 3, ... with the root pages numbers + ** Begin by finding the root pages numbers ** for all tables and indices in the database. */ assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - VdbeComment((v, "%s", pTab->zName)); - cnt++; - } + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); - VdbeComment((v, "%s", pIdx->zName)); - cnt++; + aRoot[cnt++] = pIdx->tnum; } } + aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, cnt+8 ); + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, @@ -106250,13 +109158,14 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - pParse->nMem = MAX(pParse->nMem, 8+j); + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ @@ -106291,13 +109200,11 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - " missing from index ", P4_STATIC); + sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); @@ -106312,20 +109219,19 @@ SQLITE_PRIVATE void sqlite3Pragma( int kk; for(kk=0; kk<pIdx->nKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol>=0 && iCol<pTab->nCol ); - if( pTab->aCol[iCol].notNull ) continue; + assert( iCol!=XN_ROWID && iCol<pTab->nCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeGoto(v, uniqOk); sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - "non-unique entry in index ", P4_STATIC); - sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); } sqlite3VdbeJumpHere(v, jmp4); @@ -106334,8 +109240,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, - "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); @@ -106345,17 +109250,30 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeLoadString(v, 3, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } #endif /* SQLITE_OMIT_BTREECOUNT */ } } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP3(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 4, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, /* 3 */ + }; + VdbeOp *aOp; + + aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + if( aOp ){ + aOp[0].p2 = -mxErr; + aOp[2].p4type = P4_STATIC; + aOp[2].p4.z = "ok"; + } + } } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -106401,14 +109319,10 @@ SQLITE_PRIVATE void sqlite3Pragma( const struct EncName *pEnc; if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_String8, 0, 1); assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -106437,16 +109351,18 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS /* - ** PRAGMA [database.]schema_version - ** PRAGMA [database.]schema_version = <integer> + ** PRAGMA [schema.]schema_version + ** PRAGMA [schema.]schema_version = <integer> + ** + ** PRAGMA [schema.]user_version + ** PRAGMA [schema.]user_version = <integer> ** - ** PRAGMA [database.]user_version - ** PRAGMA [database.]user_version = <integer> + ** PRAGMA [schema.]freelist_count ** - ** PRAGMA [database.]freelist_count = <integer> + ** PRAGMA [schema.]data_version ** - ** PRAGMA [database.]application_id - ** PRAGMA [database.]application_id = <integer> + ** PRAGMA [schema.]application_id + ** PRAGMA [schema.]application_id = <integer> ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both @@ -106473,14 +109389,16 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_Integer, 0, 1, 0}, /* 1 */ - { OP_SetCookie, 0, 0, 1}, /* 2 */ + { OP_SetCookie, 0, 0, 0}, /* 1 */ }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); - sqlite3VdbeChangeP1(v, addr+2, iDb); - sqlite3VdbeChangeP2(v, addr+2, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p2 = iCookie; + aOp[1].p3 = sqlite3Atoi(zRight); }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -106488,12 +109406,16 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP3(v, addr+1, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p3 = iCookie; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + sqlite3VdbeReusable(v); } } break; @@ -106509,24 +109431,25 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; - sqlite3VdbeSetNumCols(v, 1); pParse->nMem = 1; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } + sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate + ** PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { + static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -106538,12 +109461,8 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - sqlite3VdbeSetNumCols(v, 3); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); - sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -106561,7 +109480,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "wal_autocheckpoint", + returnSingleInt(v, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -106594,7 +109513,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "timeout", db->busyTimeout); + returnSingleInt(v, "timeout", db->busyTimeout); break; } @@ -106614,7 +109533,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); break; } @@ -106633,7 +109552,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(pParse, "threads", + returnSingleInt(v, "threads", sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -106646,17 +109565,15 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; + static const char *azCol[] = { "database", "status" }; int i; - sqlite3VdbeSetNumCols(v, 2); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; i<db->nDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; @@ -106664,7 +109581,7 @@ SQLITE_PRIVATE void sqlite3Pragma( SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -106755,13 +109672,12 @@ static void corruptSchema( if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ char *z; if( zObj==0 ) zObj = "?"; - z = sqlite3_mprintf("malformed database schema (%s)", zObj); - if( z && zExtra ) z = sqlite3_mprintf("%z - %s", z, zExtra); + z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); + if( zExtra ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); sqlite3DbFree(db, *pData->pzErrMsg); *pData->pzErrMsg = z; - if( z==0 ) db->mallocFailed = 1; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; + pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } /* @@ -106818,7 +109734,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char }else{ pData->rc = rc; if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ corruptSchema(pData, argv[0], sqlite3_errmsg(db)); } @@ -106864,61 +109780,27 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #ifndef SQLITE_OMIT_DEPRECATED int size; #endif - Table *pTab; Db *pDb; char const *azArg[4]; int meta[5]; InitData initData; - char const *zMasterSchema; - char const *zMasterName; + const char *zMasterName; int openedTransaction = 0; - /* - ** The master database table has a structure like this - */ - static const char master_schema[] = - "CREATE TABLE sqlite_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#ifndef SQLITE_OMIT_TEMPDB - static const char temp_master_schema[] = - "CREATE TEMP TABLE sqlite_temp_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#else - #define temp_master_schema 0 -#endif - assert( iDb>=0 && iDb<db->nDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - /* zMasterSchema and zInitScript are set to point at the master schema - ** and initialisation script appropriate for the database being - ** initialized. zMasterName is the name of the master table. - */ - if( !OMIT_TEMPDB && iDb==1 ){ - zMasterSchema = temp_master_schema; - }else{ - zMasterSchema = master_schema; - } - zMasterName = SCHEMA_TABLE(iDb); - - /* Construct the schema tables. */ - azArg[0] = zMasterName; + /* Construct the in-memory representation schema tables (sqlite_master or + ** sqlite_temp_master) by invoking the parser directly. The appropriate + ** table name will be inserted automatically by the parser so we can just + ** use the abbreviation "x" here. The parser will also automatically tag + ** the schema table as read-only. */ + azArg[0] = zMasterName = SCHEMA_TABLE(iDb); azArg[1] = "1"; - azArg[2] = zMasterSchema; + azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage integer,sql text)"; azArg[3] = 0; initData.db = db; initData.iDb = iDb; @@ -106929,10 +109811,6 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ rc = initData.rc; goto error_out; } - pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); - if( ALWAYS(pTab) ){ - pTab->tabFlags |= TF_Readonly; - } /* Create a cursor to hold the database open */ @@ -107051,7 +109929,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", + "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { @@ -107073,7 +109951,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #endif } if( db->mallocFailed ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); } if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ @@ -107101,7 +109979,7 @@ initone_error_out: error_out: if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } @@ -107199,7 +110077,7 @@ static void schemaIsValid(Parse *pParse){ if( !sqlite3BtreeIsInReadTrans(pBt) ){ rc = sqlite3BtreeBeginTrans(pBt, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } if( rc!=SQLITE_OK ) return; openedTransaction = 1; @@ -107262,6 +110140,11 @@ SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + } + pParse->disableLookaside = 0; } } @@ -107285,12 +110168,12 @@ static int sqlite3Prepare( /* Allocate the parsing context */ pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_prepare; } pParse->pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - assert( !db->mallocFailed ); + /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); /* Check to verify that it is possible to get a read lock on all @@ -107347,8 +110230,8 @@ static int sqlite3Prepare( zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); - sqlite3DbFree(db, zSqlCopy); pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + sqlite3DbFree(db, zSqlCopy); }else{ pParse->zTail = &zSql[nBytes]; } @@ -107357,15 +110240,12 @@ static int sqlite3Prepare( } assert( 0==pParse->nQueryLoop ); - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; if( pParse->checkSchema ){ schemaIsValid(pParse); } if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + pParse->rc = SQLITE_NOMEM_BKPT; } if( pzTail ){ *pzTail = pParse->zTail; @@ -107481,7 +110361,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); if( rc ){ if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } assert( pNew==0 ); return rc; @@ -107677,6 +110557,7 @@ struct SortCtx { int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -107727,36 +110608,44 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - u16 selFlags, /* Flag parameters, such as SF_Distinct */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ Select *pNew; Select standin; sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ){ assert( db->mallocFailed ); pNew = &standin; - memset(pNew, 0, sizeof(*pNew)); } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); } pNew->pEList = pEList; + pNew->op = TK_SELECT; + pNew->selFlags = selFlags; + pNew->iLimit = 0; + pNew->iOffset = 0; +#if SELECTTRACE_ENABLED + pNew->zSelName[0] = 0; +#endif + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = selFlags; - pNew->op = TK_SELECT; + pNew->pPrior = 0; + pNew->pNext = 0; pNew->pLimit = pLimit; pNew->pOffset = pOffset; + pNew->pWith = 0; assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; if( db->mallocFailed ) { clearSelect(db, pNew, pNew!=&standin); pNew = 0; @@ -108029,12 +110918,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->jointype & JT_NATURAL ){ + if( pRight->fg.jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -108119,6 +111008,7 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ int nData, /* Number of elements in the data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ @@ -108130,8 +111020,10 @@ static void pushOntoSorter( int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); + assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -108139,14 +111031,17 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(v); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -108181,6 +111076,10 @@ static void pushOntoSorter( pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); @@ -108191,15 +111090,9 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( pSelect->iLimit ){ + if( iLimit ){ int addr; - int iLimit; - if( pSelect->iOffset ){ - iLimit = pSelect->iOffset+1; - }else{ - iLimit = pSelect->iLimit; - } - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr); @@ -108215,11 +111108,8 @@ static void codeOffset( int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); } } @@ -108349,7 +111239,7 @@ static void selectInnerLoop( }else{ ecelFlags = 0; } - sqlite3ExprCodeExprList(pParse, pEList, regResult, ecelFlags); + sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -108465,7 +111355,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -108491,7 +111381,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -108517,7 +111407,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -108531,7 +111421,8 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -108617,8 +111508,8 @@ static void selectInnerLoop( ** X extra columns. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - KeyInfo *p = sqlite3DbMallocZero(0, - sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; @@ -108626,8 +111517,9 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; + memset(&p[1], 0, nExtra); }else{ - db->mallocFailed = 1; + sqlite3OomFault(db); } return p; } @@ -108804,7 +111696,7 @@ static void generateSortTail( SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; @@ -108823,9 +111715,10 @@ static void generateSortTail( struct ExprList_item *aOutEx = p->pEList->a; #endif + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; @@ -108962,7 +111855,8 @@ static const char *columnTypeImpl( char const *zOrigCol = 0; #endif - if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; + assert( pExpr!=0 ); + assert( pNC->pSrcList!=0 ); switch( pExpr->op ){ case TK_AGG_COLUMN: case TK_COLUMN: { @@ -109038,8 +111932,8 @@ static const char *columnTypeImpl( zType = "INTEGER"; zOrigCol = "rowid"; }else{ - zType = pTab->aCol[iCol].zType; zOrigCol = pTab->aCol[iCol].zName; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; @@ -109051,7 +111945,7 @@ static const char *columnTypeImpl( if( iCol<0 ){ zType = "INTEGER"; }else{ - zType = pTab->aCol[iCol].zType; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } #endif @@ -109150,7 +112044,9 @@ static void generateColumnNames( } #endif - if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; + if( pParse->colNamesSet || db->mallocFailed ) return; + assert( v!=0 ); + assert( pTabList!=0 ); pParse->colNamesSet = 1; fullNames = (db->flags & SQLITE_FullColNames)!=0; shortNames = (db->flags & SQLITE_ShortColNames)!=0; @@ -109162,7 +112058,7 @@ static void generateColumnNames( if( pEList->a[i].zName ){ char *zName = pEList->a[i].zName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ + }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){ Table *pTab; char *zCol; int iCol = p->iColumn; @@ -109210,7 +112106,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int selectColumnsFromExprList( +SQLITE_PRIVATE int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -109218,13 +112114,15 @@ static int selectColumnsFromExprList( ){ sqlite3 *db = pParse->db; /* Database connection */ int i, j; /* Loop counters */ - int cnt; /* Index added to make the name unique */ + u32 cnt; /* Index added to make the name unique */ Column *aCol, *pCol; /* For looping over result columns */ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ + Hash ht; /* Hash table of column names */ + sqlite3HashInit(&ht); if( pEList ){ nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); @@ -109233,16 +112131,16 @@ static int selectColumnsFromExprList( nCol = 0; aCol = 0; } + assert( nCol==(i16)nCol ); *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; i<nCol; i++, pCol++){ + for(i=0, pCol=aCol; i<nCol && !db->mallocFailed; i++, pCol++){ /* Get an appropriate name for the column */ p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS <name>" phrase, use <name> as the name */ - zName = sqlite3DbStrDup(db, zName); }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ @@ -109255,41 +112153,37 @@ static int selectColumnsFromExprList( int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; - zName = sqlite3MPrintf(db, "%s", - iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); + zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; }else if( pColExpr->op==TK_ID ){ assert( !ExprHasProperty(pColExpr, EP_IntValue) ); - zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); + zName = pColExpr->u.zToken; }else{ /* Use the original text of the column expression as its name */ - zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); + zName = pEList->a[i].zSpan; } } - if( db->mallocFailed ){ - sqlite3DbFree(db, zName); - break; - } + zName = sqlite3MPrintf(db, "%s", zName); /* Make sure the column name is unique. If the name is not unique, ** append an integer to the name so that it becomes unique. */ - nName = sqlite3Strlen30(zName); - for(j=cnt=0; j<i; j++){ - if( sqlite3StrICmp(aCol[j].zName, zName)==0 ){ - char *zNewName; - int k; - for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){} - if( k>=0 && zName[k]==':' ) nName = k; - zName[nName] = 0; - zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); - sqlite3DbFree(db, zName); - zName = zNewName; - j = -1; - if( zName==0 ) break; + cnt = 0; + while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + nName = sqlite3Strlen30(zName); + if( nName>0 ){ + for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} + if( zName[j]==':' ) nName = j; } + zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); + if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); } pCol->zName = zName; + sqlite3ColumnPropertiesFromName(0, pCol); + if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + sqlite3OomFault(db); + } } + sqlite3HashClear(&ht); if( db->mallocFailed ){ for(j=0; j<i; j++){ sqlite3DbFree(db, aCol[j].zName); @@ -109297,7 +112191,7 @@ static int selectColumnsFromExprList( sqlite3DbFree(db, aCol); *paCol = 0; *pnCol = 0; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return SQLITE_OK; } @@ -109313,7 +112207,7 @@ static int selectColumnsFromExprList( ** This routine requires that all identifiers in the SELECT ** statement be resolved. */ -static void selectAddColumnTypeAndCollation( +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ Table *pTab, /* Add column type information to this table */ Select *pSelect /* SELECT used to determine types and collations */ @@ -109335,13 +112229,20 @@ static void selectAddColumnTypeAndCollation( sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){ + const char *zType; + int n, m; p = a[i].pExpr; - if( pCol->zType==0 ){ - pCol->zType = sqlite3DbStrDup(db, - columnType(&sNC, p,0,0,0, &pCol->szEst)); - } + zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); + if( zType && (m = sqlite3Strlen30(zType))>0 ){ + n = sqlite3Strlen30(pCol->zName); + pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); + if( pCol->zName ){ + memcpy(&pCol->zName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ @@ -109373,12 +112274,12 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside ** is disabled */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -109434,7 +112335,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, n; + int n; if( p->iLimit ) return; /* @@ -109453,9 +112354,10 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); - }else if( n>=0 && p->nSelectRow>(u64)n ){ - p->nSelectRow = n; + sqlite3VdbeGoto(v, iBreak); + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -109469,14 +112371,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); } } } @@ -109556,7 +112452,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -109621,7 +112517,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(i<pSrc->nSrc); i++){ - if( pSrc->a[i].isRecursive ){ + if( pSrc->a[i].fg.isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -109701,7 +112597,7 @@ static void generateWithRecursiveQuery( } /* Keep running the loop until the Queue is empty */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeGoto(v, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: @@ -109839,7 +112735,6 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -109892,18 +112787,22 @@ static int multiSelect( if( p->iLimit ){ addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > (u64)nLimit + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ - p->nSelectRow = nLimit; + p->nSelectRow = sqlite3LogEst((u64)nLimit); } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -109975,7 +112874,9 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; - if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -109992,7 +112893,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -110067,7 +112968,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -110110,7 +113011,7 @@ static int multiSelect( nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto multi_select_end; } for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ @@ -110199,12 +113100,12 @@ static int generateOutputSubroutine( /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } @@ -110421,7 +113322,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -110465,7 +113366,7 @@ static int multiSelectOrderBy( } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return SQLITE_NOMEM; + if( pNew==0 ) return SQLITE_NOMEM_BKPT; pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); @@ -110481,10 +113382,11 @@ static int multiSelectOrderBy( ** to the right and the left are evaluated, they use the correct ** collation. */ - aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); + aPermute = sqlite3DbMallocRawNN(db, sizeof(int)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; - for(i=0, pItem=pOrderBy->a; i<nOrderBy; i++, pItem++){ + aPermute[0] = nOrderBy; + for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; @@ -110557,19 +113459,19 @@ static int multiSelectOrderBy( ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeEndCoroutine(v, regAddrA); + sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; @@ -110579,7 +113481,7 @@ static int multiSelectOrderBy( sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); + sqlite3VdbeEndCoroutine(v, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. @@ -110610,8 +113512,8 @@ static int multiSelectOrderBy( addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); - p->nSelectRow += pPrior->nSelectRow; + sqlite3VdbeGoto(v, addrEofA); + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } /* Generate a subroutine to run when the results from select B @@ -110624,7 +113526,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of A<B @@ -110632,7 +113534,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); /* Generate code to handle the case of A==B */ @@ -110645,7 +113547,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "A-eq-B subroutine")); addrAeqB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); } /* Generate code to handle the case of A>B @@ -110656,11 +113558,11 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); @@ -110682,7 +113584,7 @@ static int multiSelectOrderBy( if( pDest->eDest==SRT_Output ){ Select *pFirst = pPrior; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); } /* Reassembly the compound query so that it will be freed correctly @@ -110703,7 +113605,7 @@ static int multiSelectOrderBy( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -110740,7 +113642,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -110763,25 +113665,28 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -110933,7 +113838,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -111036,7 +113941,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ return 0; } @@ -111207,7 +114112,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; + jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -111228,9 +114133,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -111244,10 +114149,11 @@ static int flattenSubquery( */ for(i=0; i<nSubSrc; i++){ sqlite3IdListDelete(db, pSrc->a[i+iFrom].pUsing); + assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].jointype = jointype; + pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -111269,11 +114175,6 @@ static int flattenSubquery( pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th @@ -111293,27 +114194,20 @@ static int flattenSubquery( assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -111380,6 +114274,9 @@ static int flattenSubquery( ** enforces this restriction since this routine does not have enough ** information to know.) ** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -111402,6 +114299,7 @@ static int pushDownWhereTerms( nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ @@ -111498,9 +114396,9 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndexedBy ){ + if( pFrom->pTab && pFrom->fg.isIndexedBy ){ Table *pTab = pFrom->pTab; - char *zIndexedBy = pFrom->zIndexedBy; + char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; for(pIdx=pTab->pIndex; pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); @@ -111511,7 +114409,7 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pF pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + pFrom->pIBIndex = pIdx; } return SQLITE_OK; } @@ -111567,7 +114465,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNewSrc==0 ) return WRC_Abort; *pNew = *p; p->pSrc = pNewSrc; - p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); p->op = TK_SELECT; p->pWhere = 0; pNew->pGroupBy = 0; @@ -111586,6 +114484,19 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } +/* +** Check to see if the FROM clause term pFrom has table-valued function +** arguments. If it does, leave an error message in pParse and return +** non-zero, since pFrom is not allowed to be a table-valued function. +*/ +static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ + if( pFrom->fg.isTabFunc ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); + return 1; + } + return 0; +} + #ifndef SQLITE_OMIT_CTE /* ** Argument pWith (which may be NULL) points to a linked list of nested @@ -111598,7 +114509,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ ** object that the returned CTE belongs to. */ static struct Cte *searchWith( - With *pWith, /* Current outermost WITH clause */ + With *pWith, /* Current innermost WITH clause */ struct SrcList_item *pItem, /* FROM clause element to resolve */ With **ppContext /* OUT: WITH clause return value belongs to */ ){ @@ -111629,11 +114540,12 @@ static struct Cte *searchWith( ** statement with which it is associated. */ SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ - assert( bFree==0 || pParse->pWith==0 ); + assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) ); if( pWith ){ + assert( pParse->pWith!=pWith ); pWith->pOuter = pParse->pWith; pParse->pWith = pWith; - pParse->bFreeWith = bFree; + if( bFree ) pParse->pWithToFree = pWith; } } @@ -111672,14 +114584,15 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zErr is non-NULL at this point, then this is an illegal + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zErr ){ - sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); return SQLITE_ERROR; } + if( cannotBeFunction(pParse, pFrom) ) return SQLITE_ERROR; assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); @@ -111690,7 +114603,7 @@ static int withExpand( pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( db->mallocFailed ) return SQLITE_NOMEM; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); /* Check if this is a recursive CTE. */ @@ -111706,7 +114619,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->isRecursive = 1; + pItem->fg.isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -111722,10 +114635,11 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference: %s"; + pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; @@ -111740,16 +114654,16 @@ static int withExpand( pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references: %s"; + pCte->zCteErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference in a subquery: %s"; + pCte->zCteErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zErr = 0; + pCte->zCteErr = 0; pParse->pWith = pSavedWith; } @@ -111836,17 +114750,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->isRecursive==0 || pFrom->pTab ); - if( pFrom->isRecursive ) continue; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); -#ifndef SQLITE_OMIT_CTE - selectPopWith(pWalker, p); -#endif - return WRC_Prune; - } + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->fg.isRecursive ) continue; + assert( pFrom->pTab==0 ); #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else @@ -111863,7 +114769,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; @@ -111880,14 +114786,20 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Abort; } pTab->nRef++; + if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ + return WRC_Abort; + } #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ + if( IsVirtual(pTab) || pTab->pSelect ){ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } @@ -111907,19 +114819,20 @@ static int selectExpander(Walker *pWalker, Select *p){ /* For every "*" that occurs in the column list, insert the names of ** all columns in all tables. And for every TABLE.* insert the names ** of all columns in TABLE. The parser inserted a special expression - ** with the TK_ALL operator for each "*" that it found in the column list. - ** The following code just has to locate the TK_ALL expressions and expand - ** each one to the list of all columns in all tables. + ** with the TK_ASTERISK operator for each "*" that it found in the column + ** list. The following code just has to locate the TK_ASTERISK + ** expressions and expand each one to the list of all columns in + ** all tables. ** ** The first loop just checks to see if there are any "*" operators ** that need expanding. */ for(k=0; k<pEList->nExpr; k++){ pE = pEList->a[k].pExpr; - if( pE->op==TK_ALL ) break; + if( pE->op==TK_ASTERISK ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); - if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; + if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; } if( k<pEList->nExpr ){ /* @@ -111937,7 +114850,9 @@ static int selectExpander(Walker *pWalker, Select *p){ pE = a[k].pExpr; pRight = pE->pRight; assert( pE->op!=TK_DOT || pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ + if( pE->op!=TK_ASTERISK + && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK) + ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -111989,18 +114904,19 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } - /* If a column is marked as 'hidden' (currently only possible - ** for virtual tables), do not include it in the expanded - ** result-set list. + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. */ - if( IsHiddenColumn(&pTab->aCol[j]) ){ - assert(IsVirtual(pTab)); + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ continue; } tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 + if( (pFrom->fg.jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -112032,8 +114948,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pExpr = pRight; } pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sColname.z = zColname; - sColname.n = sqlite3Strlen30(zColname); + sqlite3TokenInit(&sColname, zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; @@ -112065,6 +114980,7 @@ static int selectExpander(Walker *pWalker, Select *p){ #if SQLITE_MAX_COLUMN if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; } #endif return WRC_Continue; @@ -112079,7 +114995,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** Walker.xSelectCallback is set to do something useful for every ** subquery in the parser tree. */ -static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ UNUSED_PARAMETER2(NotUsed, NotUsed2); return WRC_Continue; } @@ -112100,7 +115016,7 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; @@ -112135,19 +115051,19 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + assert( (p->selFlags & SF_HasTypeInfo)==0 ); + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -112167,7 +115083,7 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); w.xSelectCallback2 = selectAddSubqueryTypeInfo; - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); #endif @@ -112286,7 +115202,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; @@ -112473,7 +115389,17 @@ SQLITE_PRIVATE int sqlite3Select( struct SrcList_item *pItem = &pTabList->a[i]; Select *pSub = pItem->pSelect; int isAggSub; + Table *pTab = pItem->pTab; if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ /* This subquery can be absorbed into its parent. */ @@ -112527,7 +115453,7 @@ SQLITE_PRIVATE int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ + if( pItem->fg.viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -112545,7 +115471,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. */ - if( (pItem->jointype & JT_OUTER)==0 + if( (pItem->fg.jointype & JT_OUTER)==0 && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) ){ #if SELECTTRACE_ENABLED @@ -112557,10 +115483,24 @@ SQLITE_PRIVATE int sqlite3Select( } /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if all of these are true: + ** (1) The subquery is guaranteed to be the outer loop (so that it + ** does not need to be computed more than once) + ** (2) The ALL keyword after SELECT is omitted. (Applications are + ** allowed to say "SELECT ALL" instead of just "SELECT" to disable + ** the use of co-routines.) + ** (3) Co-routines are not disabled using sqlite3_test_control() + ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** + ** TODO: Are there other reasons beside (1) to use a co-routine + ** implementation? */ - if( pTabList->nSrc==1 - && (p->selFlags & SF_All)==0 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (p->selFlags & SF_All)==0 /* (2) */ + && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -112573,10 +115513,10 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; + pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else{ @@ -112592,7 +115532,7 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ + if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ @@ -112604,7 +115544,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); @@ -112687,10 +115627,10 @@ SQLITE_PRIVATE int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = LARGEST_INT64; + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; } @@ -112711,10 +115651,12 @@ SQLITE_PRIVATE int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + wctrlFlags |= p->selFlags & SF_FixedLimit; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, 0); + p->pEList, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -112774,9 +115716,11 @@ SQLITE_PRIVATE int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->u.x.iAlias = 0; } - if( p->nSelectRow>100 ) p->nSelectRow = 100; + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; }else{ - p->nSelectRow = 1; + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; } /* If there is both a GROUP BY and an ORDER BY clause and they are @@ -112825,7 +115769,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -112906,19 +115850,14 @@ SQLITE_PRIVATE int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; i<sAggInfo.nColumn; i++){ struct AggInfo_col *pCol = &sAggInfo.aCol[i]; if( pCol->iSorterColumn>=j ){ int r1 = j + regBase; - int r2; - - r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); - if( r1!=r2 ){ - sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); - } + sqlite3ExprCodeGetColumnToReg(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1); j++; } } @@ -112973,8 +115912,8 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -112996,7 +115935,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -113018,7 +115957,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Jump over the subroutines */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + sqlite3VdbeGoto(v, addrEnd); /* Generate a subroutine that outputs a single row of the result ** set. This subroutine first looks at the iUseFlag. If iUseFlag @@ -113153,7 +116092,8 @@ SQLITE_PRIVATE int sqlite3Select( if( flag ){ pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; - if( pMinMax && !db->mallocFailed ){ + assert( db->mallocFailed || pMinMax!=0 ); + if( !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; pMinMax->a[0].pExpr->op = TK_COLUMN; } @@ -113172,7 +116112,7 @@ SQLITE_PRIVATE int sqlite3Select( updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); + sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -113337,7 +116277,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ return 0; malloc_failed: - p->rc = SQLITE_NOMEM; + p->rc = SQLITE_NOMEM_BKPT; return 1; } @@ -113378,7 +116318,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_table( res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); if( res.azResult==0 ){ db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } res.azResult[0] = 0; rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); @@ -113407,7 +116347,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_table( if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } res.azResult = azNew; } @@ -113726,8 +116666,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( pStepList->pTrig = pTrig; pStepList = pStepList->pNext; } - nameToken.z = pTrig->zName; - nameToken.n = sqlite3Strlen30(nameToken.z); + sqlite3TokenInit(&nameToken, pTrig->zName); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) || sqlite3FixExpr(&sFix, pTrig->pWhen) @@ -113763,7 +116702,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); @@ -113998,31 +116937,12 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ */ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ - int base; - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList dropTrigger[] = { - { OP_Rewind, 0, ADDR(9), 0}, - { OP_String8, 0, 1, 0}, /* 1 */ - { OP_Column, 0, 1, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_String8, 0, 1, 0}, /* 4: "trigger" */ - { OP_Column, 0, 0, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_Delete, 0, 0, 0}, - { OP_Next, 0, ADDR(1), 0}, /* 8 */ - }; - - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3OpenMasterTable(pParse, iDb); - base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); - sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); - sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pTrigger->zName + ); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Close, 0, 0); sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); - if( pParse->nMem<3 ){ - pParse->nMem = 3; - } } } @@ -114410,8 +117330,8 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( if( pPrg ){ int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); - sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); + sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem, + (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); @@ -114695,9 +117615,9 @@ SQLITE_PRIVATE void sqlite3Update( /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; /* Content of the NEW.* table in triggers */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ @@ -114758,7 +117678,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; aToOpen = (u8*)(aRegIdx+nIdx); @@ -114824,16 +117744,20 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; - /* The SET expressions are not actually used inside the WHERE loop. - ** So reset the colUsed mask + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask. Unless this is a virtual table. In that + ** case, set all bits of the colUsed mask (to ensure that the virtual + ** table implementation makes all columns available). */ - pTabList->a[0].colUsed = 0; + pTabList->a[0].colUsed = IsVirtual(pTab) ? ALLBITS : 0; hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. + ** the key for accessing each index. + ** + ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -114842,7 +117766,8 @@ SQLITE_PRIVATE void sqlite3Update( }else{ reg = 0; for(i=0; i<pIdx->nKeyCol; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ + i16 iIdxCol = pIdx->aiColumn[i]; + if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; break; } @@ -114858,29 +117783,20 @@ SQLITE_PRIVATE void sqlite3Update( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - pWhere = 0; - pTabList = 0; - goto update_cleanup; - } -#endif - /* Allocate required registers. */ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; + if( !IsVirtual(pTab) ){ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; pParse->nMem += pTab->nCol; } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; /* Start the view context. */ if( isView ){ @@ -114903,6 +117819,15 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif + /* Begin the database scan */ if( HasRowid(pTab) ){ @@ -114942,6 +117867,7 @@ SQLITE_PRIVATE void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; i<nPk; i++){ + assert( pPk->aiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } @@ -114951,7 +117877,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -114986,7 +117912,7 @@ SQLITE_PRIVATE void sqlite3Update( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); } @@ -115064,7 +117990,6 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; i<pTab->nCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -115080,7 +118005,7 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } @@ -115122,13 +118047,14 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int j1 = 0; /* Address of jump instruction */ + int addr1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace); + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, + aXRef); /* Do FK constraint checks. */ if( hasFK ){ @@ -115138,20 +118064,20 @@ SQLITE_PRIVATE void sqlite3Update( /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } if( hasFK ){ @@ -115188,7 +118114,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + sqlite3VdbeGoto(v, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); @@ -115242,21 +118168,23 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemeral table that contains +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) +** (B) The revised rowid for the row. ** (C) The content of every column in the row. ** -** Then we loop over this ephemeral table and for each row in -** the ephemeral table call VUpdate. -** -** When finished, drop the ephemeral table. +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. ** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ @@ -115269,66 +118197,96 @@ static void updateVirtualTable( int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; - - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); + WhereInfo *pWInfo; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int bOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ + + /* Allocate nArg registers to martial the arguments to VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ + assert( v ); + ephemTab = pParse->nTab++; + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); } - assert( pTab->iPKey<0 ); for(i=0; i<pTab->nCol; i++){ if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ - assert( v ); - ephemTab = pParse->nTab++; - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_EphemTab, ephemTab); - sqlite3Select(pParse, pSelect, &dest); + bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); - for(i=0; i<pTab->nCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + if( bOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. Also, if this is a top-level parse (not a trigger), clear the + ** multi-write flag so that the VM does not open a statement journal */ + sqlite3VdbeChangeToNoop(v, addr); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + + + if( bOnePass==0 ){ + /* End the virtual table scan */ + sqlite3WhereEnd(pWInfo); + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; i<nArg; i++){ + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i, regArg+i); + } } sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + /* End of the ephemeral table scan. Or, if using the onepass strategy, + ** jump to here if the scan visited zero rows. */ + if( bOnePass==0 ){ + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + }else{ + sqlite3WhereEnd(pWInfo); + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -115374,7 +118332,7 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; VVA_ONLY( int rc; ) if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); @@ -115555,7 +118513,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) || NEVER(db->mallocFailed) ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_of_vacuum; } @@ -115761,7 +118719,7 @@ static int createModule( rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; - pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1); if( pMod ){ Module *pDel; char *zCopy = (char *)(&pMod[1]); @@ -115770,10 +118728,11 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); } } @@ -115997,23 +118956,17 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + int nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ - int j; - for(j=0; j<i; j++){ - sqlite3DbFree(db, pTable->azModuleArg[j]); - } sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ + int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; + pTable->azModuleArg = azModuleArg; } - pTable->azModuleArg = azModuleArg; } /* @@ -116140,7 +119093,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); iReg = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, iReg, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, iReg, pTab->zName); sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); } @@ -116156,7 +119109,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ - db->mallocFailed = 1; + sqlite3OomFault(db); assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } @@ -116223,13 +119176,13 @@ static int vtabCallConstructor( zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); if( !zModuleName ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); if( !pVTable ){ sqlite3DbFree(db, zModuleName); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pVTable->db = db; pVTable->pMod = pMod; @@ -116247,7 +119200,7 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); db->pVtabCtx = sCtx.pPrior; - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ @@ -116281,22 +119234,16 @@ static int vtabCallConstructor( pTab->pVTable = pVTable; for(iCol=0; iCol<pTab->nCol; iCol++){ - char *zType = pTab->aCol[iCol].zType; + char *zType = sqlite3ColumnType(&pTab->aCol[iCol], ""); int nType; int i = 0; - if( !zType ){ - pTab->tabFlags |= oooHidden; - continue; - } nType = sqlite3Strlen30(zType); - if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ - for(i=0; i<nType; i++){ - if( (0==sqlite3StrNICmp(" hidden", &zType[i], 7)) - && (zType[i+7]=='\0' || zType[i+7]==' ') - ){ - i++; - break; - } + for(i=0; i<nType; i++){ + if( 0==sqlite3StrNICmp("hidden", &zType[i], 6) + && (i==0 || zType[i-1]==' ') + && (zType[i+6]=='\0' || zType[i+6]==' ') + ){ + break; } } if( i<nType ){ @@ -116372,7 +119319,7 @@ static int growVTrans(sqlite3 *db){ int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); db->aVTrans = aVTrans; @@ -116416,7 +119363,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( !pMod ){ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -116464,7 +119411,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3 *db, const char *zCre pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pParse->declareVtab = 1; pParse->db = db; @@ -116518,6 +119465,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ VTable *p; + int (*xDestroy)(sqlite3_vtab *); for(p=pTab->pVTable; p; p=p->pNext){ assert( p->pVtab ); if( p->pVtab->nRef>0 ){ @@ -116525,7 +119473,9 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab } } p = vtabDisconnectAll(db, pTab); - rc = p->pMod->pModule->xDestroy(p->pVtab); + xDestroy = p->pMod->pModule->xDestroy; + assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -116651,7 +119601,9 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); + if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1); } } } @@ -116728,7 +119680,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( Table *pTab; sqlite3_vtab *pVtab; sqlite3_module *pMod; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0; void *pArg = 0; FuncDef *pNew; int rc = 0; @@ -116756,7 +119708,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( for(z=(unsigned char*)zLowerName; *z; z++){ *z = sqlite3UpperToLower[*z]; } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg); sqlite3DbFree(db, zLowerName); } if( rc==0 ){ @@ -116771,9 +119723,9 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( return pDef; } *pNew = *pDef; - pNew->zName = (char *)&pNew[1]; - memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); - pNew->xFunc = xFunc; + pNew->zName = (const char*)&pNew[1]; + memcpy((char*)&pNew[1], pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xSFunc = xSFunc; pNew->pUserData = pArg; pNew->funcFlags |= SQLITE_FUNC_EPHEM; return pNew; @@ -116800,7 +119752,68 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); + } +} + +/* +** Check to see if virtual tale module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if the eponymous virtual table instance exists +** when this routine returns, and return zero if it does not exist. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int nName; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + nName = sqlite3Strlen30(pMod->zName) + 1; + pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName); + if( pTab==0 ) return 0; + pMod->pEpoTab = pTab; + pTab->zName = (char*)&pTab[1]; + memcpy(pTab->zName, pMod->zName, nName); + pTab->nRef = 1; + pTab->pSchema = db->aDb[0].pSchema; + pTab->tabFlags |= TF_Virtual; + pTab->nModuleArg = 0; + pTab->iPKey = -1; + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(db, pTab, 0); + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + return 0; + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( pTab!=0 ){ + sqlite3DeleteColumnNames(db, pTab); + sqlite3VtabClear(db, pTab); + sqlite3DbFree(db, pTab); + pMod->pEpoTab = 0; } } @@ -116957,8 +119970,10 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS int iLikeRepCntr; /* LIKE range processing counter register */ int addrLikeRep; /* LIKE range processing address */ +#endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ @@ -117141,6 +120156,7 @@ struct WhereTerm { u16 eOperator; /* A WO_xx value describing <op> */ u16 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ + u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ WhereClause *pWC; /* The clause this term is part of */ Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ @@ -117173,13 +120189,15 @@ struct WhereTerm { struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ - char *zCollName; /* Required collating sequence, if not NULL */ + const char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; /* @@ -117295,10 +120313,11 @@ struct WhereInfo { WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 sorted; /* True if really sorted (not just grouped) */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ u8 nLevel; /* Number of nested loop */ @@ -117363,6 +120382,7 @@ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); @@ -117441,6 +120461,16 @@ static void explainAppendTerm( } /* +** Return the name of the i-th column of the pIdx index. +*/ +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==XN_EXPR ) return "<expr>"; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zName; +} + +/* ** Argument pLevel describes a strategy for scanning table pTab. This ** function appends text to pStr that describes the subset of table ** rows scanned by the strategy in the form of an SQL expression. @@ -117454,33 +120484,27 @@ static void explainAppendTerm( ** ** "a=? AND b>?" */ -static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; u16 nSkip = pLoop->nSkip; int i, j; - Column *aCol = pTab->aCol; - i16 *aiColumn = pIndex->aiColumn; if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; sqlite3StrAccumAppend(pStr, " (", 2); for(i=0; i<nEq; i++){ - char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName; - if( i>=nSkip ){ - explainAppendTerm(pStr, i, z, "="); - }else{ - if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3XPrintf(pStr, 0, "ANY(%s)", z); - } + const char *z = explainIndexColumnName(pIndex, i); + if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3XPrintf(pStr, i>=nSkip ? "%s=?" : "ANY(%s)", z); } j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + const char *z = explainIndexColumnName(pIndex, i); explainAppendTerm(pStr, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + const char *z = explainIndexColumnName(pIndex, j); explainAppendTerm(pStr, i, z, "<"); } sqlite3StrAccumAppend(pStr, ")", 1); @@ -117530,13 +120554,13 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); + sqlite3XPrintf(&str, " SUBQUERY %d", pItem->iSelectId); }else{ - sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); + sqlite3XPrintf(&str, " TABLE %s", pItem->zName); } if( pItem->zAlias ){ - sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); + sqlite3XPrintf(&str, " AS %s", pItem->zAlias); } if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; @@ -117560,33 +120584,32 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( } if( zFmt ){ sqlite3StrAccumAppend(&str, " USING ", 7); - sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); - explainIndexRange(&str, pLoop, pItem->pTab); + sqlite3XPrintf(&str, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop); } }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRange; + const char *zRangeOp; if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRange = "(rowid=?)"; + zRangeOp = "="; }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRange = "(rowid>? AND rowid<?)"; + zRangeOp = ">? AND rowid<"; }else if( flags&WHERE_BTM_LIMIT ){ - zRange = "(rowid>?)"; + zRangeOp = ">"; }else{ assert( flags&WHERE_TOP_LIMIT); - zRange = "(rowid<?)"; + zRangeOp = "<"; } - sqlite3StrAccumAppendAll(&str, " USING INTEGER PRIMARY KEY "); - sqlite3StrAccumAppendAll(&str, zRange); + sqlite3XPrintf(&str, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", + sqlite3XPrintf(&str, " VIRTUAL TABLE INDEX %d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + sqlite3XPrintf(&str, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); }else{ sqlite3StrAccumAppend(&str, " (~1 row)", 9); } @@ -117723,8 +120746,7 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ /* Code the OP_Affinity opcode if there is anything left to do. */ if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); sqlite3ExprCacheAffinityChange(pParse, base, n); } } @@ -117891,10 +120913,8 @@ static int codeAllEqualityTerms( nReg = pLoop->u.btree.nEq + nExtraReg; pParse->nMem += nReg; - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; - } + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); + assert( zAff!=0 || pParse->db->mallocFailed ); if( nSkip ){ int iIdxCur = pLevel->iIdxCur; @@ -117910,8 +120930,8 @@ static int codeAllEqualityTerms( sqlite3VdbeJumpHere(v, j); for(j=0; j<nSkip; j++){ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, j, regBase+j); - assert( pIdx->aiColumn[j]>=0 ); - VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + testcase( pIdx->aiColumn[j]==XN_EXPR ); + VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); } } @@ -117957,6 +120977,7 @@ static int codeAllEqualityTerms( return regBase; } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS /* ** If the most recently coded instruction is a constant range contraint ** that originated from the LIKE optimization, then change the P3 to be @@ -117968,6 +120989,10 @@ static int codeAllEqualityTerms( ** The OP_String opcodes on the second pass convert the upper and lower ** bound string contants to blobs. This routine makes the necessary changes ** to the OP_String opcodes for that to happen. +** +** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then +** only the one pass through the string space is required, so this routine +** becomes a no-op. */ static void whereLikeOptimizationStringFixup( Vdbe *v, /* prepared statement under construction */ @@ -117985,7 +121010,206 @@ static void whereLikeOptimizationStringFixup( pOp->p5 = 1; } } +#else +# define whereLikeOptimizationStringFixup(A,B,C) +#endif + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Information is passed from codeCursorHint() down to individual nodes of +** the expression tree (by sqlite3WalkExpr()) using an instance of this +** structure. +*/ +struct CCurHint { + int iTabCur; /* Cursor for the main table */ + int iIdxCur; /* Cursor for the index, if pIdx!=0. Unused otherwise */ + Index *pIdx; /* The index used to access the table */ +}; + +/* +** This function is called for every node of an expression that is a candidate +** for a cursor hint on an index cursor. For TK_COLUMN nodes that reference +** the table CCurHint.iTabCur, verify that the same column can be +** accessed through the index. If it cannot, then set pWalker->eCode to 1. +*/ +static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ + struct CCurHint *pHint = pWalker->u.pCCurHint; + assert( pHint->pIdx!=0 ); + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pHint->iTabCur + && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + } + return WRC_Continue; +} + + +/* +** This function is called on every node of an expression tree used as an +** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN +** that accesses any table other than the one identified by +** CCurHint.iTabCur, then do the following: +** +** 1) allocate a register and code an OP_Column instruction to read +** the specified column into the new register, and +** +** 2) transform the expression node to a TK_REGISTER node that reads +** from the newly populated register. +** +** Also, if the node is a TK_COLUMN that does access the table idenified +** by pCCurHint.iTabCur, and an index is being used (which we will +** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into +** an access of the index rather than the original table. +*/ +static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ + int rc = WRC_Continue; + struct CCurHint *pHint = pWalker->u.pCCurHint; + if( pExpr->op==TK_COLUMN ){ + if( pExpr->iTable!=pHint->iTabCur ){ + Vdbe *v = pWalker->pParse->pVdbe; + int reg = ++pWalker->pParse->nMem; /* Register for column value */ + sqlite3ExprCodeGetColumnOfTable( + v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg + ); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pHint->pIdx!=0 ){ + pExpr->iTable = pHint->iIdxCur; + pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + assert( pExpr->iColumn>=0 ); + } + }else if( pExpr->op==TK_AGG_FUNCTION ){ + /* An aggregate function in the WHERE clause of a query means this must + ** be a correlated sub-query, and expression pExpr is an aggregate from + ** the parent context. Do not walk the function arguments in this case. + ** + ** todo: It should be possible to replace this node with a TK_REGISTER + ** expression, as the result of the expression must be stored in a + ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + rc = WRC_Prune; + } + return rc; +} + +/* +** Insert an OP_CursorHint instruction if it is appropriate to do so. +*/ +static void codeCursorHint( + WhereInfo *pWInfo, /* The where clause */ + WhereLevel *pLevel, /* Which loop to provide hints for */ + WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ +){ + Parse *pParse = pWInfo->pParse; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + Expr *pExpr = 0; + WhereLoop *pLoop = pLevel->pWLoop; + int iCur; + WhereClause *pWC; + WhereTerm *pTerm; + int i, j; + struct CCurHint sHint; + Walker sWalker; + + if( OptimizationDisabled(db, SQLITE_CursorHints) ) return; + iCur = pLevel->iTabCur; + assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor ); + sHint.iTabCur = iCur; + sHint.iIdxCur = pLevel->iIdxCur; + sHint.pIdx = pLoop->u.btree.pIndex; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.u.pCCurHint = &sHint; + pWC = &pWInfo->sWC; + for(i=0; i<pWC->nTerm; i++){ + pTerm = &pWC->a[i]; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->prereqAll & pLevel->notReady ) continue; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize + ** the cursor. These terms are not needed as hints for a pure range + ** scan (that has no == terms) so omit them. */ + if( pLoop->u.btree.nEq==0 && pTerm!=pEndRange ){ + for(j=0; j<pLoop->nLTerm && pLoop->aLTerm[j]!=pTerm; j++){} + if( j<pLoop->nLTerm ) continue; + } + + /* No subqueries or non-deterministic functions allowed */ + if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue; + + /* For an index scan, make sure referenced columns are actually in + ** the index. */ + if( sHint.pIdx!=0 ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintCheckExpr; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + + /* If we survive all prior tests, that means this term is worth hinting */ + pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + } + if( pExpr!=0 ){ + sWalker.xExprCallback = codeCursorHintFixExpr; + sqlite3WalkExpr(&sWalker, pExpr); + sqlite3VdbeAddOp4(v, OP_CursorHint, + (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, + (const char*)pExpr, P4_EXPR); + } +} +#else +# define codeCursorHint(A,B,C) /* No-op */ +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + +/* +** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains +** a rowid value just read from cursor iIdxCur, open on index pIdx. This +** function generates code to do a deferred seek of cursor iCur to the +** rowid stored in register iRowid. +** +** Normally, this is just: +** +** OP_Seek $iCur $iRowid +** +** However, if the scan currently being coded is a branch of an OR-loop and +** the statement currently being coded is a SELECT, then P3 of the OP_Seek +** is set to iIdxCur and P4 is set to point to an array of integers +** containing one entry for each column of the table cursor iCur is open +** on. For each table column, if the column is the i'th column of the +** index, then the corresponding array entry is set to (i+1). If the column +** does not appear in the index at all, the array entry is set to 0. +*/ +static void codeDeferredSeek( + WhereInfo *pWInfo, /* Where clause context */ + Index *pIdx, /* Index scan is using */ + int iCur, /* Cursor for IPK b-tree */ + int iIdxCur /* Index cursor */ +){ + Parse *pParse = pWInfo->pParse; /* Parse context */ + Vdbe *v = pParse->pVdbe; /* Vdbe to generate code within */ + + assert( iIdxCur>0 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); + + sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur); + if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE) + && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) + ){ + int i; + Table *pTab = pIdx->pTable; + int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1)); + if( ai ){ + ai[0] = pTab->nCol; + for(i=0; i<pIdx->nColumn-1; i++){ + assert( pIdx->aiColumn[i]<pTab->nCol ); + if( pIdx->aiColumn[i]>=0 ) ai[pIdx->aiColumn[i]+1] = i+1; + } + sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); + } + } +} /* ** Generate code for the start of the iLevel-th loop in the WHERE clause @@ -118045,14 +121269,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** initialize a memory cell that records if this table matches any ** row of the left table of the join. */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->viaCoroutine ){ + if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); @@ -118069,6 +121293,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; int nConstraint = pLoop->nLTerm; + int iIn; /* Counter for IN constraints */ sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); @@ -118076,7 +121301,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( for(j=0; j<nConstraint; j++){ int iTarget = iReg+j+2; pTerm = pLoop->aLTerm[j]; - if( pTerm==0 ) continue; + if( NEVER(pTerm==0) ) continue; if( pTerm->eOperator & WO_IN ){ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); addrNotFound = pLevel->addrNxt; @@ -118091,14 +121316,50 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); VdbeCoverage(v); pLoop->u.vtab.needFree = 0; - for(j=0; j<nConstraint && j<16; j++){ - if( (pLoop->u.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pLoop->aLTerm[j]); - } - } - pLevel->op = OP_VNext; pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; pLevel->p2 = sqlite3VdbeCurrentAddr(v); + iIn = pLevel->u.in.nIn; + for(j=nConstraint-1; j>=0; j--){ + pTerm = pLoop->aLTerm[j]; + if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pTerm); + }else if( (pTerm->eOperator & WO_IN)!=0 ){ + Expr *pCompare; /* The comparison operator */ + Expr *pRight; /* RHS of the comparison */ + VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + + /* Reload the constraint value into reg[iReg+j+2]. The same value + ** was loaded into the same register prior to the OP_VFilter, but + ** the xFilter implementation might have changed the datatype or + ** encoding of the value in the register, so it *must* be reloaded. */ + assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); + if( !db->mallocFailed ){ + assert( iIn>0 ); + pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop); + assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); + assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); + assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + } + + /* Generate code that will continue to the next row if + ** the IN constraint is not satisfied */ + pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0); + assert( pCompare!=0 || db->mallocFailed ); + if( pCompare ){ + pCompare->pLeft = pTerm->pExpr->pLeft; + pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); + if( pRight ){ + pRight->iTable = iReg+j+2; + sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0); + } + pCompare->pLeft = 0; + sqlite3ExprDelete(db, pCompare); + } + } + } sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); sqlite3ExprCachePop(pParse); }else @@ -118150,6 +121411,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pStart = pEnd; pEnd = pTerm; } + codeCursorHint(pWInfo, pLevel, pEnd); if( pStart ){ Expr *pX; /* The expression that defines the start bound */ int r1, rTemp; /* Registers for holding the start boundary */ @@ -118322,6 +121584,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ assert( pRangeStart!=0 ); /* LIKE opt constraints */ assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ @@ -118334,6 +121597,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "LIKE loop counter")); pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); } +#endif if( pRangeStart==0 && (j = pIdx->aiColumn[nEq])>=0 && pIdx->pTable->aCol[j].notNull==0 @@ -118343,15 +121607,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; - /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). @@ -118363,6 +121618,16 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( SWAP(u8, bSeekPastNull, bStopAtNull); } + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + codeCursorHint(pWInfo, pLevel, pRangeEnd); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); @@ -118403,16 +121668,22 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( start_constraints = 1; } codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); - VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); - VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); - VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ + /* The skip-scan logic inside the call to codeAllEqualityConstraints() + ** above has already left the cursor sitting on the correct row, + ** so no further seeking is needed */ + }else{ + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + } /* Load the value for the inequality constraint at the end of the ** range (if any). @@ -118462,10 +121733,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ + if( pWInfo->eOnePass!=ONEPASS_OFF ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); + VdbeCoverage(v); + }else{ + codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); + } }else if( iCur!=iIdxCur ){ Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); @@ -118633,14 +121909,16 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); + testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); + if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0); } } @@ -118657,7 +121935,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int j1 = 0; /* Address of jump operation */ + int jmp1 = 0; /* Address of jump operation */ if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; @@ -118684,7 +121962,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); if( HasRowid(pTab) ){ r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + r,iSet); VdbeCoverage(v); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); @@ -118695,11 +121974,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPk<nPk; iPk++){ int iCol = pPk->aiColumn[iPk]; - int rx; - rx = sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur,r+iPk,0); - if( rx!=r+iPk ){ - sqlite3VdbeAddOp2(v, OP_SCopy, rx, r+iPk); - } + sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); } /* Check if the temp table already contains this key. If so, @@ -118714,7 +121989,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** need to insert the key into the temp table, as it will never ** be tested for. */ if( iSet ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); VdbeCoverage(v); } if( iSet>=0 ){ @@ -118733,7 +122008,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Jump here (skipping the main loop body subroutine) if the ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( j1 ) sqlite3VdbeJumpHere(v, j1); + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); /* The pSubWInfo->untestedTerms flag means that this OR term ** contained one or more AND term from a notReady table. The @@ -118779,7 +122054,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3ExprDelete(db, pAndExpr); } sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + sqlite3VdbeGoto(v, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); @@ -118794,11 +122069,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - if( pTabItem->isRecursive ){ + if( pTabItem->fg.isRecursive ){ /* Tables marked isRecursive have only a single row that is stored in ** a pseudo-cursor. No need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; }else{ + codeCursorHint(pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); @@ -118833,9 +122109,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( continue; } if( pTerm->wtFlags & TERM_LIKECOND ){ +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + continue; +#else assert( pLevel->iLikeRepCntr>0 ); skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr); VdbeCoverage(v); +#endif } sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); @@ -118971,7 +122251,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, p); @@ -119109,6 +122389,7 @@ static int isLikeOrGlob( sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ + int rc; /* Result code to return */ if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ return 0; @@ -119174,8 +122455,9 @@ static int isLikeOrGlob( } } + rc = (z!=0); sqlite3ValueFree(pVal); - return (z!=0); + return rc; } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -119184,29 +122466,48 @@ static int isLikeOrGlob( /* ** Check to see if the given expression is of the form ** -** column MATCH expr +** column OP expr +** +** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a +** column of a virtual table. ** ** If it is then return TRUE. If not, return FALSE. */ static int isMatchOfColumn( - Expr *pExpr /* Test this expression */ -){ + Expr *pExpr, /* Test this expression */ + unsigned char *peOp2 /* OUT: 0 for MATCH, or else an op2 value */ +){ + struct Op2 { + const char *zOp; + unsigned char eOp2; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; ExprList *pList; + Expr *pCol; /* Column reference */ + int i; if( pExpr->op!=TK_FUNCTION ){ return 0; } - if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ - return 0; - } pList = pExpr->x.pList; - if( pList->nExpr!=2 ){ + if( pList==0 || pList->nExpr!=2 ){ return 0; } - if( pList->a[1].pExpr->op != TK_COLUMN ){ + pCol = pList->a[1].pExpr; + if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ return 0; } - return 1; + for(i=0; i<ArraySize(aOp); i++){ + if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + return 1; + } + } + return 0; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -119419,6 +122720,7 @@ static void exprAnalyzeOrTerm( if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; + memset(pOrWc->aStatic, 0, sizeof(pOrWc->aStatic)); sqlite3WhereClauseInit(pOrWc, pWInfo); sqlite3WhereSplit(pOrWc, pExpr, TK_OR); sqlite3WhereExprAnalyze(pSrc, pOrWc); @@ -119435,7 +122737,7 @@ static void exprAnalyzeOrTerm( WhereAndInfo *pAndInfo; assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + pAndInfo = sqlite3DbMallocRawNN(db, sizeof(*pAndInfo)); if( pAndInfo ){ WhereClause *pAndWC; WhereTerm *pAndTerm; @@ -119445,11 +122747,11 @@ static void exprAnalyzeOrTerm( pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; + memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic)); sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); sqlite3WhereExprAnalyze(pSrc, pAndWC); pAndWC->pOuter = pWC; - testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); @@ -119703,6 +123005,51 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ } /* +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor +** number of the table that is indexed and *piColumn to the column number +** of the column that is indexed, or -2 if an expression is being indexed. +** +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. +*/ +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + Expr *pExpr, /* An operand of a comparison operator */ + int *piCur, /* Write the referenced table cursor number here */ + int *piColumn /* Write the referenced table column number here */ +){ + Index *pIdx; + int i; + int iCur; + if( pExpr->op==TK_COLUMN ){ + *piCur = pExpr->iTable; + *piColumn = pExpr->iColumn; + return 1; + } + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; i<pIdx->nKeyCol; i++){ + if( pIdx->aiColumn[i]!=(-2) ) continue; + if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + *piCur = iCur; + *piColumn = -2; + return 1; + } + } + } + return 0; +} + +/* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the ** subexpression and populate all the other fields of the WhereTerm @@ -119738,6 +123085,7 @@ static void exprAnalyze( int op; /* Top-level operator. pExpr->op */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ + unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ if( db->mallocFailed ){ return; @@ -119772,16 +123120,19 @@ static void exprAnalyze( pTerm->iParent = -1; pTerm->eOperator = 0; if( allowedOp(op) ){ + int iCur, iColumn; Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; + if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){ + pTerm->leftCursor = iCur; + pTerm->u.leftColumn = iColumn; pTerm->eOperator = operatorMask(op) & opMask; } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; - if( pRight && pRight->op==TK_COLUMN ){ + if( pRight + && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn) + ){ WhereTerm *pNew; Expr *pDup; u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ @@ -119809,9 +123160,8 @@ static void exprAnalyze( pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = sqlite3ExprSkipCollate(pDup->pLeft); - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; + pNew->leftCursor = iCur; + pNew->u.leftColumn = iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; @@ -119959,7 +123309,7 @@ static void exprAnalyze( ** virtual tables. The native query optimizer does not attempt ** to do anything with MATCH functions. */ - if( isMatchOfColumn(pExpr) ){ + if( isMatchOfColumn(pExpr, &eOp2) ){ int idxNew; Expr *pRight, *pLeft; WhereTerm *pNewTerm; @@ -119980,6 +123330,7 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; + pNewTerm->eMatchOp = eOp2; markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; pTerm->wtFlags |= TERM_COPIED; @@ -120082,7 +123433,8 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( /* ** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of sqlite3WhereClauseInit(). +** itself is not freed. This routine is the inverse of +** sqlite3WhereClauseInit(). */ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ int i; @@ -120155,6 +123507,46 @@ SQLITE_PRIVATE void sqlite3WhereExprAnalyze( } } +/* +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. +** +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + struct SrcList_item *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + if( pArgs==0 ) return; + for(j=k=0; j<pArgs->nExpr; j++){ + while( k<pTab->nCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; + } + pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pColRef->pTab = pTab; + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); + } +} + /************** End of whereexpr.c *******************************************/ /************** Begin file where.c *******************************************/ /* @@ -120190,8 +123582,8 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); /* ** Return the estimated number of output rows from a WHERE clause */ -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return sqlite3LogEstToInt(pWInfo->nRowOut); +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return pWInfo->nRowOut; } /* @@ -120228,9 +123620,11 @@ SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ } /* -** Return TRUE if an UPDATE or DELETE statement can operate directly on -** the rowids returned by a WHERE clause. Return FALSE if doing an -** UPDATE or DELETE might change subsequent WHERE clause results. +** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to +** operate directly on the rowis returned by a WHERE clause. Return +** ONEPASS_SINGLE (1) if the statement can operation directly because only +** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass +** optimization can be used on multiple ** ** If the ONEPASS optimization is used (if this routine returns true) ** then also write the indices of open cursors used by ONEPASS @@ -120244,7 +123638,14 @@ SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ */ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); - return pWInfo->okOnePass; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); + } +#endif + return pWInfo->eOnePass; } /* @@ -120330,37 +123731,39 @@ static void createMask(WhereMaskSet *pMaskSet, int iCursor){ */ static WhereTerm *whereScanNext(WhereScan *pScan){ int iCur; /* The cursor on the LHS of the term */ - int iColumn; /* The column on the LHS of the term. -1 for IPK */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ Expr *pX; /* An expression being tested */ WhereClause *pWC; /* Shorthand for pScan->pWC */ WhereTerm *pTerm; /* The term being tested */ int k = pScan->k; /* Where to start scanning */ while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aEquiv[pScan->iEquiv-2]; - iColumn = pScan->aEquiv[pScan->iEquiv-1]; + iCur = pScan->aiCur[pScan->iEquiv-1]; + iColumn = pScan->aiColumn[pScan->iEquiv-1]; + if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn - && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (iColumn!=XN_EXPR + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquiv<ArraySize(pScan->aEquiv) + && pScan->nEquiv<ArraySize(pScan->aiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN ){ int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); - for(j=0; j<pScan->nEquiv; j+=2){ - if( pScan->aEquiv[j]==pX->iTable - && pScan->aEquiv[j+1]==pX->iColumn ){ + for(j=0; j<pScan->nEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ break; } } if( j==pScan->nEquiv ){ - pScan->aEquiv[j] = pX->iTable; - pScan->aEquiv[j+1] = pX->iColumn; - pScan->nEquiv += 2; + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; } } if( (pTerm->eOperator & pScan->opMask)!=0 ){ @@ -120382,8 +123785,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ } if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aEquiv[0] - && pX->iColumn==pScan->aEquiv[1] + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] ){ testcase( pTerm->eOperator & WO_IS ); continue; @@ -120398,7 +123801,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ } pScan->pWC = pScan->pOrigWC; k = 0; - pScan->iEquiv += 2; + pScan->iEquiv++; } return 0; } @@ -120427,16 +123830,20 @@ static WhereTerm *whereScanInit( u32 opMask, /* Operator(s) to scan for */ Index *pIdx /* Must be compatible with this index */ ){ - int j; + int j = 0; /* memset(pScan, 0, sizeof(*pScan)); */ pScan->pOrigWC = pWC; pScan->pWC = pWC; + pScan->pIdxExpr = 0; + if( pIdx ){ + j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID; + } if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>pIdx->nColumn) ) return 0; - } pScan->zCollName = pIdx->azColl[j]; }else{ pScan->idxaff = 0; @@ -120444,10 +123851,10 @@ static WhereTerm *whereScanInit( } pScan->opMask = opMask; pScan->k = 0; - pScan->aEquiv[0] = iCur; - pScan->aEquiv[1] = iColumn; - pScan->nEquiv = 2; - pScan->iEquiv = 2; + pScan->aiCur[0] = iCur; + pScan->aiColumn[0] = iColumn; + pScan->nEquiv = 1; + pScan->iEquiv = 1; return whereScanNext(pScan); } @@ -120457,15 +123864,16 @@ static WhereTerm *whereScanInit( ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. ** +** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** ** The term returned might by Y=<expr> if there is another constraint in ** the WHERE clause that specifies that X=Y. Any such constraints will be ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aEquiv[] array holds X and all its equivalents, with each SQL variable -** taking up two slots in aEquiv[]. The first slot is for the cursor number -** and the second is for the column number. There are 22 slots in aEquiv[] -** so that means we can look for X plus up to 10 other equivalent values. -** Hence a search for X will return <expr> if X=A1 and A1=A2 and A2=A3 -** and ... and A9=A10 and A10=<expr>. +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return <expr> if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>. ** ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>" ** then try for the one with no dependencies on <expr> - in other words where @@ -120535,6 +123943,25 @@ static int findIndexCol( } /* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iCol<pIdx->nColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return 0; /* Assume an indexed expression can always yield a NULL */ + + } +} + +/* ** Return true if the DISTINCT expression-list passed as the third argument ** is redundant. ** @@ -120584,12 +124011,9 @@ static int isDistinctRedundant( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !IsUniqueIndex(pIdx) ) continue; for(i=0; i<pIdx->nKeyCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - if( 0==sqlite3WhereFindTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); - if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ - break; - } + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; } } if( i==pIdx->nKeyCol ){ @@ -120613,14 +124037,20 @@ static LogEst estLog(LogEst N){ ** Convert OP_Column opcodes to OP_Copy in previously generated code. ** ** This routine runs over generated VDBE code and translates OP_Column -** opcodes into OP_Copy, and OP_Rowid into OP_Null, when the table is being -** accessed via co-routine instead of via table lookup. +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, +** then each OP_Rowid is transformed into an instruction to increment the +** value stored in its output register. */ static void translateColumnToCopy( Vdbe *v, /* The VDBE containing code to translate */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ - int iRegister /* The first column is in this register */ + int iRegister, /* The first column is in this register */ + int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ ){ VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); @@ -120632,9 +124062,16 @@ static void translateColumnToCopy( pOp->p2 = pOp->p3; pOp->p3 = 0; }else if( pOp->opcode==OP_Rowid ){ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; + if( bIncrRowid ){ + /* Increment the value stored in the P2 operand of the OP_Rowid. */ + pOp->opcode = OP_AddImm; + pOp->p1 = pOp->p2; + pOp->p2 = 1; + }else{ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } } } } @@ -120742,6 +124179,8 @@ static void constructAutomaticIndex( Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + int addrCounter = 0; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -120831,7 +124270,7 @@ static void constructAutomaticIndex( idxCols |= cMask; pIdx->aiColumn[n] = pTerm->u.leftColumn; pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = pColl ? pColl->zName : "BINARY"; + pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; } } @@ -120843,20 +124282,20 @@ static void constructAutomaticIndex( for(i=0; i<mxBitCol; i++){ if( extraCols & MASKBIT(i) ){ pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; + pIdx->azColl[n] = sqlite3StrBINARY; n++; } } if( pSrc->colUsed & MASKBIT(BMS-1) ){ for(i=BMS-1; i<pTable->nCol; i++){ pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; + pIdx->azColl[n] = sqlite3StrBINARY; n++; } } assert( n==nKeyCol ); - pIdx->aiColumn[n] = -1; - pIdx->azColl[n] = "BINARY"; + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ assert( pLevel->iIdxCur>=0 ); @@ -120868,8 +124307,9 @@ static void constructAutomaticIndex( /* Fill the automatic index with content */ sqlite3ExprCachePush(pParse); pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->viaCoroutine ){ + if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); @@ -120883,14 +124323,17 @@ static void constructAutomaticIndex( pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->viaCoroutine ){ - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - pTabItem->viaCoroutine = 0; + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + sqlite3VdbeGoto(v, addrTop); + pTabItem->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); } @@ -120941,6 +124384,7 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); nTerm++; } @@ -120996,10 +124440,14 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; op = (u8)pTerm->eOperator & WO_ALL; if( op==WO_IN ) op = WO_EQ; + if( op==WO_MATCH ){ + op = pTerm->eMatchOp; + } pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The @@ -121038,7 +124486,6 @@ static sqlite3_index_info *allocateIndexInfo( */ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int i; int rc; TRACE_IDX_INPUTS(p); @@ -121047,7 +124494,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; + sqlite3OomFault(pParse->db); }else if( !pVtab->zErrMsg ){ sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); }else{ @@ -121057,12 +124504,16 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; +#if 0 + /* This error is now caught by the caller. + ** Search for "xBestIndex malfunction" below */ for(i=0; i<p->nConstraint; i++){ if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ sqlite3ErrorMsg(pParse, "table %s: xBestIndex returned an invalid plan", pTab->zName); } } +#endif return pParse->nErr; } @@ -121286,6 +124737,21 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ return nRet; } + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return the affinity for a single column of an index. +*/ +static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iCol<pIdx->nColumn ); + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + return pIdx->zColAff[iCol]; +} +#endif + + #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** This function is called to estimate the number of rows visited by a @@ -121335,8 +124801,7 @@ static int whereRangeSkipScanEst( int nLower = -1; int nUpper = p->nSample+1; int rc = SQLITE_OK; - int iCol = p->aiColumn[nEq]; - u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); CollSeq *pColl; sqlite3_value *p1 = 0; /* Value extracted from pLower */ @@ -121484,11 +124949,8 @@ static int whereRangeScanEst( testcase( pRec->nField!=pBuilder->nRecValid ); pRec->nField = pBuilder->nRecValid; } - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq); + assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER ); /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; @@ -121646,7 +125108,7 @@ static int whereEqualScanEst( return SQLITE_OK; } - aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; @@ -121654,7 +125116,8 @@ static int whereEqualScanEst( pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; return rc; @@ -121828,8 +125291,8 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ WhereTerm **paNew; if( p->nLSlot>=n ) return SQLITE_OK; n = (n+7)&~7; - paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); - if( paNew==0 ) return SQLITE_NOMEM; + paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM_BKPT; memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); p->aLTerm = paNew; @@ -121844,7 +125307,7 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ memset(&pTo->u, 0, sizeof(pTo->u)); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); @@ -122068,23 +125531,26 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p; WhereInfo *pWInfo = pBuilder->pWInfo; sqlite3 *db = pWInfo->pParse->db; + int rc; /* If pBuilder->pOrSet is defined, then only keep track of the costs ** and prereqs. */ if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ #if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = + u16 n = pBuilder->pOrSet->n; + int x = #endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, pTemplate->nOut); #if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } #endif + } return SQLITE_OK; } @@ -122123,8 +125589,8 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #endif if( p==0 ){ /* Allocate a new WhereLoop to add to the end of the list */ - *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); - if( p==0 ) return SQLITE_NOMEM; + *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM_BKPT; whereLoopInit(p); p->pNextLoop = 0; }else{ @@ -122148,14 +125614,14 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ whereLoopDelete(db, pToDel); } } - whereLoopXfer(db, p, pTemplate); + rc = whereLoopXfer(db, p, pTemplate); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } } - return SQLITE_OK; + return rc; } /* @@ -122274,37 +125740,33 @@ static int whereLoopAddBtreeIndex( u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ - int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; - }else if( /*pProbe->tnum<=0 ||*/ (pSrc->jointype & JT_LEFT)!=0 ){ - opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); assert( pNew->u.btree.nEq<pProbe->nColumn ); - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, pProbe); saved_nEq = pNew->u.btree.nEq; saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); @@ -122317,7 +125779,7 @@ static int whereLoopAddBtreeIndex( int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + && indexColumnNotNull(pProbe, saved_nEq) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } @@ -122327,6 +125789,18 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; + /* Do not allow IS constraints from the WHERE clause to be used by the + ** right table of a LEFT JOIN. Only constraints in the ON clause are + ** allowed */ + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + && (eOp & (WO_IS|WO_ISNULL))!=0 + ){ + testcase( eOp & WO_IS ); + testcase( eOp & WO_ISNULL ); + continue; + } + pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; @@ -122354,8 +125828,12 @@ static int whereLoopAddBtreeIndex( ** changes "x IN (?)" into "x=?". */ }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==XN_ROWID + || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ @@ -122406,7 +125884,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && iCol>=0 ){ + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; @@ -122541,18 +126019,25 @@ static int indexMightHelpWithOrderBy( int iCursor ){ ExprList *pOB; + ExprList *aColExpr; int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; ii<pOB->nExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ) return 0; - if( pExpr->iTable==iCursor ){ + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jj<pIndex->nKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jj<pIndex->nKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } + } } } return 0; @@ -122582,6 +126067,10 @@ static Bitmask columnsInIndex(Index *pIdx){ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ int i; WhereTerm *pTerm; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ Expr *pExpr = pTerm->pExpr; if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) @@ -122631,7 +126120,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra /* Extra prerequesites for using this table */ + Bitmask mPrereq /* Extra prerequesites for using this table */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ @@ -122657,9 +126146,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIndex ){ + if( pSrc->pIBIndex ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIndex; + pProbe = pSrc->pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -122679,7 +126168,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ + if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ sPk.pNext = pFirst; @@ -122691,14 +126180,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ - if( !pBuilder->pOrSet /* Not part of an OR optimization */ + if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIndex==0 /* Has no INDEXED BY clause */ - && !pSrc->notIndexed /* Has no NOT INDEXED clause */ - && HasRowid(pTab) /* Is not a WITHOUT ROWID table. (FIXME: Why not?) */ - && !pSrc->isCorrelated /* Not a correlated subquery */ - && !pSrc->isRecursive /* Not a recursive common table expression. */ + && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -122731,7 +126220,7 @@ static int whereLoopAddBtree( pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_AUTO_INDEX; - pNew->prereq = mExtra | pTerm->prereqRight; + pNew->prereq = mPrereq | pTerm->prereqRight; rc = whereLoopInsert(pBuilder, pNew); } } @@ -122752,7 +126241,7 @@ static int whereLoopAddBtree( pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; - pNew->prereq = mExtra; + pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); @@ -122819,18 +126308,166 @@ static int whereLoopAddBtree( /* If there was an INDEXED BY clause, then only that one index is ** considered. */ - if( pSrc->pIndex ) break; + if( pSrc->pIBIndex ) break; } return rc; } #ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Argument pIdxInfo is already populated with all constraints that may +** be used by the virtual table identified by pBuilder->pNew->iTab. This +** function marks a subset of those constraints usable, invokes the +** xBestIndex method and adds the returned plan to pBuilder. +** +** A constraint is marked usable if: +** +** * Argument mUsable indicates that its prerequisites are available, and +** +** * It is not one of the operators specified in the mExclude mask passed +** as the fourth argument (which in practice is either WO_IN or 0). +** +** Argument mPrereq is a mask of tables that must be scanned before the +** virtual table in question. These are added to the plans prerequisites +** before it is added to pBuilder. +** +** Output parameter *pbIn is set to true if the plan added to pBuilder +** uses one or more WO_IN terms, or false otherwise. +*/ +static int whereLoopAddVirtualOne( + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, /* Mask of tables that must be used. */ + Bitmask mUsable, /* Mask of usable tables */ + u16 mExclude, /* Exclude terms using these operators */ + sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ + int *pbIn /* OUT: True if plan uses an IN(...) op */ +){ + WhereClause *pWC = pBuilder->pWC; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; + int i; + int mxTerm; + int rc = SQLITE_OK; + WhereLoop *pNew = pBuilder->pNew; + Parse *pParse = pBuilder->pWInfo->pParse; + struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; + int nConstraint = pIdxInfo->nConstraint; + + assert( (mUsable & mPrereq)==mPrereq ); + *pbIn = 0; + pNew->prereq = mPrereq; + + /* Set the usable flag on the subset of constraints identified by + ** arguments mUsable and mExclude. */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i<nConstraint; i++, pIdxCons++){ + WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset]; + pIdxCons->usable = 0; + if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight + && (pTerm->eOperator & mExclude)==0 + ){ + pIdxCons->usable = 1; + } + } + + /* Initialize the output fields of the sqlite3_index_info structure */ + memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); + assert( pIdxInfo->needToFreeIdxStr==0 ); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + + /* Invoke the virtual table xBestIndex() method */ + rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + if( rc ) return rc; + + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; i<nConstraint; i++) pNew->aLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i<nConstraint; i++, pIdxCons++){ + int iTerm; + if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ + WhereTerm *pTerm; + int j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + || pIdxCons->usable==0 + ){ + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + return rc; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTerm<pNew->nLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm; + if( (pTerm->eOperator & WO_IN)!=0 ){ + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; + *pbIn = 1; assert( (mExclude & WO_IN)==0 ); + } + } + } + + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } + rc = whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + *pbIn, (sqlite3_uint64)mPrereq, + (sqlite3_uint64)(pNew->prereq & ~mPrereq))); + + return rc; +} + + /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. ** -** If there are no LEFT or CROSS JOIN joins in the query, both mExtra and -** mUnusable are set to 0. Otherwise, mExtra is a mask of all FROM clause +** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and +** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause ** entries that occur before the virtual table in the FROM clause and are ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the ** mUnusable mask contains all FROM clause entries that occur after the @@ -122841,177 +126478,122 @@ static int whereLoopAddBtree( ** ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; ** -** then mExtra corresponds to (t1, t2) and mUnusable to (t5, t6). +** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). ** -** All the tables in mExtra must be scanned before the current virtual +** All the tables in mPrereq must be scanned before the current virtual ** table. So any terms for which all prerequisites are satisfied by -** mExtra may be specified as "usable" in all calls to xBestIndex. +** mPrereq may be specified as "usable" in all calls to xBestIndex. ** Conversely, all tables in mUnusable must be scanned after the current ** virtual table, so any terms for which the prerequisites overlap with ** mUnusable should always be configured as "not-usable" for xBestIndex. */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra, /* Tables that must be scanned before this one */ + Bitmask mPrereq, /* Tables that must be scanned before this one */ Bitmask mUnusable /* Tables that must be scanned after this one */ ){ + int rc = SQLITE_OK; /* Return code */ WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ WhereClause *pWC; /* The WHERE clause */ struct SrcList_item *pSrc; /* The FROM clause term to search */ - Table *pTab; - sqlite3 *db; - sqlite3_index_info *pIdxInfo; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int iTerm, mxTerm; - int nConstraint; - int seenIn = 0; /* True if an IN operator is seen */ - int seenVar = 0; /* True if a non-constant constraint is seen */ - int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + sqlite3_index_info *p; /* Object to pass to xBestIndex() */ + int nConstraint; /* Number of constraints in p */ + int bIn; /* True if plan uses IN(...) operator */ WhereLoop *pNew; - int rc = SQLITE_OK; + Bitmask mBest; /* Tables used by best possible plan */ - assert( (mExtra & mUnusable)==0 ); + assert( (mPrereq & mUnusable)==0 ); pWInfo = pBuilder->pWInfo; pParse = pWInfo->pParse; - db = pParse->db; pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - pTab = pSrc->pTab; - assert( IsVirtual(pTab) ); - pIdxInfo = allocateIndexInfo(pParse, pWC, mUnusable, pSrc,pBuilder->pOrderBy); - if( pIdxInfo==0 ) return SQLITE_NOMEM; - pNew->prereq = 0; + assert( IsVirtual(pSrc->pTab) ); + p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy); + if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; pNew->nLTerm = 0; pNew->u.vtab.needFree = 0; - pUsage = pIdxInfo->aConstraintUsage; - nConstraint = pIdxInfo->nConstraint; - if( whereLoopResize(db, pNew, nConstraint) ){ - sqlite3DbFree(db, pIdxInfo); - return SQLITE_NOMEM; - } - - for(iPhase=0; iPhase<=3; iPhase++){ - if( !seenIn && (iPhase&1)!=0 ){ - iPhase++; - if( iPhase>3 ) break; - } - if( !seenVar && iPhase>1 ) break; - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - switch( iPhase ){ - case 0: /* Constants without IN operator */ - pIdxCons->usable = 0; - if( (pTerm->eOperator & WO_IN)!=0 ){ - seenIn = 1; - } - if( (pTerm->prereqRight & ~mExtra)!=0 ){ - seenVar = 1; - }else if( (pTerm->eOperator & WO_IN)==0 ){ - pIdxCons->usable = 1; - } - break; - case 1: /* Constants with IN operators */ - assert( seenIn ); - pIdxCons->usable = (pTerm->prereqRight & ~mExtra)==0; - break; - case 2: /* Variables without IN */ - assert( seenVar ); - pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; - break; - default: /* Variables with IN */ - assert( seenVar && seenIn ); - pIdxCons->usable = 1; - break; + nConstraint = p->nConstraint; + if( whereLoopResize(pParse->db, pNew, nConstraint) ){ + sqlite3DbFree(pParse->db, p); + return SQLITE_NOMEM_BKPT; + } + + /* First call xBestIndex() with all constraints usable. */ + WHERETRACE(0x40, (" VirtualOne: all usable\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, &bIn); + + /* If the call to xBestIndex() with all terms enabled produced a plan + ** that does not require any source tables (IOW: a plan with mBest==0), + ** then there is no point in making any further calls to xBestIndex() + ** since they will all return the same result (if the xBestIndex() + ** implementation is sane). */ + if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ + int seenZero = 0; /* True if a plan with no prereqs seen */ + int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ + Bitmask mPrev = 0; + Bitmask mBestNoIn = 0; + + /* If the plan produced by the earlier call uses an IN(...) term, call + ** xBestIndex again, this time with IN(...) terms disabled. */ + if( bIn ){ + WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, WO_IN, p, &bIn); + assert( bIn==0 ); + mBestNoIn = pNew->prereq & ~mPrereq; + if( mBestNoIn==0 ){ + seenZero = 1; + seenZeroNoIN = 1; + } + } + + /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) + ** in the set of terms that apply to the current virtual table. */ + while( rc==SQLITE_OK ){ + int i; + Bitmask mNext = ALLBITS; + assert( mNext>0 ); + for(i=0; i<nConstraint; i++){ + Bitmask mThis = ( + pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq + ); + if( mThis>mPrev && mThis<mNext ) mNext = mThis; } - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; - pIdxInfo->estimatedRows = 25; - rc = vtabBestIndex(pParse, pTab, pIdxInfo); - if( rc ) goto whereLoopAddVtab_exit; - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pNew->prereq = mExtra; - mxTerm = -1; - assert( pNew->nLSlot>=nConstraint ); - for(i=0; i<nConstraint; i++) pNew->aLTerm[i] = 0; - pNew->u.vtab.omitMask = 0; - for(i=0; i<nConstraint; i++, pIdxCons++){ - if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ - j = pIdxCons->iTermOffset; - if( iTerm>=nConstraint - || j<0 - || j>=pWC->nTerm - || pNew->aLTerm[iTerm]!=0 - ){ - rc = SQLITE_ERROR; - sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); - goto whereLoopAddVtab_exit; - } - testcase( iTerm==nConstraint-1 ); - testcase( j==0 ); - testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; - pNew->prereq |= pTerm->prereqRight; - assert( iTerm<pNew->nLSlot ); - pNew->aLTerm[iTerm] = pTerm; - if( iTerm>mxTerm ) mxTerm = iTerm; - testcase( iTerm==15 ); - testcase( iTerm==16 ); - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm; - if( (pTerm->eOperator & WO_IN)!=0 ){ - if( pUsage[i].omit==0 ){ - /* Do not attempt to use an IN constraint if the virtual table - ** says that the equivalent EQ constraint cannot be safely omitted. - ** If we do attempt to use such a constraint, some rows might be - ** repeated in the output. */ - break; - } - /* A virtual table that is constrained by an IN clause may not - ** consume the ORDER BY clause because (1) the order of IN terms - ** is not necessarily related to the order of output terms and - ** (2) Multiple outputs from a single IN value will not merge - ** together. */ - pIdxInfo->orderByConsumed = 0; - } - } - } - if( i>=nConstraint ){ - pNew->nLTerm = mxTerm+1; - assert( pNew->nLTerm<=pNew->nLSlot ); - pNew->u.vtab.idxNum = pIdxInfo->idxNum; - pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; - pIdxInfo->needToFreeIdxStr = 0; - pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? - pIdxInfo->nOrderBy : 0); - pNew->rSetup = 0; - pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); - pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); - whereLoopInsert(pBuilder, pNew); - if( pNew->u.vtab.needFree ){ - sqlite3_free(pNew->u.vtab.idxStr); - pNew->u.vtab.needFree = 0; + mPrev = mNext; + if( mNext==ALLBITS ) break; + if( mNext==mBest || mNext==mBestNoIn ) continue; + WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", + (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mNext|mPrereq, 0, p, &bIn); + if( pNew->prereq==mPrereq ){ + seenZero = 1; + if( bIn==0 ) seenZeroNoIN = 1; } } - } -whereLoopAddVtab_exit: - if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); - sqlite3DbFree(db, pIdxInfo); + /* If the calls to xBestIndex() in the above loop did not find a plan + ** that requires no source tables at all (i.e. one guaranteed to be + ** usable), make a call here with all source tables disabled */ + if( rc==SQLITE_OK && seenZero==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, 0, p, &bIn); + if( bIn==0 ) seenZeroNoIN = 1; + } + + /* If the calls to xBestIndex() have so far failed to find a plan + ** that requires no source tables at all and does not use an IN(...) + ** operator, make a final call to obtain one here. */ + if( rc==SQLITE_OK && seenZeroNoIN==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, WO_IN, p, &bIn); + } + } + + if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); + sqlite3DbFree(pParse->db, p); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -123022,7 +126604,7 @@ whereLoopAddVtab_exit: */ static int whereLoopAddOr( WhereLoopBuilder *pBuilder, - Bitmask mExtra, + Bitmask mPrereq, Bitmask mUnusable ){ WhereInfo *pWInfo = pBuilder->pWInfo; @@ -123083,14 +126665,14 @@ static int whereLoopAddOr( #endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mExtra, mUnusable); + rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif { - rc = whereLoopAddBtree(&sSubBuild, mExtra); + rc = whereLoopAddBtree(&sSubBuild, mPrereq); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(&sSubBuild, mExtra, mUnusable); + rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ @@ -123147,7 +126729,7 @@ static int whereLoopAddOr( */ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ WhereInfo *pWInfo = pBuilder->pWInfo; - Bitmask mExtra = 0; + Bitmask mPrereq = 0; Bitmask mPrior = 0; int iTab; SrcList *pTabList = pWInfo->pTabList; @@ -123165,25 +126747,25 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ Bitmask mUnusable = 0; pNew->iTab = iTab; pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ + if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ /* This condition is true when pItem is the FROM clause term on the ** right-hand-side of a LEFT or CROSS JOIN. */ - mExtra = mPrior; + mPrereq = mPrior; } - priorJointype = pItem->jointype; + priorJointype = pItem->fg.jointype; if( IsVirtual(pItem->pTab) ){ struct SrcList_item *p; for(p=&pItem[1]; p<pEnd; p++){ - if( mUnusable || (p->jointype & (JT_LEFT|JT_CROSS)) ){ + if( mUnusable || (p->fg.jointype & (JT_LEFT|JT_CROSS)) ){ mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); } } - rc = whereLoopAddVirtual(pBuilder, mExtra, mUnusable); + rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); }else{ - rc = whereLoopAddBtree(pBuilder, mExtra); + rc = whereLoopAddBtree(pBuilder, mPrereq); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(pBuilder, mExtra, mUnusable); + rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; @@ -123322,7 +126904,8 @@ static i8 wherePathSatisfiesOrderBy( nKeyCol = pIndex->nKeyCol; nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); isOrderDistinct = IsUniqueIndex(pIndex); } @@ -123354,7 +126937,7 @@ static i8 wherePathSatisfiesOrderBy( revIdx = pIndex->aSortOrder[j]; if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - iColumn = -1; + iColumn = XN_ROWID; revIdx = 0; } @@ -123380,9 +126963,15 @@ static i8 wherePathSatisfiesOrderBy( testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=(-1) ){ + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + continue; + } + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -123519,15 +127108,14 @@ static LogEst whereSortingCost( LogEst rScale, rSortCost; assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + estLog(nRow) + rScale + 16; + rSortCost = nRow + rScale + 16; - /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** similar but with a larger constant of proportionality. - ** Multiply by an additional factor of 3.0. */ - if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - rSortCost += 16; + /* Multiple by log(M) where M is the number of output rows. + ** Use the LIMIT for M if it is smaller */ + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){ + nRow = pWInfo->iLimit; } - + rSortCost += estLog(nRow); return rSortCost; } @@ -123589,8 +127177,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRaw(db, nSpace); - if( pSpace==0 ) return SQLITE_NOMEM; + pSpace = sqlite3DbMallocRawNN(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; memset(aFrom, 0, sizeof(aFrom[0])); @@ -123645,6 +127233,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){ + /* Do not use an automatic index if the this loop is expected + ** to run less than 2 times. */ + assert( 10==sqlite3LogEst(2) ); + continue; + } /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); @@ -123897,14 +127491,14 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ int j; Table *pTab; Index *pIdx; - + pWInfo = pBuilder->pWInfo; if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->zIndexedBy ) return 0; + if( pItem->fg.isIndexedBy ) return 0; iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; @@ -123929,7 +127523,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ) continue; opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; j<pIdx->nKeyCol; j++){ - pTerm = sqlite3WhereFindTerm(pWC, iCur, pIdx->aiColumn[j], 0, opMask, pIdx); + pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); if( pTerm==0 ) break; testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; @@ -124060,7 +127654,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ + int iAuxArg /* If WHERE_ONETABLE_ONLY is set, index cursor number, + ** If WHERE_USE_LIMIT, then the limit amount */ ){ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ @@ -124074,7 +127669,16 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ int rc; /* Return code */ + u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ + + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + )); + /* Only one of WHERE_ONETABLE_ONLY or WHERE_USE_LIMIT */ + assert( (wctrlFlags & WHERE_ONETABLE_ONLY)==0 + || (wctrlFlags & WHERE_USE_LIMIT)==0 ); /* Variable initialization */ db = pParse->db; @@ -124129,7 +127733,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pResultSet = pResultSet; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; + pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; @@ -124169,14 +127775,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Assign a bit from the bitmask to every term in the FROM clause. ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. + ** The N-th term of the FROM clause is assigned a bitmask of 1<<N. + ** + ** The rule of the previous sentence ensures thta if X is the bitmask for + ** a table T, then X-1 is the bitmask for all other tables to the left of T. + ** Knowing the bitmask for all tables to the left of a left join is + ** important. Ticket #3015. ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally @@ -124185,15 +127789,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ for(ii=0; ii<pTabList->nSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(ii=0; ii<pTabList->nSrc; ii++){ - Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; - } +#ifdef SQLITE_DEBUG + for(ii=0; ii<pTabList->nSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m==MASKBIT(ii) ); } #endif @@ -124213,8 +127814,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start ***\n")); #if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0xffff ){ + sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); + if( wctrlFlags & WHERE_USE_LIMIT ){ + sqlite3DebugPrintf(", limit: %d", iAuxArg); + } + sqlite3DebugPrintf(")\n"); + } if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ int i; for(i=0; i<sWLB.pWC->nTerm; i++){ @@ -124248,7 +127855,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } } if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = (Bitmask)(-1); + pWInfo->revMask = ALLBITS; } if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; @@ -124291,7 +127898,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( while( pWInfo->nLevel>=2 ){ WhereTerm *pTerm, *pEnd; pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ @@ -124317,15 +127924,22 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. - ** The one-pass algorithm only works if the WHERE clause constrains - ** the statement to update or delete a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ - pWInfo->okOnePass = 1; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + if( bOnerow + || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0 + && 0==(wsFlags & WHERE_VIRTUALTABLE)) + ){ + pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; + if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ + bFordelete = OPFLAG_FORDELETE; + } + pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); + } } } @@ -124356,22 +127970,29 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = OP_OpenRead; - if( pWInfo->okOnePass ){ + if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; pWInfo->aiCurOnePass[0] = pTabItem->iCursor; }; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); assert( pTabItem->iCursor==pLevel->iTabCur ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nCol<BMS && HasRowid(pTab) ){ + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); + if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){ Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, - SQLITE_INT_TO_PTR(n), P4_INT32); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + if( pLoop->u.btree.pIndex!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); + }else +#endif + { + sqlite3VdbeChangeP5(v, bFordelete); + } #ifdef SQLITE_ENABLE_COLUMN_USED_MASK sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); @@ -124383,8 +128004,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Index *pIx = pLoop->u.btree.pIndex; int iIndexCur; int op = OP_OpenRead; - /* iIdxCur is always set if to a positive value if ONEPASS is possible */ - assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + /* iAuxArg is always set if to a positive value if ONEPASS is possible */ + assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ @@ -124392,9 +128013,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** WITHOUT ROWID table. No need for a separate index */ iIndexCur = pLevel->iTabCur; op = 0; - }else if( pWInfo->okOnePass ){ + }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ Index *pJ = pTabItem->pTab->pIndex; - iIndexCur = iIdxCur; + iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ iIndexCur++; @@ -124402,8 +128023,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } op = OP_OpenWrite; pWInfo->aiCurOnePass[1] = iIndexCur; - }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ - iIndexCur = iIdxCur; + }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + iIndexCur = iAuxArg; if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; @@ -124529,11 +128150,12 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( pLevel->addrLikeRep ){ int op; if( sqlite3VdbeGetOp(v, pLevel->addrLikeRep-1)->p1 ){ @@ -124544,6 +128166,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, op, pLevel->iLikeRepCntr, pLevel->addrLikeRep); VdbeCoverage(v); } +#endif if( pLevel->iLeftJoin ){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 @@ -124557,7 +128180,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLevel->op==OP_Return ){ sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + sqlite3VdbeGoto(v, pLevel->addrFirst); } sqlite3VdbeJumpHere(v, addr); } @@ -124584,9 +128207,9 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine && !db->mallocFailed ){ + if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult); + pTabItem->regResult, 0); continue; } @@ -124600,7 +128223,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int ws = pLoop->wsFlags; - if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } if( (ws & WHERE_INDEXED)!=0 @@ -124627,7 +128250,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed ){ + if( pIdx + && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) + && !db->mallocFailed + ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); @@ -124639,6 +128265,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; + assert( x>=0 ); } x = sqlite3ColumnOfIndex(pIdx, x); if( x>=0 ){ @@ -124663,18 +128290,32 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ /************** End of where.c ***********************************************/ /************** Begin file parse.c *******************************************/ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. +/* +** 2000-05-29 ** -** This version of "lempar.c" is modified, slightly, for use by SQLite. -** The only modifications are the addition of a couple of NEVER() -** macros to disable tests that are needed in the case of a general -** LALR(1) grammar but which are always false in the -** specific grammar used by SQLite. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: */ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ /* #include <stdio.h> */ +/************ Begin %include sections from the grammar ************************/ /* #include "sqliteInt.h" */ @@ -124690,6 +128331,18 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ #define yytestcase(X) testcase(X) /* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + +/* ** An instance of this structure holds information about the ** LIMIT clause of a SELECT statement. */ @@ -124723,6 +128376,15 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; +/* +** Disable lookaside memory allocation for objects that might be +** shared across database connections. +*/ +static void disableLookaside(Parse *pParse){ + pParse->disableLookaside++; + pParse->db->lookaside.bDisable++; +} + /* ** For a compound SELECT statement, make sure p->pPrior->pNext==p for @@ -124759,46 +128421,51 @@ struct AttachKey { int type; Token key; }; ** new Expr to populate pOut. Set the span of pOut to be the identifier ** that created the expression. */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); - pOut->zStart = pValue->z; - pOut->zEnd = &pValue->z[pValue->n]; + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ + pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, &t); + pOut->zStart = t.z; + pOut->zEnd = &t.z[t.n]; } /* This routine constructs a binary expression node out of two ExprSpan ** objects and uses the result to populate a new ExprSpan object. */ static void spanBinaryExpr( - ExprSpan *pOut, /* Write the result here */ Parse *pParse, /* The parsing context. Errors accumulate here */ int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand */ + ExprSpan *pLeft, /* The left operand, and output */ ExprSpan *pRight /* The right operand */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pOut->zStart = pLeft->zStart; - pOut->zEnd = pRight->zEnd; + pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pLeft->zEnd = pRight->zEnd; + } + + /* If doNot is true, then add a TK_NOT Expr-node wrapper around the + ** outside of *ppExpr. + */ + static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ + if( doNot ){ + pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); + } } /* Construct an expression node for a unary postfix operator */ static void spanUnaryPostfix( - ExprSpan *pOut, /* Write the new expression node here */ Parse *pParse, /* Parsing context to record errors */ int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ + ExprSpan *pOperand, /* The operand, and output */ Token *pPostOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pOperand->zStart; - pOut->zEnd = &pPostOp->z[pPostOp->n]; + pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOperand->zEnd = &pPostOp->z[pPostOp->n]; } /* A routine to convert a binary TK_IS or TK_ISNOT expression into a ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( pY && pA && pY->op==TK_NULL ){ + if( pA && pY && pY->op==TK_NULL ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; @@ -124814,82 +128481,112 @@ struct AttachKey { int type; Token key; }; ExprSpan *pOperand, /* The operand */ Token *pPreOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zStart = pPreOp->z; + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zEnd = pOperand->zEnd; } -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. ** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. ** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** sqlite3ParserTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of ** which is sqlite3ParserTOKENTYPE. The entry in the union -** for base tokens is called "yy0". +** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() ** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument ** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 254 +#define YYNOCODE 251 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 70 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - Select* yy3; - ExprList* yy14; - With* yy59; - SrcList* yy65; - struct LikeOp yy96; - Expr* yy132; - u8 yy186; - int yy328; - ExprSpan yy346; - struct TrigEvent yy378; - u16 yy381; - IdList* yy408; - struct {int value; int mask;} yy429; - TriggerStep* yy473; - struct LimitVal yy476; + struct LimitVal yy64; + Expr* yy122; + Select* yy159; + IdList* yy180; + struct {int value; int mask;} yy207; + struct LikeOp yy318; + TriggerStep* yy327; + With* yy331; + ExprSpan yy342; + SrcList* yy347; + int yy392; + struct TrigEvent yy410; + ExprList* yy442; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -124898,16 +128595,18 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 642 -#define YYNRULE 327 #define YYFALLBACK 1 -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) - -/* The yyzerominor constant is used to initialize instances of -** YYMINORTYPE objects to zero. */ -static const YYMINORTYPE yyzerominor = { 0 }; +#define YYNSTATE 440 +#define YYNRULE 326 +#define YY_MAX_SHIFT 439 +#define YY_MIN_SHIFTREDUCE 649 +#define YY_MAX_SHIFTREDUCE 974 +#define YY_MIN_REDUCE 975 +#define YY_MAX_REDUCE 1300 +#define YY_ERROR_ACTION 1301 +#define YY_ACCEPT_ACTION 1302 +#define YY_NO_ACTION 1303 +/************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -124930,16 +128629,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -124968,468 +128671,451 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -*/ -#define YY_ACTTAB_COUNT (1497) +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (1499) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, - /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, - /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, - /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, - /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, - /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, - /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, - /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, - /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, - /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, - /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, - /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, - /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, - /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, - /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, - /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, - /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, - /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, - /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, - /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, - /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, - /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, - /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, - /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, - /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, - /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, - /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, - /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, - /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, - /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, - /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, - /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, - /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, - /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, - /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, - /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, - /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, - /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, - /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, - /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, - /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, - /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, - /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, - /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, - /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, - /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, - /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, - /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, - /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, - /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, - /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, - /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, - /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, - /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, - /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, - /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, - /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, - /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, - /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, - /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, - /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, - /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, - /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, - /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, - /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, - /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, - /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, - /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, - /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, - /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, - /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, - /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, - /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, - /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, - /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, - /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, - /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, - /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, - /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, - /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, - /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, - /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, - /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, - /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, - /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, - /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, - /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, - /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, - /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, - /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, - /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, - /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, - /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, - /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, - /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, - /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, - /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, - /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, - /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, - /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, - /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, - /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, - /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, - /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, - /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, - /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, - /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, - /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, - /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, - /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, - /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, - /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, - /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, - /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, - /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, - /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, - /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, - /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, - /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, - /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, - /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, - /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, - /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, - /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, - /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, - /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, - /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, - /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, - /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, - /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, - /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, - /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, - /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, - /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, - /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, - /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, - /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, - /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, - /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, - /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, - /* 1490 */ 971, 971, 971, 971, 971, 971, 338, + /* 0 */ 315, 1302, 146, 921, 2, 194, 922, 342, 952, 91, + /* 10 */ 91, 91, 91, 84, 89, 89, 89, 89, 88, 88, + /* 20 */ 87, 87, 87, 86, 339, 87, 87, 87, 86, 339, + /* 30 */ 331, 819, 819, 91, 91, 91, 91, 339, 89, 89, + /* 40 */ 89, 89, 88, 88, 87, 87, 87, 86, 339, 319, + /* 50 */ 933, 933, 92, 93, 83, 831, 834, 823, 823, 90, + /* 60 */ 90, 91, 91, 91, 91, 123, 89, 89, 89, 89, + /* 70 */ 88, 88, 87, 87, 87, 86, 339, 315, 952, 89, + /* 80 */ 89, 89, 89, 88, 88, 87, 87, 87, 86, 339, + /* 90 */ 365, 772, 360, 24, 933, 933, 947, 694, 933, 933, + /* 100 */ 773, 937, 933, 933, 434, 715, 328, 434, 819, 819, + /* 110 */ 203, 160, 278, 391, 273, 390, 190, 933, 933, 370, + /* 120 */ 934, 935, 367, 271, 953, 48, 679, 953, 48, 92, + /* 130 */ 93, 83, 831, 834, 823, 823, 90, 90, 91, 91, + /* 140 */ 91, 91, 123, 89, 89, 89, 89, 88, 88, 87, + /* 150 */ 87, 87, 86, 339, 315, 682, 337, 336, 218, 412, + /* 160 */ 398, 68, 412, 403, 934, 935, 743, 959, 934, 935, + /* 170 */ 810, 937, 934, 935, 957, 221, 958, 88, 88, 87, + /* 180 */ 87, 87, 86, 339, 291, 819, 819, 934, 935, 185, + /* 190 */ 94, 792, 388, 385, 384, 1240, 1240, 792, 804, 960, + /* 200 */ 960, 290, 798, 383, 123, 315, 92, 93, 83, 831, + /* 210 */ 834, 823, 823, 90, 90, 91, 91, 91, 91, 326, + /* 220 */ 89, 89, 89, 89, 88, 88, 87, 87, 87, 86, + /* 230 */ 339, 681, 741, 803, 803, 803, 819, 819, 944, 56, + /* 240 */ 253, 353, 242, 85, 82, 168, 253, 358, 252, 110, + /* 250 */ 96, 233, 397, 698, 677, 683, 683, 92, 93, 83, + /* 260 */ 831, 834, 823, 823, 90, 90, 91, 91, 91, 91, + /* 270 */ 433, 89, 89, 89, 89, 88, 88, 87, 87, 87, + /* 280 */ 86, 339, 315, 434, 439, 651, 396, 57, 733, 733, + /* 290 */ 234, 291, 107, 287, 395, 86, 339, 810, 427, 728, + /* 300 */ 933, 933, 185, 953, 30, 388, 385, 384, 215, 949, + /* 310 */ 434, 933, 933, 819, 819, 697, 383, 162, 161, 407, + /* 320 */ 400, 85, 82, 168, 677, 804, 335, 113, 771, 798, + /* 330 */ 953, 48, 22, 351, 92, 93, 83, 831, 834, 823, + /* 340 */ 823, 90, 90, 91, 91, 91, 91, 870, 89, 89, + /* 350 */ 89, 89, 88, 88, 87, 87, 87, 86, 339, 315, + /* 360 */ 803, 803, 803, 268, 123, 412, 394, 1, 933, 933, + /* 370 */ 934, 935, 933, 933, 85, 82, 168, 232, 5, 343, + /* 380 */ 194, 934, 935, 952, 85, 82, 168, 54, 956, 434, + /* 390 */ 819, 819, 431, 938, 939, 792, 67, 759, 350, 144, + /* 400 */ 166, 770, 123, 896, 889, 955, 348, 288, 758, 953, + /* 410 */ 47, 92, 93, 83, 831, 834, 823, 823, 90, 90, + /* 420 */ 91, 91, 91, 91, 892, 89, 89, 89, 89, 88, + /* 430 */ 88, 87, 87, 87, 86, 339, 315, 113, 934, 935, + /* 440 */ 687, 893, 934, 935, 253, 358, 252, 85, 82, 168, + /* 450 */ 820, 820, 956, 952, 338, 938, 939, 894, 701, 721, + /* 460 */ 359, 289, 233, 397, 434, 349, 434, 819, 819, 955, + /* 470 */ 866, 722, 23, 389, 832, 835, 692, 357, 904, 667, + /* 480 */ 194, 702, 402, 952, 953, 48, 953, 48, 92, 93, + /* 490 */ 83, 831, 834, 823, 823, 90, 90, 91, 91, 91, + /* 500 */ 91, 824, 89, 89, 89, 89, 88, 88, 87, 87, + /* 510 */ 87, 86, 339, 315, 434, 113, 434, 680, 434, 332, + /* 520 */ 434, 408, 889, 356, 380, 940, 401, 720, 948, 864, + /* 530 */ 191, 165, 329, 689, 953, 9, 953, 9, 953, 9, + /* 540 */ 953, 9, 718, 948, 819, 819, 953, 8, 325, 111, + /* 550 */ 327, 153, 224, 952, 410, 113, 189, 337, 336, 913, + /* 560 */ 1295, 852, 75, 1295, 73, 92, 93, 83, 831, 834, + /* 570 */ 823, 823, 90, 90, 91, 91, 91, 91, 359, 89, + /* 580 */ 89, 89, 89, 88, 88, 87, 87, 87, 86, 339, + /* 590 */ 315, 730, 148, 236, 797, 366, 789, 892, 1179, 434, + /* 600 */ 960, 960, 400, 148, 314, 212, 873, 911, 757, 404, + /* 610 */ 872, 300, 320, 434, 893, 311, 237, 271, 405, 953, + /* 620 */ 34, 819, 819, 225, 371, 945, 360, 913, 1296, 113, + /* 630 */ 894, 1296, 417, 953, 35, 1245, 922, 342, 259, 247, + /* 640 */ 290, 315, 92, 93, 83, 831, 834, 823, 823, 90, + /* 650 */ 90, 91, 91, 91, 91, 148, 89, 89, 89, 89, + /* 660 */ 88, 88, 87, 87, 87, 86, 339, 310, 434, 796, + /* 670 */ 434, 240, 819, 819, 266, 911, 876, 876, 373, 346, + /* 680 */ 167, 654, 655, 656, 259, 244, 19, 246, 953, 11, + /* 690 */ 953, 26, 222, 92, 93, 83, 831, 834, 823, 823, + /* 700 */ 90, 90, 91, 91, 91, 91, 757, 89, 89, 89, + /* 710 */ 89, 88, 88, 87, 87, 87, 86, 339, 315, 434, + /* 720 */ 261, 434, 264, 696, 434, 241, 434, 344, 971, 308, + /* 730 */ 757, 434, 796, 434, 324, 434, 393, 423, 434, 953, + /* 740 */ 36, 953, 37, 20, 953, 38, 953, 27, 434, 819, + /* 750 */ 819, 953, 28, 953, 39, 953, 40, 738, 953, 41, + /* 760 */ 71, 738, 737, 245, 307, 973, 737, 259, 953, 10, + /* 770 */ 92, 93, 83, 831, 834, 823, 823, 90, 90, 91, + /* 780 */ 91, 91, 91, 434, 89, 89, 89, 89, 88, 88, + /* 790 */ 87, 87, 87, 86, 339, 315, 434, 372, 434, 259, + /* 800 */ 149, 434, 167, 953, 42, 188, 187, 186, 219, 434, + /* 810 */ 748, 434, 974, 434, 796, 434, 953, 98, 953, 43, + /* 820 */ 862, 953, 44, 434, 920, 2, 819, 819, 757, 953, + /* 830 */ 31, 953, 45, 953, 46, 953, 32, 74, 307, 912, + /* 840 */ 220, 259, 259, 953, 115, 909, 315, 92, 93, 83, + /* 850 */ 831, 834, 823, 823, 90, 90, 91, 91, 91, 91, + /* 860 */ 434, 89, 89, 89, 89, 88, 88, 87, 87, 87, + /* 870 */ 86, 339, 434, 248, 434, 215, 949, 819, 819, 333, + /* 880 */ 953, 116, 895, 860, 176, 259, 974, 400, 361, 259, + /* 890 */ 951, 887, 953, 117, 953, 52, 884, 315, 92, 93, + /* 900 */ 83, 831, 834, 823, 823, 90, 90, 91, 91, 91, + /* 910 */ 91, 434, 89, 89, 89, 89, 88, 88, 87, 87, + /* 920 */ 87, 86, 339, 434, 113, 434, 258, 883, 819, 819, + /* 930 */ 727, 953, 33, 363, 259, 673, 321, 189, 430, 321, + /* 940 */ 368, 365, 364, 953, 99, 953, 49, 365, 315, 92, + /* 950 */ 81, 83, 831, 834, 823, 823, 90, 90, 91, 91, + /* 960 */ 91, 91, 434, 89, 89, 89, 89, 88, 88, 87, + /* 970 */ 87, 87, 86, 339, 434, 723, 434, 214, 165, 819, + /* 980 */ 819, 772, 953, 100, 322, 124, 1269, 158, 65, 710, + /* 990 */ 773, 700, 699, 320, 953, 101, 953, 97, 255, 315, + /* 1000 */ 216, 93, 83, 831, 834, 823, 823, 90, 90, 91, + /* 1010 */ 91, 91, 91, 434, 89, 89, 89, 89, 88, 88, + /* 1020 */ 87, 87, 87, 86, 339, 434, 251, 434, 707, 708, + /* 1030 */ 819, 819, 223, 953, 114, 908, 794, 254, 309, 193, + /* 1040 */ 67, 381, 869, 869, 199, 953, 112, 953, 105, 269, + /* 1050 */ 726, 260, 67, 83, 831, 834, 823, 823, 90, 90, + /* 1060 */ 91, 91, 91, 91, 263, 89, 89, 89, 89, 88, + /* 1070 */ 88, 87, 87, 87, 86, 339, 79, 429, 690, 3, + /* 1080 */ 1174, 228, 434, 113, 340, 340, 868, 868, 265, 79, + /* 1090 */ 429, 735, 3, 859, 70, 432, 434, 340, 340, 434, + /* 1100 */ 1259, 434, 953, 104, 434, 670, 416, 766, 432, 434, + /* 1110 */ 193, 434, 413, 434, 418, 806, 953, 102, 420, 953, + /* 1120 */ 103, 953, 48, 123, 953, 51, 810, 418, 424, 953, + /* 1130 */ 53, 953, 50, 953, 25, 267, 123, 711, 113, 810, + /* 1140 */ 428, 277, 695, 272, 764, 113, 76, 77, 690, 434, + /* 1150 */ 795, 113, 276, 78, 436, 435, 412, 414, 798, 76, + /* 1160 */ 77, 113, 855, 859, 376, 199, 78, 436, 435, 953, + /* 1170 */ 29, 798, 744, 113, 755, 79, 429, 675, 3, 415, + /* 1180 */ 109, 292, 293, 340, 340, 806, 802, 678, 672, 803, + /* 1190 */ 803, 803, 805, 18, 432, 661, 660, 662, 927, 209, + /* 1200 */ 150, 352, 803, 803, 803, 805, 18, 6, 306, 280, + /* 1210 */ 282, 284, 786, 418, 250, 386, 243, 886, 694, 362, + /* 1220 */ 286, 163, 275, 79, 429, 810, 3, 857, 856, 159, + /* 1230 */ 419, 340, 340, 298, 930, 968, 126, 196, 965, 903, + /* 1240 */ 901, 323, 432, 136, 55, 76, 77, 742, 147, 58, + /* 1250 */ 121, 129, 78, 436, 435, 65, 783, 798, 354, 131, + /* 1260 */ 355, 418, 379, 132, 133, 134, 175, 139, 151, 369, + /* 1270 */ 888, 180, 791, 810, 61, 851, 871, 69, 429, 375, + /* 1280 */ 3, 756, 210, 257, 181, 340, 340, 145, 803, 803, + /* 1290 */ 803, 805, 18, 76, 77, 377, 432, 262, 182, 183, + /* 1300 */ 78, 436, 435, 663, 312, 798, 392, 714, 713, 712, + /* 1310 */ 330, 705, 692, 313, 704, 418, 686, 406, 752, 685, + /* 1320 */ 274, 684, 942, 64, 279, 195, 281, 810, 753, 839, + /* 1330 */ 751, 283, 72, 750, 285, 422, 803, 803, 803, 805, + /* 1340 */ 18, 334, 426, 95, 411, 229, 409, 76, 77, 230, + /* 1350 */ 734, 66, 231, 294, 78, 436, 435, 204, 295, 798, + /* 1360 */ 217, 296, 297, 669, 21, 305, 304, 303, 206, 301, + /* 1370 */ 437, 928, 664, 205, 208, 207, 438, 658, 657, 652, + /* 1380 */ 118, 108, 119, 226, 650, 341, 157, 170, 169, 239, + /* 1390 */ 803, 803, 803, 805, 18, 125, 120, 235, 238, 317, + /* 1400 */ 318, 345, 106, 790, 867, 127, 865, 128, 130, 724, + /* 1410 */ 249, 172, 174, 882, 135, 137, 59, 138, 173, 60, + /* 1420 */ 885, 123, 171, 177, 178, 881, 7, 12, 179, 256, + /* 1430 */ 874, 140, 193, 962, 374, 141, 666, 152, 378, 276, + /* 1440 */ 184, 382, 142, 122, 62, 13, 387, 703, 270, 14, + /* 1450 */ 63, 227, 809, 808, 837, 732, 15, 841, 736, 4, + /* 1460 */ 765, 211, 399, 164, 213, 143, 760, 201, 70, 316, + /* 1470 */ 67, 838, 836, 891, 198, 192, 16, 197, 890, 917, + /* 1480 */ 154, 17, 202, 421, 918, 155, 200, 156, 425, 840, + /* 1490 */ 807, 1261, 676, 80, 302, 299, 347, 1260, 923, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 0 */ 19, 144, 145, 146, 147, 24, 1, 2, 27, 80, /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, - /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, - /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, - /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, - /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, - /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, - /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, - /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, - /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 20 */ 91, 92, 93, 94, 95, 91, 92, 93, 94, 95, + /* 30 */ 19, 50, 51, 80, 81, 82, 83, 95, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 157, + /* 50 */ 27, 28, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 66, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 85, + /* 80 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 90 */ 152, 33, 152, 22, 27, 28, 179, 180, 27, 28, + /* 100 */ 42, 27, 27, 28, 152, 188, 95, 152, 50, 51, + /* 110 */ 99, 100, 101, 102, 103, 104, 105, 27, 28, 227, + /* 120 */ 97, 98, 230, 112, 172, 173, 172, 172, 173, 71, /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, - /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, - /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, - /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, - /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, - /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, - /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, - /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 140 */ 82, 83, 66, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 172, 89, 90, 218, 207, + /* 160 */ 208, 26, 207, 208, 97, 98, 91, 100, 97, 98, + /* 170 */ 69, 97, 97, 98, 107, 237, 109, 89, 90, 91, + /* 180 */ 92, 93, 94, 95, 152, 50, 51, 97, 98, 99, + /* 190 */ 55, 59, 102, 103, 104, 119, 120, 59, 97, 132, + /* 200 */ 133, 152, 101, 113, 66, 19, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 187, /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, - /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, - /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, - /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, - /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, - /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, - /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, - /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, - /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, - /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, - /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 230 */ 95, 172, 210, 132, 133, 134, 50, 51, 185, 53, + /* 240 */ 108, 109, 110, 221, 222, 223, 108, 109, 110, 22, + /* 250 */ 22, 119, 120, 181, 27, 27, 28, 71, 72, 73, + /* 260 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 270 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 280 */ 94, 95, 19, 152, 148, 149, 115, 24, 117, 118, + /* 290 */ 154, 152, 156, 152, 163, 94, 95, 69, 249, 163, + /* 300 */ 27, 28, 99, 172, 173, 102, 103, 104, 194, 195, + /* 310 */ 152, 27, 28, 50, 51, 181, 113, 89, 90, 152, + /* 320 */ 206, 221, 222, 223, 97, 97, 187, 196, 175, 101, + /* 330 */ 172, 173, 196, 219, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 11, 85, 86, /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, - /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, - /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, - /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, - /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, - /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, - /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, - /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, - /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, - /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, - /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, - /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, - /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, - /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 360 */ 132, 133, 134, 23, 66, 207, 208, 22, 27, 28, + /* 370 */ 97, 98, 27, 28, 221, 222, 223, 199, 22, 243, + /* 380 */ 24, 97, 98, 27, 221, 222, 223, 209, 152, 152, + /* 390 */ 50, 51, 168, 169, 170, 59, 26, 124, 100, 58, + /* 400 */ 152, 175, 66, 240, 163, 169, 170, 152, 124, 172, + /* 410 */ 173, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 12, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 196, 97, 98, + /* 440 */ 23, 29, 97, 98, 108, 109, 110, 221, 222, 223, + /* 450 */ 50, 51, 152, 97, 168, 169, 170, 45, 37, 47, + /* 460 */ 219, 224, 119, 120, 152, 229, 152, 50, 51, 169, + /* 470 */ 170, 59, 231, 52, 74, 75, 106, 236, 152, 21, + /* 480 */ 24, 60, 163, 27, 172, 173, 172, 173, 71, 72, /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, - /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, - /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, - /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, - /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, - /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, - /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, - /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 500 */ 83, 101, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 152, 196, 152, 23, 152, 207, + /* 520 */ 152, 207, 163, 65, 19, 171, 152, 190, 191, 229, + /* 530 */ 211, 212, 111, 179, 172, 173, 172, 173, 172, 173, + /* 540 */ 172, 173, 190, 191, 50, 51, 172, 173, 186, 22, + /* 550 */ 186, 24, 186, 97, 186, 196, 51, 89, 90, 22, + /* 560 */ 23, 103, 137, 26, 139, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 219, 85, /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, - /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, - /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, - /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, - /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, - /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 590 */ 19, 195, 152, 152, 23, 236, 163, 12, 140, 152, + /* 600 */ 132, 133, 206, 152, 164, 23, 31, 70, 26, 19, + /* 610 */ 35, 160, 107, 152, 29, 164, 152, 112, 28, 172, + /* 620 */ 173, 50, 51, 183, 49, 185, 152, 22, 23, 196, + /* 630 */ 45, 26, 47, 172, 173, 0, 1, 2, 152, 16, + /* 640 */ 152, 19, 71, 72, 73, 74, 75, 76, 77, 78, /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, - /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, - /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, - /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, - /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, - /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, - /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, - /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, - /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, - /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, - /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, - /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, - /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, - /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, - /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, - /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, - /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 164, 152, 152, + /* 670 */ 152, 152, 50, 51, 16, 70, 108, 109, 110, 193, + /* 680 */ 98, 7, 8, 9, 152, 62, 22, 64, 172, 173, + /* 690 */ 172, 173, 218, 71, 72, 73, 74, 75, 76, 77, + /* 700 */ 78, 79, 80, 81, 82, 83, 124, 85, 86, 87, + /* 710 */ 88, 89, 90, 91, 92, 93, 94, 95, 19, 152, + /* 720 */ 62, 152, 64, 181, 152, 193, 152, 241, 246, 247, + /* 730 */ 26, 152, 152, 152, 217, 152, 91, 249, 152, 172, + /* 740 */ 173, 172, 173, 79, 172, 173, 172, 173, 152, 50, + /* 750 */ 51, 172, 173, 172, 173, 172, 173, 116, 172, 173, + /* 760 */ 138, 116, 121, 140, 22, 23, 121, 152, 172, 173, + /* 770 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 780 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 790 */ 91, 92, 93, 94, 95, 19, 152, 217, 152, 152, + /* 800 */ 24, 152, 98, 172, 173, 108, 109, 110, 193, 152, + /* 810 */ 213, 152, 70, 152, 152, 152, 172, 173, 172, 173, + /* 820 */ 152, 172, 173, 152, 146, 147, 50, 51, 124, 172, + /* 830 */ 173, 172, 173, 172, 173, 172, 173, 138, 22, 23, + /* 840 */ 193, 152, 152, 172, 173, 152, 19, 71, 72, 73, /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, - /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, - /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 870 */ 94, 95, 152, 152, 152, 194, 195, 50, 51, 217, + /* 880 */ 172, 173, 193, 193, 26, 152, 70, 206, 152, 152, + /* 890 */ 26, 163, 172, 173, 172, 173, 152, 19, 71, 72, /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, - /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, - /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, - /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 920 */ 93, 94, 95, 152, 196, 152, 193, 152, 50, 51, + /* 930 */ 193, 172, 173, 19, 152, 166, 167, 51, 166, 167, + /* 940 */ 152, 152, 28, 172, 173, 172, 173, 152, 19, 71, /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, - /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, - /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, - /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 970 */ 92, 93, 94, 95, 152, 193, 152, 211, 212, 50, + /* 980 */ 51, 33, 172, 173, 244, 245, 23, 123, 130, 26, + /* 990 */ 42, 100, 101, 107, 172, 173, 172, 173, 152, 19, + /* 1000 */ 22, 72, 73, 74, 75, 76, 77, 78, 79, 80, /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, - /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, - /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, - /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, - /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1020 */ 91, 92, 93, 94, 95, 152, 237, 152, 7, 8, + /* 1030 */ 50, 51, 237, 172, 173, 23, 23, 23, 26, 26, + /* 1040 */ 26, 23, 132, 133, 26, 172, 173, 172, 173, 23, + /* 1050 */ 163, 152, 26, 73, 74, 75, 76, 77, 78, 79, /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, - /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, - /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, - /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, - /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, - /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, - /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, - /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, - /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, - /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, - /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, - /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, - /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, - /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, - /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, - /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, - /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, - /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, - /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, - /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, - /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, - /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, - /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, - /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, - /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, - /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, - /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, - /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, - /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, - /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, - /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, - /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, - /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, - /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, - /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, - /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, - /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, - /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, - /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, - /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, - /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, - /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, - /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, - /* 1490 */ 253, 253, 253, 253, 253, 253, 141, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 27, 22, + /* 1080 */ 23, 210, 152, 196, 27, 28, 132, 133, 152, 19, + /* 1090 */ 20, 23, 22, 27, 26, 38, 152, 27, 28, 152, + /* 1100 */ 122, 152, 172, 173, 152, 163, 191, 23, 38, 152, + /* 1110 */ 26, 152, 163, 152, 57, 27, 172, 173, 163, 172, + /* 1120 */ 173, 172, 173, 66, 172, 173, 69, 57, 163, 172, + /* 1130 */ 173, 172, 173, 172, 173, 152, 66, 152, 196, 69, + /* 1140 */ 163, 101, 152, 152, 152, 196, 89, 90, 97, 152, + /* 1150 */ 152, 196, 112, 96, 97, 98, 207, 208, 101, 89, + /* 1160 */ 90, 196, 23, 97, 233, 26, 96, 97, 98, 172, + /* 1170 */ 173, 101, 152, 196, 152, 19, 20, 23, 22, 152, + /* 1180 */ 26, 152, 152, 27, 28, 97, 152, 152, 152, 132, + /* 1190 */ 133, 134, 135, 136, 38, 152, 152, 152, 152, 232, + /* 1200 */ 197, 214, 132, 133, 134, 135, 136, 198, 150, 210, + /* 1210 */ 210, 210, 201, 57, 238, 176, 214, 201, 180, 238, + /* 1220 */ 214, 184, 175, 19, 20, 69, 22, 175, 175, 198, + /* 1230 */ 226, 27, 28, 200, 155, 39, 242, 122, 41, 159, + /* 1240 */ 159, 159, 38, 22, 239, 89, 90, 91, 220, 239, + /* 1250 */ 71, 189, 96, 97, 98, 130, 201, 101, 18, 192, + /* 1260 */ 159, 57, 18, 192, 192, 192, 158, 189, 220, 159, + /* 1270 */ 201, 158, 189, 69, 137, 201, 235, 19, 20, 46, + /* 1280 */ 22, 159, 159, 234, 158, 27, 28, 22, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 177, 38, 159, 158, 158, + /* 1300 */ 96, 97, 98, 159, 177, 101, 107, 174, 174, 174, + /* 1310 */ 48, 182, 106, 177, 182, 57, 174, 125, 216, 176, + /* 1320 */ 174, 174, 174, 107, 215, 159, 215, 69, 216, 159, + /* 1330 */ 216, 215, 137, 216, 215, 177, 132, 133, 134, 135, + /* 1340 */ 136, 95, 177, 129, 126, 225, 127, 89, 90, 228, + /* 1350 */ 205, 128, 228, 204, 96, 97, 98, 25, 203, 101, + /* 1360 */ 5, 202, 201, 162, 26, 10, 11, 12, 13, 14, + /* 1370 */ 161, 13, 17, 153, 6, 153, 151, 151, 151, 151, + /* 1380 */ 165, 178, 165, 178, 4, 3, 22, 32, 15, 34, + /* 1390 */ 132, 133, 134, 135, 136, 245, 165, 142, 43, 248, + /* 1400 */ 248, 68, 16, 120, 23, 131, 23, 111, 123, 20, + /* 1410 */ 16, 56, 125, 1, 123, 131, 79, 111, 63, 79, + /* 1420 */ 28, 66, 67, 36, 122, 1, 5, 22, 107, 140, + /* 1430 */ 54, 54, 26, 61, 44, 107, 20, 24, 19, 112, + /* 1440 */ 105, 53, 22, 40, 22, 22, 53, 30, 23, 22, + /* 1450 */ 22, 53, 23, 23, 23, 116, 22, 11, 23, 22, + /* 1460 */ 28, 23, 26, 122, 23, 22, 124, 122, 26, 114, + /* 1470 */ 26, 23, 23, 23, 22, 36, 36, 26, 23, 23, + /* 1480 */ 22, 36, 122, 24, 23, 22, 26, 22, 24, 23, + /* 1490 */ 23, 122, 23, 22, 15, 23, 141, 122, 1, }; -#define YY_SHIFT_USE_DFLT (-86) -#define YY_SHIFT_COUNT (429) -#define YY_SHIFT_MIN (-85) -#define YY_SHIFT_MAX (1383) +#define YY_SHIFT_USE_DFLT (-72) +#define YY_SHIFT_COUNT (439) +#define YY_SHIFT_MIN (-71) +#define YY_SHIFT_MAX (1497) static const short yy_shift_ofst[] = { - /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, - /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, - /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, - /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, - /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, - /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 0 */ 5, 1057, 1355, 1070, 1204, 1204, 1204, 138, -19, 58, + /* 10 */ 58, 186, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 67, + /* 20 */ 67, 90, 132, 336, 76, 135, 263, 340, 417, 494, + /* 30 */ 571, 622, 699, 776, 827, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 878, + /* 50 */ 827, 929, 980, 980, 1156, 1204, 1204, 1204, 1204, 1204, /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, - /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, - /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, - /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, - /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, - /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, - /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, - /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, - /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, - /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, - /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, - /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, - /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, - /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, - /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, - /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, - /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, - /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, - /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, - /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, - /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, - /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, - /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, - /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, - /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, - /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, - /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, - /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, - /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, - /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, - /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, - /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, + /* 80 */ 1204, 1204, 1204, 1258, 1204, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, -71, -47, -47, + /* 100 */ -47, -47, -47, -6, 88, -66, 23, 458, 505, 468, + /* 110 */ 468, 23, 201, 343, -58, -72, -72, -72, 11, 11, + /* 120 */ 11, 412, 412, 341, 537, 605, 23, 23, 23, 23, + /* 130 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + /* 140 */ 23, 23, 23, 23, 23, 23, 635, 298, 74, 74, + /* 150 */ 343, -1, -1, -1, -1, -1, -1, -72, -72, -72, + /* 160 */ 228, 101, 101, 203, 75, 71, 273, 284, 345, 23, + /* 170 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + /* 180 */ 23, 23, 23, 23, 23, 23, 421, 421, 421, 23, + /* 190 */ 23, 582, 23, 23, 23, 356, 23, 23, 585, 23, + /* 200 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 568, + /* 210 */ 575, 456, 456, 456, 704, 171, 645, 674, 858, 590, + /* 220 */ 590, 914, 858, 914, 370, 963, 886, 948, 590, 425, + /* 230 */ 948, 948, 864, 641, 527, 1196, 1115, 1115, 1197, 1197, + /* 240 */ 1115, 1221, 1179, 1125, 1240, 1240, 1240, 1240, 1115, 1244, + /* 250 */ 1125, 1221, 1179, 1179, 1125, 1115, 1244, 1137, 1233, 1115, + /* 260 */ 1115, 1244, 1265, 1115, 1244, 1115, 1244, 1265, 1199, 1199, + /* 270 */ 1199, 1262, 1265, 1199, 1206, 1199, 1262, 1199, 1199, 1192, + /* 280 */ 1216, 1192, 1216, 1192, 1216, 1192, 1216, 1115, 1115, 1195, + /* 290 */ 1265, 1246, 1246, 1265, 1214, 1218, 1223, 1219, 1125, 1332, + /* 300 */ 1338, 1358, 1358, 1368, 1368, 1368, 1368, -72, -72, -72, + /* 310 */ -72, -72, -72, -72, -72, 400, 623, 742, 816, 658, + /* 320 */ 697, 227, 1012, 664, 1013, 1014, 1018, 1026, 1051, 891, + /* 330 */ 1021, 1040, 1068, 1084, 1066, 1139, 910, 954, 1154, 1088, + /* 340 */ 978, 1380, 1382, 1364, 1255, 1373, 1333, 1386, 1381, 1383, + /* 350 */ 1283, 1274, 1296, 1285, 1389, 1287, 1394, 1412, 1291, 1284, + /* 360 */ 1337, 1340, 1306, 1392, 1387, 1302, 1424, 1421, 1405, 1321, + /* 370 */ 1289, 1376, 1406, 1377, 1372, 1390, 1328, 1413, 1416, 1419, + /* 380 */ 1327, 1335, 1420, 1388, 1422, 1423, 1425, 1427, 1393, 1417, + /* 390 */ 1428, 1398, 1403, 1429, 1430, 1431, 1339, 1434, 1435, 1437, + /* 400 */ 1436, 1341, 1438, 1441, 1432, 1439, 1443, 1342, 1442, 1440, + /* 410 */ 1444, 1445, 1442, 1448, 1449, 1450, 1451, 1455, 1452, 1446, + /* 420 */ 1456, 1458, 1459, 1460, 1461, 1463, 1464, 1460, 1466, 1465, + /* 430 */ 1467, 1469, 1471, 1345, 1360, 1369, 1375, 1472, 1479, 1497, }; -#define YY_REDUCE_USE_DFLT (-110) -#define YY_REDUCE_COUNT (305) -#define YY_REDUCE_MIN (-109) -#define YY_REDUCE_MAX (1323) +#define YY_REDUCE_USE_DFLT (-144) +#define YY_REDUCE_COUNT (314) +#define YY_REDUCE_MIN (-143) +#define YY_REDUCE_MAX (1231) static const short yy_reduce_ofst[] = { - /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, - /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, - /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, - /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, - /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, - /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, - /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, - /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, - /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, - /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, - /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, - /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, - /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, - /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, - /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, - /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, - /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, - /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, - /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, - /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, - /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, - /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, - /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, - /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, - /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, - /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, - /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, - /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, - /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, + /* 0 */ -143, 949, 136, 131, -48, -45, 158, 241, 22, 153, + /* 10 */ 226, 163, 362, 364, 366, 312, 314, 368, 237, 236, + /* 20 */ 300, 440, 114, 359, 319, 100, 100, 100, 100, 100, + /* 30 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 40 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 50 */ 100, 100, 100, 100, 374, 447, 461, 516, 518, 567, + /* 60 */ 569, 572, 574, 579, 581, 583, 586, 596, 631, 644, + /* 70 */ 646, 649, 657, 659, 661, 663, 671, 708, 720, 722, + /* 80 */ 759, 771, 773, 810, 822, 824, 861, 873, 875, 930, + /* 90 */ 944, 947, 952, 957, 959, 961, 997, 100, 100, 100, + /* 100 */ 100, 100, 100, 100, 100, 100, 486, -108, -83, 224, + /* 110 */ 286, 451, 100, 681, 100, 100, 100, 100, 354, 354, + /* 120 */ 354, 337, 352, 49, 482, 482, 503, 532, -60, 615, + /* 130 */ 647, 689, 690, 737, 782, -62, 517, 789, 474, 795, + /* 140 */ 580, 733, 32, 662, 488, 139, 678, 433, 769, 772, + /* 150 */ 396, 728, 887, 942, 955, 965, 977, 740, 766, 178, + /* 160 */ -46, -17, 59, 53, 118, 141, 167, 248, 255, 326, + /* 170 */ 441, 464, 519, 668, 693, 721, 736, 744, 775, 788, + /* 180 */ 846, 899, 912, 936, 983, 985, 72, 134, 542, 990, + /* 190 */ 991, 597, 992, 998, 1020, 871, 1022, 1027, 915, 1029, + /* 200 */ 1030, 1034, 118, 1035, 1036, 1043, 1044, 1045, 1046, 931, + /* 210 */ 967, 999, 1000, 1001, 597, 1003, 1009, 1058, 1011, 987, + /* 220 */ 1002, 976, 1016, 981, 1039, 1037, 1038, 1047, 1006, 1004, + /* 230 */ 1052, 1053, 1033, 1031, 1079, 994, 1080, 1081, 1005, 1010, + /* 240 */ 1082, 1028, 1062, 1055, 1067, 1071, 1072, 1073, 1101, 1108, + /* 250 */ 1069, 1048, 1078, 1083, 1074, 1110, 1113, 1041, 1049, 1122, + /* 260 */ 1123, 1126, 1118, 1138, 1140, 1144, 1141, 1127, 1133, 1134, + /* 270 */ 1135, 1129, 1136, 1142, 1143, 1146, 1132, 1147, 1148, 1102, + /* 280 */ 1109, 1112, 1111, 1114, 1116, 1117, 1119, 1166, 1170, 1120, + /* 290 */ 1158, 1121, 1124, 1165, 1145, 1149, 1155, 1159, 1161, 1201, + /* 300 */ 1209, 1220, 1222, 1225, 1226, 1227, 1228, 1151, 1152, 1150, + /* 310 */ 1215, 1217, 1203, 1205, 1231, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, - /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, - /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, - /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, - /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, - /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, - /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, - /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, - /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, - /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, - /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, - /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, - /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, - /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, - /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, - /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, - /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, - /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, - /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, - /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, - /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, - /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, - /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, - /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, - /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, - /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, - /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, - /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, - /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, - /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, - /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, - /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, - /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, - /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, - /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, - /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, - /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, - /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, - /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, - /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, - /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, - /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, - /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, - /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, - /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, - /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, - /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, - /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, - /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, - /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, - /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, - /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, - /* 640 */ 645, 643, + /* 0 */ 1250, 1240, 1240, 1240, 1174, 1174, 1174, 1240, 1071, 1100, + /* 10 */ 1100, 1224, 1301, 1301, 1301, 1301, 1301, 1301, 1173, 1301, + /* 20 */ 1301, 1301, 1301, 1240, 1075, 1106, 1301, 1301, 1301, 1301, + /* 30 */ 1301, 1301, 1301, 1301, 1223, 1225, 1114, 1113, 1206, 1087, + /* 40 */ 1111, 1104, 1108, 1175, 1169, 1170, 1168, 1172, 1176, 1301, + /* 50 */ 1107, 1138, 1153, 1137, 1301, 1301, 1301, 1301, 1301, 1301, + /* 60 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 70 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 80 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 90 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1147, 1152, 1159, + /* 100 */ 1151, 1148, 1140, 1139, 1141, 1142, 1301, 994, 1042, 1301, + /* 110 */ 1301, 1301, 1143, 1301, 1144, 1156, 1155, 1154, 1231, 1258, + /* 120 */ 1257, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 130 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 140 */ 1301, 1301, 1301, 1301, 1301, 1301, 1250, 1240, 1000, 1000, + /* 150 */ 1301, 1240, 1240, 1240, 1240, 1240, 1240, 1236, 1075, 1066, + /* 160 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 170 */ 1228, 1226, 1301, 1187, 1301, 1301, 1301, 1301, 1301, 1301, + /* 180 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 190 */ 1301, 1301, 1301, 1301, 1301, 1071, 1301, 1301, 1301, 1301, + /* 200 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1252, 1301, + /* 210 */ 1201, 1071, 1071, 1071, 1073, 1055, 1065, 979, 1110, 1089, + /* 220 */ 1089, 1290, 1110, 1290, 1017, 1272, 1014, 1100, 1089, 1171, + /* 230 */ 1100, 1100, 1072, 1065, 1301, 1293, 1080, 1080, 1292, 1292, + /* 240 */ 1080, 1119, 1045, 1110, 1051, 1051, 1051, 1051, 1080, 991, + /* 250 */ 1110, 1119, 1045, 1045, 1110, 1080, 991, 1205, 1287, 1080, + /* 260 */ 1080, 991, 1180, 1080, 991, 1080, 991, 1180, 1043, 1043, + /* 270 */ 1043, 1032, 1180, 1043, 1017, 1043, 1032, 1043, 1043, 1093, + /* 280 */ 1088, 1093, 1088, 1093, 1088, 1093, 1088, 1080, 1080, 1301, + /* 290 */ 1180, 1184, 1184, 1180, 1105, 1094, 1103, 1101, 1110, 997, + /* 300 */ 1035, 1255, 1255, 1251, 1251, 1251, 1251, 1298, 1298, 1236, + /* 310 */ 1267, 1267, 1019, 1019, 1267, 1301, 1301, 1301, 1301, 1301, + /* 320 */ 1301, 1262, 1301, 1189, 1301, 1301, 1301, 1301, 1301, 1301, + /* 330 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 340 */ 1125, 1301, 975, 1233, 1301, 1301, 1232, 1301, 1301, 1301, + /* 350 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 360 */ 1301, 1301, 1301, 1301, 1301, 1289, 1301, 1301, 1301, 1301, + /* 370 */ 1301, 1301, 1204, 1203, 1301, 1301, 1301, 1301, 1301, 1301, + /* 380 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 390 */ 1301, 1301, 1301, 1301, 1301, 1301, 1057, 1301, 1301, 1301, + /* 400 */ 1276, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1102, 1301, + /* 410 */ 1095, 1301, 1280, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 420 */ 1301, 1301, 1301, 1242, 1301, 1301, 1301, 1241, 1301, 1301, + /* 430 */ 1301, 1301, 1301, 1127, 1301, 1126, 1130, 1301, 985, 1301, }; +/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens into fallback tokens. If a construct -** like the following: +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: ** ** %fallback ID X Y Z. ** @@ -125437,6 +129123,10 @@ static const YYACTIONTYPE yy_default[] = { ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { @@ -125524,9 +129214,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -125541,7 +129235,9 @@ struct yyParser { #ifdef YYTRACKMAXSTACKDEPTH int yyidxMax; /* Maximum value of yyidx */ #endif +#ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ +#endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ @@ -125629,29 +129325,28 @@ static const char *const yyTokenName[] = { "nm", "savepoint_opt", "create_table", "create_table_args", "createkw", "temp", "ifnotexists", "dbnm", "columnlist", "conslist_opt", "table_options", "select", - "column", "columnid", "type", "carglist", - "typetoken", "typename", "signed", "plus_num", - "minus_num", "ccons", "term", "expr", - "onconf", "sortorder", "autoinc", "idxlist_opt", - "refargs", "defer_subclause", "refarg", "refact", - "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", - "idxlist", "defer_subclause_opt", "orconf", "resolvetype", - "raisetype", "ifexists", "fullname", "selectnowith", - "oneselect", "with", "multiselect_op", "distinct", - "selcollist", "from", "where_opt", "groupby_opt", - "having_opt", "orderby_opt", "limit_opt", "values", - "nexprlist", "exprlist", "sclp", "as", - "seltablist", "stl_prefix", "joinop", "indexed_opt", - "on_opt", "using_opt", "joinop2", "idlist", - "sortlist", "setlist", "insert_cmd", "inscollist_opt", - "likeop", "between_op", "in_op", "case_operand", - "case_exprlist", "case_else", "uniqueflag", "collate", - "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", - "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", - "trnm", "tridxby", "database_kw_opt", "key_opt", - "add_column_fullname", "kwcolumn_opt", "create_vtab", "vtabarglist", - "vtabarg", "vtabargtoken", "lp", "anylist", - "wqlist", + "columnname", "carglist", "typetoken", "typename", + "signed", "plus_num", "minus_num", "ccons", + "term", "expr", "onconf", "sortorder", + "autoinc", "eidlist_opt", "refargs", "defer_subclause", + "refarg", "refact", "init_deferred_pred_opt", "conslist", + "tconscomma", "tcons", "sortlist", "eidlist", + "defer_subclause_opt", "orconf", "resolvetype", "raisetype", + "ifexists", "fullname", "selectnowith", "oneselect", + "with", "multiselect_op", "distinct", "selcollist", + "from", "where_opt", "groupby_opt", "having_opt", + "orderby_opt", "limit_opt", "values", "nexprlist", + "exprlist", "sclp", "as", "seltablist", + "stl_prefix", "joinop", "indexed_opt", "on_opt", + "using_opt", "idlist", "setlist", "insert_cmd", + "idlist_opt", "likeop", "between_op", "in_op", + "case_operand", "case_exprlist", "case_else", "uniqueflag", + "collate", "nmnum", "trigger_decl", "trigger_cmd_list", + "trigger_time", "trigger_event", "foreach_clause", "when_clause", + "trigger_cmd", "trnm", "tridxby", "database_kw_opt", + "key_opt", "add_column_fullname", "kwcolumn_opt", "create_vtab", + "vtabarglist", "vtabarg", "vtabargtoken", "lp", + "anylist", "wqlist", }; #endif /* NDEBUG */ @@ -125659,333 +129354,332 @@ static const char *const yyTokenName[] = { /* For tracing reduce actions, the names of all rules are required. */ static const char *const yyRuleName[] = { - /* 0 */ "input ::= cmdlist", - /* 1 */ "cmdlist ::= cmdlist ecmd", - /* 2 */ "cmdlist ::= ecmd", - /* 3 */ "ecmd ::= SEMI", - /* 4 */ "ecmd ::= explain cmdx SEMI", - /* 5 */ "explain ::=", - /* 6 */ "explain ::= EXPLAIN", - /* 7 */ "explain ::= EXPLAIN QUERY PLAN", - /* 8 */ "cmdx ::= cmd", - /* 9 */ "cmd ::= BEGIN transtype trans_opt", - /* 10 */ "trans_opt ::=", - /* 11 */ "trans_opt ::= TRANSACTION", - /* 12 */ "trans_opt ::= TRANSACTION nm", - /* 13 */ "transtype ::=", - /* 14 */ "transtype ::= DEFERRED", - /* 15 */ "transtype ::= IMMEDIATE", - /* 16 */ "transtype ::= EXCLUSIVE", - /* 17 */ "cmd ::= COMMIT trans_opt", - /* 18 */ "cmd ::= END trans_opt", - /* 19 */ "cmd ::= ROLLBACK trans_opt", - /* 20 */ "savepoint_opt ::= SAVEPOINT", - /* 21 */ "savepoint_opt ::=", - /* 22 */ "cmd ::= SAVEPOINT nm", - /* 23 */ "cmd ::= RELEASE savepoint_opt nm", - /* 24 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", - /* 25 */ "cmd ::= create_table create_table_args", - /* 26 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", - /* 27 */ "createkw ::= CREATE", - /* 28 */ "ifnotexists ::=", - /* 29 */ "ifnotexists ::= IF NOT EXISTS", - /* 30 */ "temp ::= TEMP", - /* 31 */ "temp ::=", - /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", - /* 33 */ "create_table_args ::= AS select", - /* 34 */ "table_options ::=", - /* 35 */ "table_options ::= WITHOUT nm", - /* 36 */ "columnlist ::= columnlist COMMA column", - /* 37 */ "columnlist ::= column", - /* 38 */ "column ::= columnid type carglist", - /* 39 */ "columnid ::= nm", - /* 40 */ "nm ::= ID|INDEXED", - /* 41 */ "nm ::= STRING", - /* 42 */ "nm ::= JOIN_KW", - /* 43 */ "type ::=", - /* 44 */ "type ::= typetoken", - /* 45 */ "typetoken ::= typename", - /* 46 */ "typetoken ::= typename LP signed RP", - /* 47 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 48 */ "typename ::= ID|STRING", - /* 49 */ "typename ::= typename ID|STRING", - /* 50 */ "signed ::= plus_num", - /* 51 */ "signed ::= minus_num", - /* 52 */ "carglist ::= carglist ccons", - /* 53 */ "carglist ::=", - /* 54 */ "ccons ::= CONSTRAINT nm", - /* 55 */ "ccons ::= DEFAULT term", - /* 56 */ "ccons ::= DEFAULT LP expr RP", - /* 57 */ "ccons ::= DEFAULT PLUS term", - /* 58 */ "ccons ::= DEFAULT MINUS term", - /* 59 */ "ccons ::= DEFAULT ID|INDEXED", - /* 60 */ "ccons ::= NULL onconf", - /* 61 */ "ccons ::= NOT NULL onconf", - /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 63 */ "ccons ::= UNIQUE onconf", - /* 64 */ "ccons ::= CHECK LP expr RP", - /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", - /* 66 */ "ccons ::= defer_subclause", - /* 67 */ "ccons ::= COLLATE ID|STRING", - /* 68 */ "autoinc ::=", - /* 69 */ "autoinc ::= AUTOINCR", - /* 70 */ "refargs ::=", - /* 71 */ "refargs ::= refargs refarg", - /* 72 */ "refarg ::= MATCH nm", - /* 73 */ "refarg ::= ON INSERT refact", - /* 74 */ "refarg ::= ON DELETE refact", - /* 75 */ "refarg ::= ON UPDATE refact", - /* 76 */ "refact ::= SET NULL", - /* 77 */ "refact ::= SET DEFAULT", - /* 78 */ "refact ::= CASCADE", - /* 79 */ "refact ::= RESTRICT", - /* 80 */ "refact ::= NO ACTION", - /* 81 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 82 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 83 */ "init_deferred_pred_opt ::=", - /* 84 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 85 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 86 */ "conslist_opt ::=", - /* 87 */ "conslist_opt ::= COMMA conslist", - /* 88 */ "conslist ::= conslist tconscomma tcons", - /* 89 */ "conslist ::= tcons", - /* 90 */ "tconscomma ::= COMMA", - /* 91 */ "tconscomma ::=", - /* 92 */ "tcons ::= CONSTRAINT nm", - /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", - /* 95 */ "tcons ::= CHECK LP expr RP onconf", - /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", - /* 97 */ "defer_subclause_opt ::=", - /* 98 */ "defer_subclause_opt ::= defer_subclause", - /* 99 */ "onconf ::=", - /* 100 */ "onconf ::= ON CONFLICT resolvetype", - /* 101 */ "orconf ::=", - /* 102 */ "orconf ::= OR resolvetype", - /* 103 */ "resolvetype ::= raisetype", - /* 104 */ "resolvetype ::= IGNORE", - /* 105 */ "resolvetype ::= REPLACE", - /* 106 */ "cmd ::= DROP TABLE ifexists fullname", - /* 107 */ "ifexists ::= IF EXISTS", - /* 108 */ "ifexists ::=", - /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", - /* 110 */ "cmd ::= DROP VIEW ifexists fullname", - /* 111 */ "cmd ::= select", - /* 112 */ "select ::= with selectnowith", - /* 113 */ "selectnowith ::= oneselect", - /* 114 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 115 */ "multiselect_op ::= UNION", - /* 116 */ "multiselect_op ::= UNION ALL", - /* 117 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 118 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 119 */ "oneselect ::= values", - /* 120 */ "values ::= VALUES LP nexprlist RP", - /* 121 */ "values ::= values COMMA LP exprlist RP", - /* 122 */ "distinct ::= DISTINCT", - /* 123 */ "distinct ::= ALL", - /* 124 */ "distinct ::=", - /* 125 */ "sclp ::= selcollist COMMA", - /* 126 */ "sclp ::=", - /* 127 */ "selcollist ::= sclp expr as", - /* 128 */ "selcollist ::= sclp STAR", - /* 129 */ "selcollist ::= sclp nm DOT STAR", - /* 130 */ "as ::= AS nm", - /* 131 */ "as ::= ID|STRING", - /* 132 */ "as ::=", - /* 133 */ "from ::=", - /* 134 */ "from ::= FROM seltablist", - /* 135 */ "stl_prefix ::= seltablist joinop", - /* 136 */ "stl_prefix ::=", - /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 140 */ "dbnm ::=", - /* 141 */ "dbnm ::= DOT nm", - /* 142 */ "fullname ::= nm dbnm", - /* 143 */ "joinop ::= COMMA|JOIN", - /* 144 */ "joinop ::= JOIN_KW JOIN", - /* 145 */ "joinop ::= JOIN_KW nm JOIN", - /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 147 */ "on_opt ::= ON expr", - /* 148 */ "on_opt ::=", - /* 149 */ "indexed_opt ::=", - /* 150 */ "indexed_opt ::= INDEXED BY nm", - /* 151 */ "indexed_opt ::= NOT INDEXED", - /* 152 */ "using_opt ::= USING LP idlist RP", - /* 153 */ "using_opt ::=", - /* 154 */ "orderby_opt ::=", - /* 155 */ "orderby_opt ::= ORDER BY sortlist", - /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 157 */ "sortlist ::= expr sortorder", - /* 158 */ "sortorder ::= ASC", - /* 159 */ "sortorder ::= DESC", - /* 160 */ "sortorder ::=", - /* 161 */ "groupby_opt ::=", - /* 162 */ "groupby_opt ::= GROUP BY nexprlist", - /* 163 */ "having_opt ::=", - /* 164 */ "having_opt ::= HAVING expr", - /* 165 */ "limit_opt ::=", - /* 166 */ "limit_opt ::= LIMIT expr", - /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 170 */ "where_opt ::=", - /* 171 */ "where_opt ::= WHERE expr", - /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 173 */ "setlist ::= setlist COMMA nm EQ expr", - /* 174 */ "setlist ::= nm EQ expr", - /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", - /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 177 */ "insert_cmd ::= INSERT orconf", - /* 178 */ "insert_cmd ::= REPLACE", - /* 179 */ "inscollist_opt ::=", - /* 180 */ "inscollist_opt ::= LP idlist RP", - /* 181 */ "idlist ::= idlist COMMA nm", - /* 182 */ "idlist ::= nm", - /* 183 */ "expr ::= term", - /* 184 */ "expr ::= LP expr RP", - /* 185 */ "term ::= NULL", - /* 186 */ "expr ::= ID|INDEXED", - /* 187 */ "expr ::= JOIN_KW", - /* 188 */ "expr ::= nm DOT nm", - /* 189 */ "expr ::= nm DOT nm DOT nm", - /* 190 */ "term ::= INTEGER|FLOAT|BLOB", - /* 191 */ "term ::= STRING", - /* 192 */ "expr ::= VARIABLE", - /* 193 */ "expr ::= expr COLLATE ID|STRING", - /* 194 */ "expr ::= CAST LP expr AS typetoken RP", - /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 196 */ "expr ::= ID|INDEXED LP STAR RP", - /* 197 */ "term ::= CTIME_KW", - /* 198 */ "expr ::= expr AND expr", - /* 199 */ "expr ::= expr OR expr", - /* 200 */ "expr ::= expr LT|GT|GE|LE expr", - /* 201 */ "expr ::= expr EQ|NE expr", - /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 203 */ "expr ::= expr PLUS|MINUS expr", - /* 204 */ "expr ::= expr STAR|SLASH|REM expr", - /* 205 */ "expr ::= expr CONCAT expr", - /* 206 */ "likeop ::= LIKE_KW|MATCH", - /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 208 */ "expr ::= expr likeop expr", - /* 209 */ "expr ::= expr likeop expr ESCAPE expr", - /* 210 */ "expr ::= expr ISNULL|NOTNULL", - /* 211 */ "expr ::= expr NOT NULL", - /* 212 */ "expr ::= expr IS expr", - /* 213 */ "expr ::= expr IS NOT expr", - /* 214 */ "expr ::= NOT expr", - /* 215 */ "expr ::= BITNOT expr", - /* 216 */ "expr ::= MINUS expr", - /* 217 */ "expr ::= PLUS expr", - /* 218 */ "between_op ::= BETWEEN", - /* 219 */ "between_op ::= NOT BETWEEN", - /* 220 */ "expr ::= expr between_op expr AND expr", - /* 221 */ "in_op ::= IN", - /* 222 */ "in_op ::= NOT IN", - /* 223 */ "expr ::= expr in_op LP exprlist RP", - /* 224 */ "expr ::= LP select RP", - /* 225 */ "expr ::= expr in_op LP select RP", - /* 226 */ "expr ::= expr in_op nm dbnm", - /* 227 */ "expr ::= EXISTS LP select RP", - /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 230 */ "case_exprlist ::= WHEN expr THEN expr", - /* 231 */ "case_else ::= ELSE expr", - /* 232 */ "case_else ::=", - /* 233 */ "case_operand ::= expr", - /* 234 */ "case_operand ::=", - /* 235 */ "exprlist ::= nexprlist", - /* 236 */ "exprlist ::=", - /* 237 */ "nexprlist ::= nexprlist COMMA expr", - /* 238 */ "nexprlist ::= expr", - /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", - /* 240 */ "uniqueflag ::= UNIQUE", - /* 241 */ "uniqueflag ::=", - /* 242 */ "idxlist_opt ::=", - /* 243 */ "idxlist_opt ::= LP idxlist RP", - /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 245 */ "idxlist ::= nm collate sortorder", - /* 246 */ "collate ::=", - /* 247 */ "collate ::= COLLATE ID|STRING", - /* 248 */ "cmd ::= DROP INDEX ifexists fullname", - /* 249 */ "cmd ::= VACUUM", - /* 250 */ "cmd ::= VACUUM nm", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "nmnum ::= plus_num", - /* 257 */ "nmnum ::= nm", - /* 258 */ "nmnum ::= ON", - /* 259 */ "nmnum ::= DELETE", - /* 260 */ "nmnum ::= DEFAULT", - /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 262 */ "plus_num ::= INTEGER|FLOAT", - /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 266 */ "trigger_time ::= BEFORE", - /* 267 */ "trigger_time ::= AFTER", - /* 268 */ "trigger_time ::= INSTEAD OF", - /* 269 */ "trigger_time ::=", - /* 270 */ "trigger_event ::= DELETE|INSERT", - /* 271 */ "trigger_event ::= UPDATE", - /* 272 */ "trigger_event ::= UPDATE OF idlist", - /* 273 */ "foreach_clause ::=", - /* 274 */ "foreach_clause ::= FOR EACH ROW", - /* 275 */ "when_clause ::=", - /* 276 */ "when_clause ::= WHEN expr", - /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 279 */ "trnm ::= nm", - /* 280 */ "trnm ::= nm DOT nm", - /* 281 */ "tridxby ::=", - /* 282 */ "tridxby ::= INDEXED BY nm", - /* 283 */ "tridxby ::= NOT INDEXED", - /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 287 */ "trigger_cmd ::= select", - /* 288 */ "expr ::= RAISE LP IGNORE RP", - /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 290 */ "raisetype ::= ROLLBACK", - /* 291 */ "raisetype ::= ABORT", - /* 292 */ "raisetype ::= FAIL", - /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 295 */ "cmd ::= DETACH database_kw_opt expr", - /* 296 */ "key_opt ::=", - /* 297 */ "key_opt ::= KEY expr", - /* 298 */ "database_kw_opt ::= DATABASE", - /* 299 */ "database_kw_opt ::=", - /* 300 */ "cmd ::= REINDEX", - /* 301 */ "cmd ::= REINDEX nm dbnm", - /* 302 */ "cmd ::= ANALYZE", - /* 303 */ "cmd ::= ANALYZE nm dbnm", - /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 306 */ "add_column_fullname ::= fullname", - /* 307 */ "kwcolumn_opt ::=", - /* 308 */ "kwcolumn_opt ::= COLUMNKW", - /* 309 */ "cmd ::= create_vtab", - /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 312 */ "vtabarglist ::= vtabarg", - /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 314 */ "vtabarg ::=", - /* 315 */ "vtabarg ::= vtabarg vtabargtoken", - /* 316 */ "vtabargtoken ::= ANY", - /* 317 */ "vtabargtoken ::= lp anylist RP", - /* 318 */ "lp ::= LP", - /* 319 */ "anylist ::=", - /* 320 */ "anylist ::= anylist LP anylist RP", - /* 321 */ "anylist ::= anylist ANY", - /* 322 */ "with ::=", - /* 323 */ "with ::= WITH wqlist", - /* 324 */ "with ::= WITH RECURSIVE wqlist", - /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", - /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", + /* 0 */ "explain ::= EXPLAIN", + /* 1 */ "explain ::= EXPLAIN QUERY PLAN", + /* 2 */ "cmdx ::= cmd", + /* 3 */ "cmd ::= BEGIN transtype trans_opt", + /* 4 */ "transtype ::=", + /* 5 */ "transtype ::= DEFERRED", + /* 6 */ "transtype ::= IMMEDIATE", + /* 7 */ "transtype ::= EXCLUSIVE", + /* 8 */ "cmd ::= COMMIT trans_opt", + /* 9 */ "cmd ::= END trans_opt", + /* 10 */ "cmd ::= ROLLBACK trans_opt", + /* 11 */ "cmd ::= SAVEPOINT nm", + /* 12 */ "cmd ::= RELEASE savepoint_opt nm", + /* 13 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 14 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 15 */ "createkw ::= CREATE", + /* 16 */ "ifnotexists ::=", + /* 17 */ "ifnotexists ::= IF NOT EXISTS", + /* 18 */ "temp ::= TEMP", + /* 19 */ "temp ::=", + /* 20 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 21 */ "create_table_args ::= AS select", + /* 22 */ "table_options ::=", + /* 23 */ "table_options ::= WITHOUT nm", + /* 24 */ "columnname ::= nm typetoken", + /* 25 */ "typetoken ::=", + /* 26 */ "typetoken ::= typename LP signed RP", + /* 27 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 28 */ "typename ::= typename ID|STRING", + /* 29 */ "ccons ::= CONSTRAINT nm", + /* 30 */ "ccons ::= DEFAULT term", + /* 31 */ "ccons ::= DEFAULT LP expr RP", + /* 32 */ "ccons ::= DEFAULT PLUS term", + /* 33 */ "ccons ::= DEFAULT MINUS term", + /* 34 */ "ccons ::= DEFAULT ID|INDEXED", + /* 35 */ "ccons ::= NOT NULL onconf", + /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 37 */ "ccons ::= UNIQUE onconf", + /* 38 */ "ccons ::= CHECK LP expr RP", + /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 40 */ "ccons ::= defer_subclause", + /* 41 */ "ccons ::= COLLATE ID|STRING", + /* 42 */ "autoinc ::=", + /* 43 */ "autoinc ::= AUTOINCR", + /* 44 */ "refargs ::=", + /* 45 */ "refargs ::= refargs refarg", + /* 46 */ "refarg ::= MATCH nm", + /* 47 */ "refarg ::= ON INSERT refact", + /* 48 */ "refarg ::= ON DELETE refact", + /* 49 */ "refarg ::= ON UPDATE refact", + /* 50 */ "refact ::= SET NULL", + /* 51 */ "refact ::= SET DEFAULT", + /* 52 */ "refact ::= CASCADE", + /* 53 */ "refact ::= RESTRICT", + /* 54 */ "refact ::= NO ACTION", + /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 57 */ "init_deferred_pred_opt ::=", + /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 60 */ "conslist_opt ::=", + /* 61 */ "tconscomma ::= COMMA", + /* 62 */ "tcons ::= CONSTRAINT nm", + /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 65 */ "tcons ::= CHECK LP expr RP onconf", + /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 67 */ "defer_subclause_opt ::=", + /* 68 */ "onconf ::=", + /* 69 */ "onconf ::= ON CONFLICT resolvetype", + /* 70 */ "orconf ::=", + /* 71 */ "orconf ::= OR resolvetype", + /* 72 */ "resolvetype ::= IGNORE", + /* 73 */ "resolvetype ::= REPLACE", + /* 74 */ "cmd ::= DROP TABLE ifexists fullname", + /* 75 */ "ifexists ::= IF EXISTS", + /* 76 */ "ifexists ::=", + /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 78 */ "cmd ::= DROP VIEW ifexists fullname", + /* 79 */ "cmd ::= select", + /* 80 */ "select ::= with selectnowith", + /* 81 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 82 */ "multiselect_op ::= UNION", + /* 83 */ "multiselect_op ::= UNION ALL", + /* 84 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 85 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 86 */ "values ::= VALUES LP nexprlist RP", + /* 87 */ "values ::= values COMMA LP exprlist RP", + /* 88 */ "distinct ::= DISTINCT", + /* 89 */ "distinct ::= ALL", + /* 90 */ "distinct ::=", + /* 91 */ "sclp ::=", + /* 92 */ "selcollist ::= sclp expr as", + /* 93 */ "selcollist ::= sclp STAR", + /* 94 */ "selcollist ::= sclp nm DOT STAR", + /* 95 */ "as ::= AS nm", + /* 96 */ "as ::=", + /* 97 */ "from ::=", + /* 98 */ "from ::= FROM seltablist", + /* 99 */ "stl_prefix ::= seltablist joinop", + /* 100 */ "stl_prefix ::=", + /* 101 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 102 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 103 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 104 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 105 */ "dbnm ::=", + /* 106 */ "dbnm ::= DOT nm", + /* 107 */ "fullname ::= nm dbnm", + /* 108 */ "joinop ::= COMMA|JOIN", + /* 109 */ "joinop ::= JOIN_KW JOIN", + /* 110 */ "joinop ::= JOIN_KW nm JOIN", + /* 111 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 112 */ "on_opt ::= ON expr", + /* 113 */ "on_opt ::=", + /* 114 */ "indexed_opt ::=", + /* 115 */ "indexed_opt ::= INDEXED BY nm", + /* 116 */ "indexed_opt ::= NOT INDEXED", + /* 117 */ "using_opt ::= USING LP idlist RP", + /* 118 */ "using_opt ::=", + /* 119 */ "orderby_opt ::=", + /* 120 */ "orderby_opt ::= ORDER BY sortlist", + /* 121 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 122 */ "sortlist ::= expr sortorder", + /* 123 */ "sortorder ::= ASC", + /* 124 */ "sortorder ::= DESC", + /* 125 */ "sortorder ::=", + /* 126 */ "groupby_opt ::=", + /* 127 */ "groupby_opt ::= GROUP BY nexprlist", + /* 128 */ "having_opt ::=", + /* 129 */ "having_opt ::= HAVING expr", + /* 130 */ "limit_opt ::=", + /* 131 */ "limit_opt ::= LIMIT expr", + /* 132 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 133 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 134 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 135 */ "where_opt ::=", + /* 136 */ "where_opt ::= WHERE expr", + /* 137 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 138 */ "setlist ::= setlist COMMA nm EQ expr", + /* 139 */ "setlist ::= nm EQ expr", + /* 140 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", + /* 141 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", + /* 142 */ "insert_cmd ::= INSERT orconf", + /* 143 */ "insert_cmd ::= REPLACE", + /* 144 */ "idlist_opt ::=", + /* 145 */ "idlist_opt ::= LP idlist RP", + /* 146 */ "idlist ::= idlist COMMA nm", + /* 147 */ "idlist ::= nm", + /* 148 */ "expr ::= LP expr RP", + /* 149 */ "term ::= NULL", + /* 150 */ "expr ::= ID|INDEXED", + /* 151 */ "expr ::= JOIN_KW", + /* 152 */ "expr ::= nm DOT nm", + /* 153 */ "expr ::= nm DOT nm DOT nm", + /* 154 */ "term ::= INTEGER|FLOAT|BLOB", + /* 155 */ "term ::= STRING", + /* 156 */ "expr ::= VARIABLE", + /* 157 */ "expr ::= expr COLLATE ID|STRING", + /* 158 */ "expr ::= CAST LP expr AS typetoken RP", + /* 159 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 160 */ "expr ::= ID|INDEXED LP STAR RP", + /* 161 */ "term ::= CTIME_KW", + /* 162 */ "expr ::= expr AND expr", + /* 163 */ "expr ::= expr OR expr", + /* 164 */ "expr ::= expr LT|GT|GE|LE expr", + /* 165 */ "expr ::= expr EQ|NE expr", + /* 166 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 167 */ "expr ::= expr PLUS|MINUS expr", + /* 168 */ "expr ::= expr STAR|SLASH|REM expr", + /* 169 */ "expr ::= expr CONCAT expr", + /* 170 */ "likeop ::= LIKE_KW|MATCH", + /* 171 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 172 */ "expr ::= expr likeop expr", + /* 173 */ "expr ::= expr likeop expr ESCAPE expr", + /* 174 */ "expr ::= expr ISNULL|NOTNULL", + /* 175 */ "expr ::= expr NOT NULL", + /* 176 */ "expr ::= expr IS expr", + /* 177 */ "expr ::= expr IS NOT expr", + /* 178 */ "expr ::= NOT expr", + /* 179 */ "expr ::= BITNOT expr", + /* 180 */ "expr ::= MINUS expr", + /* 181 */ "expr ::= PLUS expr", + /* 182 */ "between_op ::= BETWEEN", + /* 183 */ "between_op ::= NOT BETWEEN", + /* 184 */ "expr ::= expr between_op expr AND expr", + /* 185 */ "in_op ::= IN", + /* 186 */ "in_op ::= NOT IN", + /* 187 */ "expr ::= expr in_op LP exprlist RP", + /* 188 */ "expr ::= LP select RP", + /* 189 */ "expr ::= expr in_op LP select RP", + /* 190 */ "expr ::= expr in_op nm dbnm", + /* 191 */ "expr ::= EXISTS LP select RP", + /* 192 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 193 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 194 */ "case_exprlist ::= WHEN expr THEN expr", + /* 195 */ "case_else ::= ELSE expr", + /* 196 */ "case_else ::=", + /* 197 */ "case_operand ::= expr", + /* 198 */ "case_operand ::=", + /* 199 */ "exprlist ::=", + /* 200 */ "nexprlist ::= nexprlist COMMA expr", + /* 201 */ "nexprlist ::= expr", + /* 202 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 203 */ "uniqueflag ::= UNIQUE", + /* 204 */ "uniqueflag ::=", + /* 205 */ "eidlist_opt ::=", + /* 206 */ "eidlist_opt ::= LP eidlist RP", + /* 207 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 208 */ "eidlist ::= nm collate sortorder", + /* 209 */ "collate ::=", + /* 210 */ "collate ::= COLLATE ID|STRING", + /* 211 */ "cmd ::= DROP INDEX ifexists fullname", + /* 212 */ "cmd ::= VACUUM", + /* 213 */ "cmd ::= VACUUM nm", + /* 214 */ "cmd ::= PRAGMA nm dbnm", + /* 215 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 216 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 217 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 218 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 219 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 220 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 221 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 222 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 223 */ "trigger_time ::= BEFORE", + /* 224 */ "trigger_time ::= AFTER", + /* 225 */ "trigger_time ::= INSTEAD OF", + /* 226 */ "trigger_time ::=", + /* 227 */ "trigger_event ::= DELETE|INSERT", + /* 228 */ "trigger_event ::= UPDATE", + /* 229 */ "trigger_event ::= UPDATE OF idlist", + /* 230 */ "when_clause ::=", + /* 231 */ "when_clause ::= WHEN expr", + /* 232 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 233 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 234 */ "trnm ::= nm DOT nm", + /* 235 */ "tridxby ::= INDEXED BY nm", + /* 236 */ "tridxby ::= NOT INDEXED", + /* 237 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 238 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", + /* 239 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 240 */ "trigger_cmd ::= select", + /* 241 */ "expr ::= RAISE LP IGNORE RP", + /* 242 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 243 */ "raisetype ::= ROLLBACK", + /* 244 */ "raisetype ::= ABORT", + /* 245 */ "raisetype ::= FAIL", + /* 246 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 247 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 248 */ "cmd ::= DETACH database_kw_opt expr", + /* 249 */ "key_opt ::=", + /* 250 */ "key_opt ::= KEY expr", + /* 251 */ "cmd ::= REINDEX", + /* 252 */ "cmd ::= REINDEX nm dbnm", + /* 253 */ "cmd ::= ANALYZE", + /* 254 */ "cmd ::= ANALYZE nm dbnm", + /* 255 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 256 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 257 */ "add_column_fullname ::= fullname", + /* 258 */ "cmd ::= create_vtab", + /* 259 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 260 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 261 */ "vtabarg ::=", + /* 262 */ "vtabargtoken ::= ANY", + /* 263 */ "vtabargtoken ::= lp anylist RP", + /* 264 */ "lp ::= LP", + /* 265 */ "with ::=", + /* 266 */ "with ::= WITH wqlist", + /* 267 */ "with ::= WITH RECURSIVE wqlist", + /* 268 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 269 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", + /* 270 */ "input ::= cmdlist", + /* 271 */ "cmdlist ::= cmdlist ecmd", + /* 272 */ "cmdlist ::= ecmd", + /* 273 */ "ecmd ::= SEMI", + /* 274 */ "ecmd ::= explain cmdx SEMI", + /* 275 */ "explain ::=", + /* 276 */ "trans_opt ::=", + /* 277 */ "trans_opt ::= TRANSACTION", + /* 278 */ "trans_opt ::= TRANSACTION nm", + /* 279 */ "savepoint_opt ::= SAVEPOINT", + /* 280 */ "savepoint_opt ::=", + /* 281 */ "cmd ::= create_table create_table_args", + /* 282 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 283 */ "columnlist ::= columnname carglist", + /* 284 */ "nm ::= ID|INDEXED", + /* 285 */ "nm ::= STRING", + /* 286 */ "nm ::= JOIN_KW", + /* 287 */ "typetoken ::= typename", + /* 288 */ "typename ::= ID|STRING", + /* 289 */ "signed ::= plus_num", + /* 290 */ "signed ::= minus_num", + /* 291 */ "carglist ::= carglist ccons", + /* 292 */ "carglist ::=", + /* 293 */ "ccons ::= NULL onconf", + /* 294 */ "conslist_opt ::= COMMA conslist", + /* 295 */ "conslist ::= conslist tconscomma tcons", + /* 296 */ "conslist ::= tcons", + /* 297 */ "tconscomma ::=", + /* 298 */ "defer_subclause_opt ::= defer_subclause", + /* 299 */ "resolvetype ::= raisetype", + /* 300 */ "selectnowith ::= oneselect", + /* 301 */ "oneselect ::= values", + /* 302 */ "sclp ::= selcollist COMMA", + /* 303 */ "as ::= ID|STRING", + /* 304 */ "expr ::= term", + /* 305 */ "exprlist ::= nexprlist", + /* 306 */ "nmnum ::= plus_num", + /* 307 */ "nmnum ::= nm", + /* 308 */ "nmnum ::= ON", + /* 309 */ "nmnum ::= DELETE", + /* 310 */ "nmnum ::= DEFAULT", + /* 311 */ "plus_num ::= INTEGER|FLOAT", + /* 312 */ "foreach_clause ::=", + /* 313 */ "foreach_clause ::= FOR EACH ROW", + /* 314 */ "trnm ::= nm", + /* 315 */ "tridxby ::=", + /* 316 */ "database_kw_opt ::= DATABASE", + /* 317 */ "database_kw_opt ::=", + /* 318 */ "kwcolumn_opt ::=", + /* 319 */ "kwcolumn_opt ::= COLUMNKW", + /* 320 */ "vtabarglist ::= vtabarg", + /* 321 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 322 */ "vtabarg ::= vtabarg vtabargtoken", + /* 323 */ "anylist ::=", + /* 324 */ "anylist ::= anylist LP anylist RP", + /* 325 */ "anylist ::= anylist ANY", }; #endif /* NDEBUG */ @@ -126013,6 +129707,15 @@ static void yyGrowStack(yyParser *p){ } #endif +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -126025,9 +129728,9 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to sqlite3Parser and sqlite3ParserFree. */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(u64)){ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH @@ -126042,10 +129745,12 @@ SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(u64)){ return pParser; } -/* The following function deletes the value associated with a -** symbol. The symbol can be either a terminal or nonterminal. -** "yymajor" is the symbol code, and "yypminor" is a pointer to -** the value. +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. */ static void yy_destructor( yyParser *yypParser, /* The parser */ @@ -126061,81 +129766,83 @@ static void yy_destructor( ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are not used + ** which appear on the RHS of the rule, but which are *not* used ** inside the C code. */ +/********* Begin destructor definitions ***************************************/ case 163: /* select */ - case 195: /* selectnowith */ - case 196: /* oneselect */ - case 207: /* values */ + case 194: /* selectnowith */ + case 195: /* oneselect */ + case 206: /* values */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy3)); +sqlite3SelectDelete(pParse->db, (yypminor->yy159)); } break; - case 174: /* term */ - case 175: /* expr */ + case 172: /* term */ + case 173: /* expr */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy342).pExpr); } break; - case 179: /* idxlist_opt */ - case 188: /* idxlist */ - case 200: /* selcollist */ - case 203: /* groupby_opt */ - case 205: /* orderby_opt */ - case 208: /* nexprlist */ - case 209: /* exprlist */ - case 210: /* sclp */ - case 220: /* sortlist */ - case 221: /* setlist */ - case 228: /* case_exprlist */ + case 177: /* eidlist_opt */ + case 186: /* sortlist */ + case 187: /* eidlist */ + case 199: /* selcollist */ + case 202: /* groupby_opt */ + case 204: /* orderby_opt */ + case 207: /* nexprlist */ + case 208: /* exprlist */ + case 209: /* sclp */ + case 218: /* setlist */ + case 225: /* case_exprlist */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy442)); } break; - case 194: /* fullname */ - case 201: /* from */ - case 212: /* seltablist */ - case 213: /* stl_prefix */ + case 193: /* fullname */ + case 200: /* from */ + case 211: /* seltablist */ + case 212: /* stl_prefix */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy347)); } break; - case 197: /* with */ - case 252: /* wqlist */ + case 196: /* with */ + case 249: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy59)); +sqlite3WithDelete(pParse->db, (yypminor->yy331)); } break; - case 202: /* where_opt */ - case 204: /* having_opt */ - case 216: /* on_opt */ - case 227: /* case_operand */ - case 229: /* case_else */ - case 238: /* when_clause */ - case 243: /* key_opt */ + case 201: /* where_opt */ + case 203: /* having_opt */ + case 215: /* on_opt */ + case 224: /* case_operand */ + case 226: /* case_else */ + case 235: /* when_clause */ + case 240: /* key_opt */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy132)); +sqlite3ExprDelete(pParse->db, (yypminor->yy122)); } break; - case 217: /* using_opt */ - case 219: /* idlist */ - case 223: /* inscollist_opt */ + case 216: /* using_opt */ + case 217: /* idlist */ + case 220: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy408)); +sqlite3IdListDelete(pParse->db, (yypminor->yy180)); } break; - case 234: /* trigger_cmd_list */ - case 239: /* trigger_cmd */ + case 231: /* trigger_cmd_list */ + case 236: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy473)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy327)); } break; - case 236: /* trigger_event */ + case 233: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy410).b); } break; +/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } @@ -126145,49 +129852,37 @@ sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); ** ** If there is a destructor routine associated with the token which ** is popped from the stack, then call it. -** -** Return the major token number for the symbol popped. */ -static int yy_pop_parser_stack(yyParser *pParser){ - YYCODETYPE yymajor; - yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; - - /* There is no mechanism by which the parser stack can be popped below - ** empty in SQLite. */ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; assert( pParser->yyidx>=0 ); + yytos = &pParser->yystack[pParser->yyidx--]; #ifndef NDEBUG - if( yyTraceFILE && pParser->yyidx>=0 ){ + if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sPopping %s\n", yyTracePrompt, yyTokenName[yytos->major]); } #endif - yymajor = yytos->major; - yy_destructor(pParser, yymajor, &yytos->minor); - pParser->yyidx--; - return yymajor; + yy_destructor(pParser, yytos->major, &yytos->minor); } /* -** Deallocate and destroy a parser. Destructors are all called for +** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. ** -** Inputs: -** <ul> -** <li> A pointer to the parser. This should be a pointer -** obtained from sqlite3ParserAlloc. -** <li> A pointer to a function used to reclaim memory obtained -** from malloc. -** </ul> +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. */ SQLITE_PRIVATE void sqlite3ParserFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ yyParser *pParser = (yyParser*)p; - /* In SQLite, we never try to destroy a parser that was not successfully - ** created in the first place. */ - if( NEVER(pParser==0) ) return; +#ifndef YYPARSEFREENEVERNULL + if( pParser==0 ) return; +#endif while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 free(pParser->yystack); @@ -126208,75 +129903,72 @@ SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ -static int yy_find_shift_action( +static unsigned int yy_find_shift_action( yyParser *pParser, /* The parser */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ + if( stateno>=YY_MIN_REDUCE ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); + do{ + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ #ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0]) - && (iFallback = yyFallback[iLookAhead])!=0 ){ + YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0]) + && (iFallback = yyFallback[iLookAhead])!=0 ){ #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } #endif - return yy_find_shift_action(pParser, iFallback); - } + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } #endif #ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( + { + int j = i - iLookAhead + YYWILDCARD; + if( #if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && + j>=0 && #endif #if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j<YY_ACTTAB_COUNT && + j<YY_ACTTAB_COUNT && #endif - yy_lookahead[j]==YYWILDCARD - ){ + yy_lookahead[j]==YYWILDCARD + ){ #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); - } + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } #endif /* NDEBUG */ - return yy_action[j]; + return yy_action[j]; + } } - } #endif /* YYWILDCARD */ + } + return yy_default[stateno]; + }else{ + return yy_action[i]; } - return yy_default[stateno]; - }else{ - return yy_action[i]; - } + }while(1); } /* ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_reduce_action( int stateno, /* Current state number */ @@ -126308,7 +130000,7 @@ static int yy_find_reduce_action( /* ** The following routine is called if the stack overflows. */ -static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ +static void yyStackOverflow(yyParser *yypParser){ sqlite3ParserARG_FETCH; yypParser->yyidx--; #ifndef NDEBUG @@ -126319,20 +130011,41 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ - UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); +/******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ } /* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + if( yyNewState<YYNSTATE ){ + fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n", + yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%sShift '%s'\n", + yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + +/* ** Perform a shift action. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ + sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; yypParser->yyidx++; @@ -126343,14 +130056,14 @@ static void yy_shift( #endif #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ - yyStackOverflow(yypParser, yypMinor); + yyStackOverflow(yypParser); return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ - yyStackOverflow(yypParser, yypMinor); + yyStackOverflow(yypParser); return; } } @@ -126358,17 +130071,8 @@ static void yy_shift( yytos = &yypParser->yystack[yypParser->yyidx]; yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; - yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - } -#endif + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -126378,19 +130082,10 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 144, 1 }, - { 145, 2 }, - { 145, 1 }, - { 146, 1 }, - { 146, 3 }, - { 147, 0 }, { 147, 1 }, { 147, 3 }, { 148, 1 }, { 149, 3 }, - { 151, 0 }, - { 151, 1 }, - { 151, 2 }, { 150, 0 }, { 150, 1 }, { 150, 1 }, @@ -126398,12 +130093,9 @@ static const struct { { 149, 2 }, { 149, 2 }, { 149, 2 }, - { 153, 1 }, - { 153, 0 }, { 149, 2 }, { 149, 3 }, { 149, 5 }, - { 149, 2 }, { 154, 6 }, { 156, 1 }, { 158, 0 }, @@ -126414,218 +130106,193 @@ static const struct { { 155, 2 }, { 162, 0 }, { 162, 2 }, - { 160, 3 }, - { 160, 1 }, - { 164, 3 }, - { 165, 1 }, - { 152, 1 }, - { 152, 1 }, - { 152, 1 }, + { 164, 2 }, { 166, 0 }, - { 166, 1 }, - { 168, 1 }, - { 168, 4 }, - { 168, 6 }, - { 169, 1 }, - { 169, 2 }, - { 170, 1 }, - { 170, 1 }, + { 166, 4 }, + { 166, 6 }, { 167, 2 }, - { 167, 0 }, - { 173, 2 }, - { 173, 2 }, - { 173, 4 }, - { 173, 3 }, - { 173, 3 }, - { 173, 2 }, - { 173, 2 }, - { 173, 3 }, - { 173, 5 }, - { 173, 2 }, - { 173, 4 }, - { 173, 4 }, - { 173, 1 }, - { 173, 2 }, + { 171, 2 }, + { 171, 2 }, + { 171, 4 }, + { 171, 3 }, + { 171, 3 }, + { 171, 2 }, + { 171, 3 }, + { 171, 5 }, + { 171, 2 }, + { 171, 4 }, + { 171, 4 }, + { 171, 1 }, + { 171, 2 }, + { 176, 0 }, + { 176, 1 }, { 178, 0 }, - { 178, 1 }, - { 180, 0 }, + { 178, 2 }, { 180, 2 }, - { 182, 2 }, - { 182, 3 }, - { 182, 3 }, - { 182, 3 }, - { 183, 2 }, - { 183, 2 }, - { 183, 1 }, - { 183, 1 }, - { 183, 2 }, - { 181, 3 }, + { 180, 3 }, + { 180, 3 }, + { 180, 3 }, { 181, 2 }, - { 184, 0 }, - { 184, 2 }, - { 184, 2 }, + { 181, 2 }, + { 181, 1 }, + { 181, 1 }, + { 181, 2 }, + { 179, 3 }, + { 179, 2 }, + { 182, 0 }, + { 182, 2 }, + { 182, 2 }, { 161, 0 }, - { 161, 2 }, - { 185, 3 }, - { 185, 1 }, - { 186, 1 }, - { 186, 0 }, - { 187, 2 }, - { 187, 7 }, - { 187, 5 }, - { 187, 5 }, - { 187, 10 }, + { 184, 1 }, + { 185, 2 }, + { 185, 7 }, + { 185, 5 }, + { 185, 5 }, + { 185, 10 }, + { 188, 0 }, + { 174, 0 }, + { 174, 3 }, { 189, 0 }, - { 189, 1 }, - { 176, 0 }, - { 176, 3 }, - { 190, 0 }, - { 190, 2 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, + { 189, 2 }, + { 190, 1 }, + { 190, 1 }, { 149, 4 }, - { 193, 2 }, - { 193, 0 }, - { 149, 8 }, + { 192, 2 }, + { 192, 0 }, + { 149, 9 }, { 149, 4 }, { 149, 1 }, { 163, 2 }, - { 195, 1 }, - { 195, 3 }, + { 194, 3 }, + { 197, 1 }, + { 197, 2 }, + { 197, 1 }, + { 195, 9 }, + { 206, 4 }, + { 206, 5 }, { 198, 1 }, - { 198, 2 }, { 198, 1 }, - { 196, 9 }, - { 196, 1 }, - { 207, 4 }, - { 207, 5 }, - { 199, 1 }, - { 199, 1 }, - { 199, 0 }, + { 198, 0 }, + { 209, 0 }, + { 199, 3 }, + { 199, 2 }, + { 199, 4 }, { 210, 2 }, { 210, 0 }, - { 200, 3 }, + { 200, 0 }, { 200, 2 }, - { 200, 4 }, - { 211, 2 }, - { 211, 1 }, - { 211, 0 }, - { 201, 0 }, - { 201, 2 }, - { 213, 2 }, - { 213, 0 }, - { 212, 7 }, - { 212, 7 }, - { 212, 7 }, + { 212, 2 }, + { 212, 0 }, + { 211, 7 }, + { 211, 9 }, + { 211, 7 }, + { 211, 7 }, { 159, 0 }, { 159, 2 }, - { 194, 2 }, - { 214, 1 }, - { 214, 2 }, + { 193, 2 }, + { 213, 1 }, + { 213, 2 }, + { 213, 3 }, + { 213, 4 }, + { 215, 2 }, + { 215, 0 }, + { 214, 0 }, { 214, 3 }, - { 214, 4 }, - { 216, 2 }, + { 214, 2 }, + { 216, 4 }, { 216, 0 }, - { 215, 0 }, - { 215, 3 }, - { 215, 2 }, - { 217, 4 }, - { 217, 0 }, - { 205, 0 }, - { 205, 3 }, - { 220, 4 }, - { 220, 2 }, - { 177, 1 }, - { 177, 1 }, - { 177, 0 }, - { 203, 0 }, - { 203, 3 }, { 204, 0 }, - { 204, 2 }, - { 206, 0 }, - { 206, 2 }, - { 206, 4 }, - { 206, 4 }, - { 149, 6 }, + { 204, 3 }, + { 186, 4 }, + { 186, 2 }, + { 175, 1 }, + { 175, 1 }, + { 175, 0 }, { 202, 0 }, - { 202, 2 }, + { 202, 3 }, + { 203, 0 }, + { 203, 2 }, + { 205, 0 }, + { 205, 2 }, + { 205, 4 }, + { 205, 4 }, + { 149, 6 }, + { 201, 0 }, + { 201, 2 }, { 149, 8 }, - { 221, 5 }, - { 221, 3 }, + { 218, 5 }, + { 218, 3 }, { 149, 6 }, { 149, 7 }, - { 222, 2 }, - { 222, 1 }, - { 223, 0 }, - { 223, 3 }, - { 219, 3 }, + { 219, 2 }, { 219, 1 }, - { 175, 1 }, - { 175, 3 }, - { 174, 1 }, - { 175, 1 }, - { 175, 1 }, - { 175, 3 }, - { 175, 5 }, - { 174, 1 }, - { 174, 1 }, - { 175, 1 }, - { 175, 3 }, - { 175, 6 }, - { 175, 5 }, - { 175, 4 }, - { 174, 1 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 175, 3 }, - { 224, 1 }, - { 224, 2 }, - { 175, 3 }, - { 175, 5 }, - { 175, 2 }, - { 175, 3 }, - { 175, 3 }, - { 175, 4 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 175, 2 }, - { 225, 1 }, - { 225, 2 }, - { 175, 5 }, - { 226, 1 }, + { 220, 0 }, + { 220, 3 }, + { 217, 3 }, + { 217, 1 }, + { 173, 3 }, + { 172, 1 }, + { 173, 1 }, + { 173, 1 }, + { 173, 3 }, + { 173, 5 }, + { 172, 1 }, + { 172, 1 }, + { 173, 1 }, + { 173, 3 }, + { 173, 6 }, + { 173, 5 }, + { 173, 4 }, + { 172, 1 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 173, 3 }, + { 221, 1 }, + { 221, 2 }, + { 173, 3 }, + { 173, 5 }, + { 173, 2 }, + { 173, 3 }, + { 173, 3 }, + { 173, 4 }, + { 173, 2 }, + { 173, 2 }, + { 173, 2 }, + { 173, 2 }, + { 222, 1 }, + { 222, 2 }, + { 173, 5 }, + { 223, 1 }, + { 223, 2 }, + { 173, 5 }, + { 173, 3 }, + { 173, 5 }, + { 173, 4 }, + { 173, 4 }, + { 173, 5 }, + { 225, 5 }, + { 225, 4 }, { 226, 2 }, - { 175, 5 }, - { 175, 3 }, - { 175, 5 }, - { 175, 4 }, - { 175, 4 }, - { 175, 5 }, - { 228, 5 }, - { 228, 4 }, - { 229, 2 }, - { 229, 0 }, + { 226, 0 }, + { 224, 1 }, + { 224, 0 }, + { 208, 0 }, + { 207, 3 }, + { 207, 1 }, + { 149, 12 }, { 227, 1 }, { 227, 0 }, - { 209, 1 }, - { 209, 0 }, - { 208, 3 }, - { 208, 1 }, - { 149, 12 }, - { 230, 1 }, - { 230, 0 }, - { 179, 0 }, - { 179, 3 }, - { 188, 5 }, - { 188, 3 }, - { 231, 0 }, - { 231, 2 }, + { 177, 0 }, + { 177, 3 }, + { 187, 5 }, + { 187, 3 }, + { 228, 0 }, + { 228, 2 }, { 149, 4 }, { 149, 1 }, { 149, 2 }, @@ -126634,77 +130301,113 @@ static const struct { { 149, 6 }, { 149, 5 }, { 149, 6 }, + { 169, 2 }, + { 170, 2 }, + { 149, 5 }, + { 230, 11 }, { 232, 1 }, { 232, 1 }, - { 232, 1 }, - { 232, 1 }, - { 232, 1 }, - { 171, 2 }, - { 171, 1 }, - { 172, 2 }, - { 149, 5 }, - { 233, 11 }, - { 235, 1 }, - { 235, 1 }, - { 235, 2 }, + { 232, 2 }, + { 232, 0 }, + { 233, 1 }, + { 233, 1 }, + { 233, 3 }, { 235, 0 }, - { 236, 1 }, - { 236, 1 }, - { 236, 3 }, - { 237, 0 }, + { 235, 2 }, + { 231, 3 }, + { 231, 2 }, { 237, 3 }, - { 238, 0 }, + { 238, 3 }, { 238, 2 }, - { 234, 3 }, - { 234, 2 }, - { 240, 1 }, - { 240, 3 }, - { 241, 0 }, - { 241, 3 }, - { 241, 2 }, - { 239, 7 }, - { 239, 5 }, - { 239, 5 }, - { 239, 1 }, - { 175, 4 }, - { 175, 6 }, - { 192, 1 }, - { 192, 1 }, - { 192, 1 }, + { 236, 7 }, + { 236, 5 }, + { 236, 5 }, + { 236, 1 }, + { 173, 4 }, + { 173, 6 }, + { 191, 1 }, + { 191, 1 }, + { 191, 1 }, { 149, 4 }, { 149, 6 }, { 149, 3 }, - { 243, 0 }, - { 243, 2 }, - { 242, 1 }, - { 242, 0 }, + { 240, 0 }, + { 240, 2 }, { 149, 1 }, { 149, 3 }, { 149, 1 }, { 149, 3 }, { 149, 6 }, - { 149, 6 }, - { 244, 1 }, - { 245, 0 }, - { 245, 1 }, + { 149, 7 }, + { 241, 1 }, { 149, 1 }, { 149, 4 }, - { 246, 8 }, + { 243, 8 }, + { 245, 0 }, + { 246, 1 }, + { 246, 3 }, { 247, 1 }, - { 247, 3 }, + { 196, 0 }, + { 196, 2 }, + { 196, 3 }, + { 249, 6 }, + { 249, 8 }, + { 144, 1 }, + { 145, 2 }, + { 145, 1 }, + { 146, 1 }, + { 146, 3 }, + { 147, 0 }, + { 151, 0 }, + { 151, 1 }, + { 151, 2 }, + { 153, 1 }, + { 153, 0 }, + { 149, 2 }, + { 160, 4 }, + { 160, 2 }, + { 152, 1 }, + { 152, 1 }, + { 152, 1 }, + { 166, 1 }, + { 167, 1 }, + { 168, 1 }, + { 168, 1 }, + { 165, 2 }, + { 165, 0 }, + { 171, 2 }, + { 161, 2 }, + { 183, 3 }, + { 183, 1 }, + { 184, 0 }, + { 188, 1 }, + { 190, 1 }, + { 194, 1 }, + { 195, 1 }, + { 209, 2 }, + { 210, 1 }, + { 173, 1 }, + { 208, 1 }, + { 229, 1 }, + { 229, 1 }, + { 229, 1 }, + { 229, 1 }, + { 229, 1 }, + { 169, 1 }, + { 234, 0 }, + { 234, 3 }, + { 237, 1 }, + { 238, 0 }, + { 239, 1 }, + { 239, 0 }, + { 242, 0 }, + { 242, 1 }, + { 244, 1 }, + { 244, 3 }, + { 245, 2 }, { 248, 0 }, + { 248, 4 }, { 248, 2 }, - { 249, 1 }, - { 249, 3 }, - { 250, 1 }, - { 251, 0 }, - { 251, 4 }, - { 251, 2 }, - { 197, 0 }, - { 197, 2 }, - { 197, 3 }, - { 252, 6 }, - { 252, 8 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -126715,40 +130418,46 @@ static void yy_accept(yyParser*); /* Forward Declaration */ */ static void yy_reduce( yyParser *yypParser, /* The parser */ - int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ int yyact; /* The next action */ - YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ sqlite3ParserARG_FETCH; yymsp = &yypParser->yystack[yypParser->yyidx]; #ifndef NDEBUG - if( yyTraceFILE && yyruleno>=0 - && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ - yygotominor = yyzerominor; - + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfo[yyruleno].nrhs==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yyidx>=YYSTACKDEPTH-1 ){ + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yyidx>=yypParser->yystksz-1 ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz-1 ){ + yyStackOverflow(yypParser); + return; + } + } +#endif + } switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -126759,327 +130468,287 @@ static void yy_reduce( ** #line <lineno> <thisfile> ** break; */ - case 5: /* explain ::= */ -{ sqlite3BeginParse(pParse, 0); } - break; - case 6: /* explain ::= EXPLAIN */ -{ sqlite3BeginParse(pParse, 1); } +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* explain ::= EXPLAIN */ +{ pParse->explain = 1; } break; - case 7: /* explain ::= EXPLAIN QUERY PLAN */ -{ sqlite3BeginParse(pParse, 2); } + case 1: /* explain ::= EXPLAIN QUERY PLAN */ +{ pParse->explain = 2; } break; - case 8: /* cmdx ::= cmd */ + case 2: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } break; - case 9: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy328);} + case 3: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy392);} break; - case 13: /* transtype ::= */ -{yygotominor.yy328 = TK_DEFERRED;} + case 4: /* transtype ::= */ +{yymsp[1].minor.yy392 = TK_DEFERRED;} break; - case 14: /* transtype ::= DEFERRED */ - case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); - case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); - case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); - case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); -{yygotominor.yy328 = yymsp[0].major;} + case 5: /* transtype ::= DEFERRED */ + case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); + case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); +{yymsp[0].minor.yy392 = yymsp[0].major; /*A-overwrites-X*/} break; - case 17: /* cmd ::= COMMIT trans_opt */ - case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); + case 8: /* cmd ::= COMMIT trans_opt */ + case 9: /* cmd ::= END trans_opt */ yytestcase(yyruleno==9); {sqlite3CommitTransaction(pParse);} break; - case 19: /* cmd ::= ROLLBACK trans_opt */ + case 10: /* cmd ::= ROLLBACK trans_opt */ {sqlite3RollbackTransaction(pParse);} break; - case 22: /* cmd ::= SAVEPOINT nm */ + case 11: /* cmd ::= SAVEPOINT nm */ { sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); } break; - case 23: /* cmd ::= RELEASE savepoint_opt nm */ + case 12: /* cmd ::= RELEASE savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); } break; - case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + case 13: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); } break; - case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + case 14: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy328,0,0,yymsp[-2].minor.yy328); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy392,0,0,yymsp[-2].minor.yy392); } break; - case 27: /* createkw ::= CREATE */ -{ - pParse->db->lookaside.bEnabled = 0; - yygotominor.yy0 = yymsp[0].minor.yy0; -} + case 15: /* createkw ::= CREATE */ +{disableLookaside(pParse);} + break; + case 16: /* ifnotexists ::= */ + case 19: /* temp ::= */ yytestcase(yyruleno==19); + case 22: /* table_options ::= */ yytestcase(yyruleno==22); + case 42: /* autoinc ::= */ yytestcase(yyruleno==42); + case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); + case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); + case 76: /* ifexists ::= */ yytestcase(yyruleno==76); + case 90: /* distinct ::= */ yytestcase(yyruleno==90); + case 209: /* collate ::= */ yytestcase(yyruleno==209); +{yymsp[1].minor.yy392 = 0;} break; - case 28: /* ifnotexists ::= */ - case 31: /* temp ::= */ yytestcase(yyruleno==31); - case 68: /* autoinc ::= */ yytestcase(yyruleno==68); - case 81: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==81); - case 83: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==83); - case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); - case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); - case 108: /* ifexists ::= */ yytestcase(yyruleno==108); - case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); - case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); -{yygotominor.yy328 = 0;} + case 17: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy392 = 1;} break; - case 29: /* ifnotexists ::= IF NOT EXISTS */ - case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); - case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); - case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); - case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); - case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); - case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); -{yygotominor.yy328 = 1;} + case 18: /* temp ::= TEMP */ + case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); +{yymsp[0].minor.yy392 = 1;} break; - case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ + case 20: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy186,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy392,0); } break; - case 33: /* create_table_args ::= AS select */ + case 21: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy3); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy159); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); } break; - case 34: /* table_options ::= */ -{yygotominor.yy186 = 0;} - break; - case 35: /* table_options ::= WITHOUT nm */ + case 23: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yygotominor.yy186 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy392 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yygotominor.yy186 = 0; + yymsp[-1].minor.yy392 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } break; - case 38: /* column ::= columnid type carglist */ -{ - yygotominor.yy0.z = yymsp[-2].minor.yy0.z; - yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; -} - break; - case 39: /* columnid ::= nm */ -{ - sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); - yygotominor.yy0 = yymsp[0].minor.yy0; - pParse->constraintName.n = 0; -} - break; - case 40: /* nm ::= ID|INDEXED */ - case 41: /* nm ::= STRING */ yytestcase(yyruleno==41); - case 42: /* nm ::= JOIN_KW */ yytestcase(yyruleno==42); - case 45: /* typetoken ::= typename */ yytestcase(yyruleno==45); - case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); - case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); - case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); - case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); - case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); - case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); - case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); - case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); - case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); - case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); - case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); - case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); - case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); - case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); - case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); -{yygotominor.yy0 = yymsp[0].minor.yy0;} + case 24: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; - case 44: /* type ::= typetoken */ -{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} + case 25: /* typetoken ::= */ + case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); + case 96: /* as ::= */ yytestcase(yyruleno==96); +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; - case 46: /* typetoken ::= typename LP signed RP */ + case 26: /* typetoken ::= typename LP signed RP */ { - yygotominor.yy0.z = yymsp[-3].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); + yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); } break; - case 47: /* typetoken ::= typename LP signed COMMA signed RP */ + case 27: /* typetoken ::= typename LP signed COMMA signed RP */ { - yygotominor.yy0.z = yymsp[-5].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); + yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); } break; - case 49: /* typename ::= typename ID|STRING */ -{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + case 28: /* typename ::= typename ID|STRING */ +{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} break; - case 54: /* ccons ::= CONSTRAINT nm */ - case 92: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==92); + case 29: /* ccons ::= CONSTRAINT nm */ + case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); {pParse->constraintName = yymsp[0].minor.yy0;} break; - case 55: /* ccons ::= DEFAULT term */ - case 57: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==57); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy346);} + case 30: /* ccons ::= DEFAULT term */ + case 32: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==32); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy342);} break; - case 56: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy346);} + case 31: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy342);} break; - case 58: /* ccons ::= DEFAULT MINUS term */ + case 33: /* ccons ::= DEFAULT MINUS term */ { ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy346.pExpr, 0, 0); + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy342.pExpr, 0, 0); v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy346.zEnd; + v.zEnd = yymsp[0].minor.yy342.zEnd; sqlite3AddDefaultValue(pParse,&v); } break; - case 59: /* ccons ::= DEFAULT ID|INDEXED */ + case 34: /* ccons ::= DEFAULT ID|INDEXED */ { ExprSpan v; - spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); + spanExpr(&v, pParse, TK_STRING, yymsp[0].minor.yy0); sqlite3AddDefaultValue(pParse,&v); } break; - case 61: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy328);} + case 35: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy392);} break; - case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy328,yymsp[0].minor.yy328,yymsp[-2].minor.yy328);} + case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy392,yymsp[0].minor.yy392,yymsp[-2].minor.yy392);} break; - case 63: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy328,0,0,0,0);} + case 37: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy392,0,0,0,0);} break; - case 64: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} + case 38: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy342.pExpr);} break; - case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} + case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy442,yymsp[0].minor.yy392);} break; - case 66: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy328);} + case 40: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy392);} break; - case 67: /* ccons ::= COLLATE ID|STRING */ + case 41: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; - case 70: /* refargs ::= */ -{ yygotominor.yy328 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 44: /* refargs ::= */ +{ yymsp[1].minor.yy392 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 45: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy392 = (yymsp[-1].minor.yy392 & ~yymsp[0].minor.yy207.mask) | yymsp[0].minor.yy207.value; } + break; + case 46: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy207.value = 0; yymsp[-1].minor.yy207.mask = 0x000000; } break; - case 71: /* refargs ::= refargs refarg */ -{ yygotominor.yy328 = (yymsp[-1].minor.yy328 & ~yymsp[0].minor.yy429.mask) | yymsp[0].minor.yy429.value; } + case 47: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy207.value = 0; yymsp[-2].minor.yy207.mask = 0x000000; } break; - case 72: /* refarg ::= MATCH nm */ - case 73: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==73); -{ yygotominor.yy429.value = 0; yygotominor.yy429.mask = 0x000000; } + case 48: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy392; yymsp[-2].minor.yy207.mask = 0x0000ff; } break; - case 74: /* refarg ::= ON DELETE refact */ -{ yygotominor.yy429.value = yymsp[0].minor.yy328; yygotominor.yy429.mask = 0x0000ff; } + case 49: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy392<<8; yymsp[-2].minor.yy207.mask = 0x00ff00; } break; - case 75: /* refarg ::= ON UPDATE refact */ -{ yygotominor.yy429.value = yymsp[0].minor.yy328<<8; yygotominor.yy429.mask = 0x00ff00; } + case 50: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy392 = OE_SetNull; /* EV: R-33326-45252 */} break; - case 76: /* refact ::= SET NULL */ -{ yygotominor.yy328 = OE_SetNull; /* EV: R-33326-45252 */} + case 51: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy392 = OE_SetDflt; /* EV: R-33326-45252 */} break; - case 77: /* refact ::= SET DEFAULT */ -{ yygotominor.yy328 = OE_SetDflt; /* EV: R-33326-45252 */} + case 52: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy392 = OE_Cascade; /* EV: R-33326-45252 */} break; - case 78: /* refact ::= CASCADE */ -{ yygotominor.yy328 = OE_Cascade; /* EV: R-33326-45252 */} + case 53: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy392 = OE_Restrict; /* EV: R-33326-45252 */} break; - case 79: /* refact ::= RESTRICT */ -{ yygotominor.yy328 = OE_Restrict; /* EV: R-33326-45252 */} + case 54: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy392 = OE_None; /* EV: R-33326-45252 */} break; - case 80: /* refact ::= NO ACTION */ -{ yygotominor.yy328 = OE_None; /* EV: R-33326-45252 */} + case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy392 = 0;} break; - case 82: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 98: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==98); - case 100: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==100); - case 103: /* resolvetype ::= raisetype */ yytestcase(yyruleno==103); -{yygotominor.yy328 = yymsp[0].minor.yy328;} + case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); + case 142: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==142); +{yymsp[-1].minor.yy392 = yymsp[0].minor.yy392;} break; - case 86: /* conslist_opt ::= */ -{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); + case 183: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==183); + case 186: /* in_op ::= NOT IN */ yytestcase(yyruleno==186); + case 210: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==210); +{yymsp[-1].minor.yy392 = 1;} break; - case 87: /* conslist_opt ::= COMMA conslist */ -{yygotominor.yy0 = yymsp[-1].minor.yy0;} + case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy392 = 0;} break; - case 90: /* tconscomma ::= COMMA */ + case 61: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} + case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy442,yymsp[0].minor.yy392,yymsp[-2].minor.yy392,0);} break; - case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} + case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy442,yymsp[0].minor.yy392,0,0,0,0);} break; - case 95: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} + case 65: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy342.pExpr);} break; - case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy442, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy442, yymsp[-1].minor.yy392); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy392); } break; - case 99: /* onconf ::= */ -{yygotominor.yy328 = OE_Default;} + case 68: /* onconf ::= */ + case 70: /* orconf ::= */ yytestcase(yyruleno==70); +{yymsp[1].minor.yy392 = OE_Default;} break; - case 101: /* orconf ::= */ -{yygotominor.yy186 = OE_Default;} + case 69: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy392 = yymsp[0].minor.yy392;} break; - case 102: /* orconf ::= OR resolvetype */ -{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;} + case 72: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy392 = OE_Ignore;} break; - case 104: /* resolvetype ::= IGNORE */ -{yygotominor.yy328 = OE_Ignore;} + case 73: /* resolvetype ::= REPLACE */ + case 143: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==143); +{yymsp[0].minor.yy392 = OE_Replace;} break; - case 105: /* resolvetype ::= REPLACE */ -{yygotominor.yy328 = OE_Replace;} - break; - case 106: /* cmd ::= DROP TABLE ifexists fullname */ + case 74: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); + sqlite3DropTable(pParse, yymsp[0].minor.yy347, 0, yymsp[-1].minor.yy392); } break; - case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy442, yymsp[0].minor.yy159, yymsp[-7].minor.yy392, yymsp[-5].minor.yy392); } break; - case 110: /* cmd ::= DROP VIEW ifexists fullname */ + case 78: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328); + sqlite3DropTable(pParse, yymsp[0].minor.yy347, 1, yymsp[-1].minor.yy392); } break; - case 111: /* cmd ::= select */ + case 79: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy3, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); + sqlite3Select(pParse, yymsp[0].minor.yy159, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); } break; - case 112: /* select ::= with selectnowith */ + case 80: /* select ::= with selectnowith */ { - Select *p = yymsp[0].minor.yy3; + Select *p = yymsp[0].minor.yy159; if( p ){ - p->pWith = yymsp[-1].minor.yy59; + p->pWith = yymsp[-1].minor.yy331; parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy331); } - yygotominor.yy3 = p; + yymsp[-1].minor.yy159 = p; /*A-overwrites-W*/ } break; - case 113: /* selectnowith ::= oneselect */ - case 119: /* oneselect ::= values */ yytestcase(yyruleno==119); -{yygotominor.yy3 = yymsp[0].minor.yy3;} - break; - case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ + case 81: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy3; - Select *pLhs = yymsp[-2].minor.yy3; + Select *pRhs = yymsp[0].minor.yy159; + Select *pLhs = yymsp[-2].minor.yy159; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -127089,23 +130758,30 @@ static void yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy328; + pRhs->op = (u8)yymsp[-1].minor.yy392; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy392!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yygotominor.yy3 = pRhs; + yymsp[-2].minor.yy159 = pRhs; } break; - case 116: /* multiselect_op ::= UNION ALL */ -{yygotominor.yy328 = TK_ALL;} + case 82: /* multiselect_op ::= UNION */ + case 84: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==84); +{yymsp[0].minor.yy392 = yymsp[0].major; /*A-overwrites-OP*/} + break; + case 83: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy392 = TK_ALL;} break; - case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + case 85: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset); +#if SELECTTRACE_ENABLED + Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ +#endif + yymsp[-8].minor.yy159 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy442,yymsp[-5].minor.yy347,yymsp[-4].minor.yy122,yymsp[-3].minor.yy442,yymsp[-2].minor.yy122,yymsp[-1].minor.yy442,yymsp[-7].minor.yy392,yymsp[0].minor.yy64.pLimit,yymsp[0].minor.yy64.pOffset); #if SELECTTRACE_ENABLED /* Populate the Select.zSelName[] string that is used to help with ** query planner debugging, to differentiate between multiple Select @@ -127116,447 +130792,448 @@ static void yy_reduce( ** comment to be the zSelName value. Otherwise, the label is #N where ** is an integer that is incremented with each SELECT statement seen. */ - if( yygotominor.yy3!=0 ){ - const char *z = yymsp[-8].minor.yy0.z+6; + if( yymsp[-8].minor.yy159!=0 ){ + const char *z = s.z+6; int i; - sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "#%d", + sqlite3_snprintf(sizeof(yymsp[-8].minor.yy159->zSelName), yymsp[-8].minor.yy159->zSelName, "#%d", ++pParse->nSelect); while( z[0]==' ' ) z++; if( z[0]=='/' && z[1]=='*' ){ z += 2; while( z[0]==' ' ) z++; for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "%.*s", i, z); + sqlite3_snprintf(sizeof(yymsp[-8].minor.yy159->zSelName), yymsp[-8].minor.yy159->zSelName, "%.*s", i, z); } } #endif /* SELECTRACE_ENABLED */ } break; - case 120: /* values ::= VALUES LP nexprlist RP */ + case 86: /* values ::= VALUES LP nexprlist RP */ { - yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); + yymsp[-3].minor.yy159 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy442,0,0,0,0,0,SF_Values,0,0); } break; - case 121: /* values ::= values COMMA LP exprlist RP */ + case 87: /* values ::= values COMMA LP exprlist RP */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy3; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); + Select *pRight, *pLeft = yymsp[-4].minor.yy159; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy442,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; - pLeft = yymsp[-4].minor.yy3; pRight->pPrior = pLeft; - yygotominor.yy3 = pRight; + yymsp[-4].minor.yy159 = pRight; }else{ - yygotominor.yy3 = pLeft; + yymsp[-4].minor.yy159 = pLeft; } } break; - case 122: /* distinct ::= DISTINCT */ -{yygotominor.yy381 = SF_Distinct;} - break; - case 123: /* distinct ::= ALL */ -{yygotominor.yy381 = SF_All;} - break; - case 124: /* distinct ::= */ -{yygotominor.yy381 = 0;} + case 88: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy392 = SF_Distinct;} break; - case 125: /* sclp ::= selcollist COMMA */ - case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); -{yygotominor.yy14 = yymsp[-1].minor.yy14;} + case 89: /* distinct ::= ALL */ +{yymsp[0].minor.yy392 = SF_All;} break; - case 126: /* sclp ::= */ - case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); - case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); - case 236: /* exprlist ::= */ yytestcase(yyruleno==236); - case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); -{yygotominor.yy14 = 0;} + case 91: /* sclp ::= */ + case 119: /* orderby_opt ::= */ yytestcase(yyruleno==119); + case 126: /* groupby_opt ::= */ yytestcase(yyruleno==126); + case 199: /* exprlist ::= */ yytestcase(yyruleno==199); + case 205: /* eidlist_opt ::= */ yytestcase(yyruleno==205); +{yymsp[1].minor.yy442 = 0;} break; - case 127: /* selcollist ::= sclp expr as */ + case 92: /* selcollist ::= sclp expr as */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346); + yymsp[-2].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy442, yymsp[-1].minor.yy342.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-2].minor.yy442, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-2].minor.yy442,&yymsp[-1].minor.yy342); } break; - case 128: /* selcollist ::= sclp STAR */ + case 93: /* selcollist ::= sclp STAR */ { - Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p); + Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + yymsp[-1].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy442, p); } break; - case 129: /* selcollist ::= sclp nm DOT STAR */ + case 94: /* selcollist ::= sclp nm DOT STAR */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &yymsp[0].minor.yy0); Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot); + yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442, pDot); } break; - case 132: /* as ::= */ -{yygotominor.yy0.n = 0;} + case 95: /* as ::= AS nm */ + case 106: /* dbnm ::= DOT nm */ yytestcase(yyruleno==106); + case 219: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==219); + case 220: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==220); +{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 133: /* from ::= */ -{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));} + case 97: /* from ::= */ +{yymsp[1].minor.yy347 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy347));} break; - case 134: /* from ::= FROM seltablist */ + case 98: /* from ::= FROM seltablist */ { - yygotominor.yy65 = yymsp[0].minor.yy65; - sqlite3SrcListShiftJoinType(yygotominor.yy65); + yymsp[-1].minor.yy347 = yymsp[0].minor.yy347; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy347); } break; - case 135: /* stl_prefix ::= seltablist joinop */ + case 99: /* stl_prefix ::= seltablist joinop */ { - yygotominor.yy65 = yymsp[-1].minor.yy65; - if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; + if( ALWAYS(yymsp[-1].minor.yy347 && yymsp[-1].minor.yy347->nSrc>0) ) yymsp[-1].minor.yy347->a[yymsp[-1].minor.yy347->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy392; } break; - case 136: /* stl_prefix ::= */ -{yygotominor.yy65 = 0;} + case 100: /* stl_prefix ::= */ +{yymsp[1].minor.yy347 = 0;} + break; + case 101: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy347, &yymsp[-2].minor.yy0); +} break; - case 137: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 102: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ { - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); - sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); + yymsp[-8].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy347,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy347, yymsp[-4].minor.yy442); } break; - case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 103: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy159,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); } break; - case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 104: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ - yygotominor.yy65 = yymsp[-4].minor.yy65; - }else if( yymsp[-4].minor.yy65->nSrc==1 ){ - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); - if( yygotominor.yy65 ){ - struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy65->a; + if( yymsp[-6].minor.yy347==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy122==0 && yymsp[0].minor.yy180==0 ){ + yymsp[-6].minor.yy347 = yymsp[-4].minor.yy347; + }else if( yymsp[-4].minor.yy347->nSrc==1 ){ + yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + if( yymsp[-6].minor.yy347 ){ + struct SrcList_item *pNew = &yymsp[-6].minor.yy347->a[yymsp[-6].minor.yy347->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy347->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65); + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy347); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0); - yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy347); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy347,0,0,0,0,SF_NestedFrom,0,0); + yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); } } break; - case 140: /* dbnm ::= */ - case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); -{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} + case 105: /* dbnm ::= */ + case 114: /* indexed_opt ::= */ yytestcase(yyruleno==114); +{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 142: /* fullname ::= nm dbnm */ -{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + case 107: /* fullname ::= nm dbnm */ +{yymsp[-1].minor.yy347 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 143: /* joinop ::= COMMA|JOIN */ -{ yygotominor.yy328 = JT_INNER; } + case 108: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy392 = JT_INNER; } break; - case 144: /* joinop ::= JOIN_KW JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + case 109: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 145: /* joinop ::= JOIN_KW nm JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } + case 110: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 146: /* joinop ::= JOIN_KW nm nm JOIN */ -{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } + case 111: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 147: /* on_opt ::= ON expr */ - case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); - case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); - case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); - case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); -{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} + case 112: /* on_opt ::= ON expr */ + case 129: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==129); + case 136: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==136); + case 195: /* case_else ::= ELSE expr */ yytestcase(yyruleno==195); +{yymsp[-1].minor.yy122 = yymsp[0].minor.yy342.pExpr;} break; - case 148: /* on_opt ::= */ - case 163: /* having_opt ::= */ yytestcase(yyruleno==163); - case 170: /* where_opt ::= */ yytestcase(yyruleno==170); - case 232: /* case_else ::= */ yytestcase(yyruleno==232); - case 234: /* case_operand ::= */ yytestcase(yyruleno==234); -{yygotominor.yy132 = 0;} + case 113: /* on_opt ::= */ + case 128: /* having_opt ::= */ yytestcase(yyruleno==128); + case 135: /* where_opt ::= */ yytestcase(yyruleno==135); + case 196: /* case_else ::= */ yytestcase(yyruleno==196); + case 198: /* case_operand ::= */ yytestcase(yyruleno==198); +{yymsp[1].minor.yy122 = 0;} break; - case 151: /* indexed_opt ::= NOT INDEXED */ -{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} + case 115: /* indexed_opt ::= INDEXED BY nm */ +{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 152: /* using_opt ::= USING LP idlist RP */ - case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); -{yygotominor.yy408 = yymsp[-1].minor.yy408;} + case 116: /* indexed_opt ::= NOT INDEXED */ +{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 153: /* using_opt ::= */ - case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); -{yygotominor.yy408 = 0;} + case 117: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy180 = yymsp[-1].minor.yy180;} break; - case 155: /* orderby_opt ::= ORDER BY sortlist */ - case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); - case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); -{yygotominor.yy14 = yymsp[0].minor.yy14;} + case 118: /* using_opt ::= */ + case 144: /* idlist_opt ::= */ yytestcase(yyruleno==144); +{yymsp[1].minor.yy180 = 0;} break; - case 156: /* sortlist ::= sortlist COMMA expr sortorder */ + case 120: /* orderby_opt ::= ORDER BY sortlist */ + case 127: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==127); +{yymsp[-2].minor.yy442 = yymsp[0].minor.yy442;} + break; + case 121: /* sortlist ::= sortlist COMMA expr sortorder */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442,yymsp[-1].minor.yy342.pExpr); + sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy442,yymsp[0].minor.yy392); } break; - case 157: /* sortlist ::= expr sortorder */ + case 122: /* sortlist ::= expr sortorder */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-1].minor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy342.pExpr); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy442,yymsp[0].minor.yy392); } break; - case 158: /* sortorder ::= ASC */ - case 160: /* sortorder ::= */ yytestcase(yyruleno==160); -{yygotominor.yy328 = SQLITE_SO_ASC;} + case 123: /* sortorder ::= ASC */ +{yymsp[0].minor.yy392 = SQLITE_SO_ASC;} + break; + case 124: /* sortorder ::= DESC */ +{yymsp[0].minor.yy392 = SQLITE_SO_DESC;} break; - case 159: /* sortorder ::= DESC */ -{yygotominor.yy328 = SQLITE_SO_DESC;} + case 125: /* sortorder ::= */ +{yymsp[1].minor.yy392 = SQLITE_SO_UNDEFINED;} break; - case 165: /* limit_opt ::= */ -{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} + case 130: /* limit_opt ::= */ +{yymsp[1].minor.yy64.pLimit = 0; yymsp[1].minor.yy64.pOffset = 0;} break; - case 166: /* limit_opt ::= LIMIT expr */ -{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} + case 131: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy64.pLimit = yymsp[0].minor.yy342.pExpr; yymsp[-1].minor.yy64.pOffset = 0;} break; - case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} + case 132: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy64.pLimit = yymsp[-2].minor.yy342.pExpr; yymsp[-3].minor.yy64.pOffset = yymsp[0].minor.yy342.pExpr;} break; - case 168: /* limit_opt ::= LIMIT expr COMMA expr */ -{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} + case 133: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy64.pOffset = yymsp[-2].minor.yy342.pExpr; yymsp[-3].minor.yy64.pLimit = yymsp[0].minor.yy342.pExpr;} break; - case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ + case 134: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ { - sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); + sqlite3WithPush(pParse, yymsp[-5].minor.yy331, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy347, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy347,yymsp[0].minor.yy122); } break; - case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 137: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ { - sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); + sqlite3WithPush(pParse, yymsp[-7].minor.yy331, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy347, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy442,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy347,yymsp[-1].minor.yy442,yymsp[0].minor.yy122,yymsp[-5].minor.yy392); } break; - case 173: /* setlist ::= setlist COMMA nm EQ expr */ + case 138: /* setlist ::= setlist COMMA nm EQ expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy442, yymsp[0].minor.yy342.pExpr); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy442, &yymsp[-2].minor.yy0, 1); } break; - case 174: /* setlist ::= nm EQ expr */ + case 139: /* setlist ::= nm EQ expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy442 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy342.pExpr); + sqlite3ExprListSetName(pParse, yylhsminor.yy442, &yymsp[-2].minor.yy0, 1); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ + case 140: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ { - sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); - sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); + sqlite3WithPush(pParse, yymsp[-5].minor.yy331, 1); + sqlite3Insert(pParse, yymsp[-2].minor.yy347, yymsp[0].minor.yy159, yymsp[-1].minor.yy180, yymsp[-4].minor.yy392); } break; - case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ + case 141: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ { - sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); - sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); + sqlite3WithPush(pParse, yymsp[-6].minor.yy331, 1); + sqlite3Insert(pParse, yymsp[-3].minor.yy347, 0, yymsp[-2].minor.yy180, yymsp[-5].minor.yy392); } break; - case 177: /* insert_cmd ::= INSERT orconf */ -{yygotominor.yy186 = yymsp[0].minor.yy186;} - break; - case 178: /* insert_cmd ::= REPLACE */ -{yygotominor.yy186 = OE_Replace;} - break; - case 181: /* idlist ::= idlist COMMA nm */ -{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} + case 145: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy180 = yymsp[-1].minor.yy180;} break; - case 182: /* idlist ::= nm */ -{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} + case 146: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy180 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy180,&yymsp[0].minor.yy0);} break; - case 183: /* expr ::= term */ -{yygotominor.yy346 = yymsp[0].minor.yy346;} + case 147: /* idlist ::= nm */ +{yymsp[0].minor.yy180 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 184: /* expr ::= LP expr RP */ -{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} + case 148: /* expr ::= LP expr RP */ +{spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ yymsp[-2].minor.yy342.pExpr = yymsp[-1].minor.yy342.pExpr;} break; - case 185: /* term ::= NULL */ - case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); - case 191: /* term ::= STRING */ yytestcase(yyruleno==191); -{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} + case 149: /* term ::= NULL */ + case 154: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==154); + case 155: /* term ::= STRING */ yytestcase(yyruleno==155); +{spanExpr(&yymsp[0].minor.yy342,pParse,yymsp[0].major,yymsp[0].minor.yy0);/*A-overwrites-X*/} break; - case 186: /* expr ::= ID|INDEXED */ - case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); -{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} + case 150: /* expr ::= ID|INDEXED */ + case 151: /* expr ::= JOIN_KW */ yytestcase(yyruleno==151); +{spanExpr(&yymsp[0].minor.yy342,pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 188: /* expr ::= nm DOT nm */ + case 152: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-2].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); } break; - case 189: /* expr ::= nm DOT nm DOT nm */ + case 153: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + spanSet(&yymsp[-4].minor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); } break; - case 192: /* expr ::= VARIABLE */ + case 156: /* expr ::= VARIABLE */ { - if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ + Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ + if( t.n>=2 && t.z[0]=='#' && sqlite3Isdigit(t.z[1]) ){ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ + spanSet(&yymsp[0].minor.yy342, &t, &t); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = 0; + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + yymsp[0].minor.yy342.pExpr = 0; }else{ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); - if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable); + yymsp[0].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &t); + if( yymsp[0].minor.yy342.pExpr ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy342.pExpr->iTable); } }else{ - spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr); + spanExpr(&yymsp[0].minor.yy342, pParse, TK_VARIABLE, t); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy342.pExpr); } - spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 193: /* expr ::= expr COLLATE ID|STRING */ + case 157: /* expr ::= expr COLLATE ID|STRING */ { - yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0, 1); - yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-2].minor.yy342.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy342.pExpr, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 194: /* expr ::= CAST LP expr AS typetoken RP */ + case 158: /* expr ::= CAST LP expr AS typetoken RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); + spanSet(&yymsp[-5].minor.yy342,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-5].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy342.pExpr, 0, &yymsp[-1].minor.yy0); } break; - case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 159: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { - if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + if( yymsp[-1].minor.yy442 && yymsp[-1].minor.yy442->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy381==SF_Distinct && yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->flags |= EP_Distinct; + yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy442, &yymsp[-4].minor.yy0); + spanSet(&yylhsminor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy392==SF_Distinct && yylhsminor.yy342.pExpr ){ + yylhsminor.yy342.pExpr->flags |= EP_Distinct; } } + yymsp[-4].minor.yy342 = yylhsminor.yy342; break; - case 196: /* expr ::= ID|INDEXED LP STAR RP */ + case 160: /* expr ::= ID|INDEXED LP STAR RP */ { - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yylhsminor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } + yymsp[-3].minor.yy342 = yylhsminor.yy342; break; - case 197: /* term ::= CTIME_KW */ + case 161: /* term ::= CTIME_KW */ { - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); - spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + spanSet(&yylhsminor.yy342, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy342 = yylhsminor.yy342; break; - case 198: /* expr ::= expr AND expr */ - case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); -{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} + case 162: /* expr ::= expr AND expr */ + case 163: /* expr ::= expr OR expr */ yytestcase(yyruleno==163); + case 164: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==164); + case 165: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==165); + case 166: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==166); + case 167: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==167); + case 168: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==168); + case 169: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==169); +{spanBinaryExpr(pParse,yymsp[-1].major,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy342);} break; - case 206: /* likeop ::= LIKE_KW|MATCH */ -{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} + case 170: /* likeop ::= LIKE_KW|MATCH */ +{yymsp[0].minor.yy318.eOperator = yymsp[0].minor.yy0; yymsp[0].minor.yy318.bNot = 0;/*A-overwrites-X*/} break; - case 207: /* likeop ::= NOT LIKE_KW|MATCH */ -{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} + case 171: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy318.eOperator = yymsp[0].minor.yy0; yymsp[-1].minor.yy318.bNot = 1;} break; - case 208: /* expr ::= expr likeop expr */ + case 172: /* expr ::= expr likeop expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator); - if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; - if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy342.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy342.pExpr); + yymsp[-2].minor.yy342.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy318.eOperator); + exprNot(pParse, yymsp[-1].minor.yy318.bNot, &yymsp[-2].minor.yy342); + yymsp[-2].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; + if( yymsp[-2].minor.yy342.pExpr ) yymsp[-2].minor.yy342.pExpr->flags |= EP_InfixFunc; } break; - case 209: /* expr ::= expr likeop expr ESCAPE expr */ + case 173: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator); - if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; - if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy342.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy342.pExpr); + yymsp[-4].minor.yy342.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy318.eOperator); + exprNot(pParse, yymsp[-3].minor.yy318.bNot, &yymsp[-4].minor.yy342); + yymsp[-4].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; + if( yymsp[-4].minor.yy342.pExpr ) yymsp[-4].minor.yy342.pExpr->flags |= EP_InfixFunc; } break; - case 210: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} + case 174: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(pParse,yymsp[0].major,&yymsp[-1].minor.yy342,&yymsp[0].minor.yy0);} break; - case 211: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} + case 175: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(pParse,TK_NOTNULL,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy0);} break; - case 212: /* expr ::= expr IS expr */ + case 176: /* expr ::= expr IS expr */ { - spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); + spanBinaryExpr(pParse,TK_IS,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy342); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy342.pExpr, yymsp[-2].minor.yy342.pExpr, TK_ISNULL); } break; - case 213: /* expr ::= expr IS NOT expr */ + case 177: /* expr ::= expr IS NOT expr */ { - spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); + spanBinaryExpr(pParse,TK_ISNOT,&yymsp[-3].minor.yy342,&yymsp[0].minor.yy342); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy342.pExpr, yymsp[-3].minor.yy342.pExpr, TK_NOTNULL); } break; - case 214: /* expr ::= NOT expr */ - case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); -{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 178: /* expr ::= NOT expr */ + case 179: /* expr ::= BITNOT expr */ yytestcase(yyruleno==179); +{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,yymsp[-1].major,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 216: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 180: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,TK_UMINUS,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 217: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + case 181: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,TK_UPLUS,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} break; - case 220: /* expr ::= expr between_op expr AND expr */ + case 182: /* between_op ::= BETWEEN */ + case 185: /* in_op ::= IN */ yytestcase(yyruleno==185); +{yymsp[0].minor.yy392 = 0;} + break; + case 184: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy342.pExpr); + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy342.pExpr, 0, 0); + if( yymsp[-4].minor.yy342.pExpr ){ + yymsp[-4].minor.yy342.pExpr->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); + yymsp[-4].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; } break; - case 223: /* expr ::= expr in_op LP exprlist RP */ + case 187: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy14==0 ){ + if( yymsp[-1].minor.yy442==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -127565,9 +131242,9 @@ static void yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); - }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy342.pExpr); + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy392]); + }else if( yymsp[-1].minor.yy442->nExpr==1 ){ /* Expressions of the form: ** ** expr1 IN (?1) @@ -127584,433 +131261,436 @@ static void yy_reduce( ** affinity or the collating sequence to use for comparison. Otherwise, ** the semantics would be subtly different from IN or NOT IN. */ - Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; - yymsp[-1].minor.yy14->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + Expr *pRHS = yymsp[-1].minor.yy442->a[0].pExpr; + yymsp[-1].minor.yy442->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy442); /* pRHS cannot be NULL because a malloc error would have been detected ** before now and control would have never reached this point */ if( ALWAYS(pRHS) ){ pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy392 ? TK_NE : TK_EQ, yymsp[-4].minor.yy342.pExpr, pRHS, 0); }else{ - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; - sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy342.pExpr, 0, 0); + if( yymsp[-4].minor.yy342.pExpr ){ + yymsp[-4].minor.yy342.pExpr->x.pList = yymsp[-1].minor.yy442; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy342.pExpr); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy442); } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); } - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-4].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 224: /* expr ::= LP select RP */ + case 188: /* expr ::= LP select RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ + yymsp[-2].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yymsp[-2].minor.yy342.pExpr ){ + yymsp[-2].minor.yy342.pExpr->x.pSelect = yymsp[-1].minor.yy159; + ExprSetProperty(yymsp[-2].minor.yy342.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-2].minor.yy342.pExpr); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); } - yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 225: /* expr ::= expr in_op LP select RP */ + case 189: /* expr ::= expr in_op LP select RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy342.pExpr, 0, 0); + if( yymsp[-4].minor.yy342.pExpr ){ + yymsp[-4].minor.yy342.pExpr->x.pSelect = yymsp[-1].minor.yy159; + ExprSetProperty(yymsp[-4].minor.yy342.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy342.pExpr); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); } - if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); + yymsp[-4].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 226: /* expr ::= expr in_op nm dbnm */ + case 190: /* expr ::= expr in_op nm dbnm */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy342.pExpr, 0, 0); + if( yymsp[-3].minor.yy342.pExpr ){ + yymsp[-3].minor.yy342.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yymsp[-3].minor.yy342.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-3].minor.yy342.pExpr); }else{ sqlite3SrcListDelete(pParse->db, pSrc); } - if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); - yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart; - yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + exprNot(pParse, yymsp[-2].minor.yy392, &yymsp[-3].minor.yy342); + yymsp[-3].minor.yy342.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; } break; - case 227: /* expr ::= EXISTS LP select RP */ + case 191: /* expr ::= EXISTS LP select RP */ { - Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + Expr *p; + spanSet(&yymsp[-3].minor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ + p = yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ - p->x.pSelect = yymsp[-1].minor.yy3; + p->x.pSelect = yymsp[-1].minor.yy159; ExprSetProperty(p, EP_xIsSelect|EP_Subquery); sqlite3ExprSetHeightAndFlags(pParse, p); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); } - yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 192: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; - sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + spanSet(&yymsp[-4].minor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-C*/ + yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy122, 0, 0); + if( yymsp[-4].minor.yy342.pExpr ){ + yymsp[-4].minor.yy342.pExpr->x.pList = yymsp[-1].minor.yy122 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[-1].minor.yy122) : yymsp[-2].minor.yy442; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy342.pExpr); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy442); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy122); } - yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 193: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); + yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, yymsp[-2].minor.yy342.pExpr); + yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, yymsp[0].minor.yy342.pExpr); } break; - case 230: /* case_exprlist ::= WHEN expr THEN expr */ + case 194: /* case_exprlist ::= WHEN expr THEN expr */ { - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); + yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); + yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442, yymsp[0].minor.yy342.pExpr); } break; - case 237: /* nexprlist ::= nexprlist COMMA expr */ -{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} + case 197: /* case_operand ::= expr */ +{yymsp[0].minor.yy122 = yymsp[0].minor.yy342.pExpr; /*A-overwrites-X*/} + break; + case 200: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[0].minor.yy342.pExpr);} break; - case 238: /* nexprlist ::= expr */ -{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} + case 201: /* nexprlist ::= expr */ +{yymsp[0].minor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy342.pExpr); /*A-overwrites-Y*/} break; - case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ + case 202: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy442, yymsp[-10].minor.yy392, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy122, SQLITE_SO_ASC, yymsp[-8].minor.yy392); } break; - case 240: /* uniqueflag ::= UNIQUE */ - case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); -{yygotominor.yy328 = OE_Abort;} + case 203: /* uniqueflag ::= UNIQUE */ + case 244: /* raisetype ::= ABORT */ yytestcase(yyruleno==244); +{yymsp[0].minor.yy392 = OE_Abort;} break; - case 241: /* uniqueflag ::= */ -{yygotominor.yy328 = OE_None;} + case 204: /* uniqueflag ::= */ +{yymsp[1].minor.yy392 = OE_None;} break; - case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ + case 206: /* eidlist_opt ::= LP eidlist RP */ +{yymsp[-2].minor.yy442 = yymsp[-1].minor.yy442;} + break; + case 207: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); - sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-4].minor.yy442 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy442, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } break; - case 245: /* idxlist ::= nm collate sortorder */ + case 208: /* eidlist ::= nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yymsp[-2].minor.yy442 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); /*A-overwrites-Y*/ } break; - case 246: /* collate ::= */ -{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} - break; - case 248: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} + case 211: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy347, yymsp[-1].minor.yy392);} break; - case 249: /* cmd ::= VACUUM */ - case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); + case 212: /* cmd ::= VACUUM */ + case 213: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==213); {sqlite3Vacuum(pParse);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 214: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 215: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 216: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 217: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 218: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 221: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy327, &all); } break; - case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 222: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); - yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy392, yymsp[-4].minor.yy410.a, yymsp[-4].minor.yy410.b, yymsp[-2].minor.yy347, yymsp[0].minor.yy122, yymsp[-10].minor.yy392, yymsp[-8].minor.yy392); + yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 266: /* trigger_time ::= BEFORE */ - case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); -{ yygotominor.yy328 = TK_BEFORE; } + case 223: /* trigger_time ::= BEFORE */ +{ yymsp[0].minor.yy392 = TK_BEFORE; } break; - case 267: /* trigger_time ::= AFTER */ -{ yygotominor.yy328 = TK_AFTER; } + case 224: /* trigger_time ::= AFTER */ +{ yymsp[0].minor.yy392 = TK_AFTER; } break; - case 268: /* trigger_time ::= INSTEAD OF */ -{ yygotominor.yy328 = TK_INSTEAD;} + case 225: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy392 = TK_INSTEAD;} break; - case 270: /* trigger_event ::= DELETE|INSERT */ - case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); -{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} + case 226: /* trigger_time ::= */ +{ yymsp[1].minor.yy392 = TK_BEFORE; } break; - case 272: /* trigger_event ::= UPDATE OF idlist */ -{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} + case 227: /* trigger_event ::= DELETE|INSERT */ + case 228: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==228); +{yymsp[0].minor.yy410.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy410.b = 0;} break; - case 275: /* when_clause ::= */ - case 296: /* key_opt ::= */ yytestcase(yyruleno==296); -{ yygotominor.yy132 = 0; } + case 229: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy410.a = TK_UPDATE; yymsp[-2].minor.yy410.b = yymsp[0].minor.yy180;} break; - case 276: /* when_clause ::= WHEN expr */ - case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); -{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } + case 230: /* when_clause ::= */ + case 249: /* key_opt ::= */ yytestcase(yyruleno==249); +{ yymsp[1].minor.yy122 = 0; } break; - case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 231: /* when_clause ::= WHEN expr */ + case 250: /* key_opt ::= KEY expr */ yytestcase(yyruleno==250); +{ yymsp[-1].minor.yy122 = yymsp[0].minor.yy342.pExpr; } + break; + case 232: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy473!=0 ); - yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; - yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473; - yygotominor.yy473 = yymsp[-2].minor.yy473; + assert( yymsp[-2].minor.yy327!=0 ); + yymsp[-2].minor.yy327->pLast->pNext = yymsp[-1].minor.yy327; + yymsp[-2].minor.yy327->pLast = yymsp[-1].minor.yy327; } break; - case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 233: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy473!=0 ); - yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; - yygotominor.yy473 = yymsp[-1].minor.yy473; + assert( yymsp[-1].minor.yy327!=0 ); + yymsp[-1].minor.yy327->pLast = yymsp[-1].minor.yy327; } break; - case 280: /* trnm ::= nm DOT nm */ + case 234: /* trnm ::= nm DOT nm */ { - yygotominor.yy0 = yymsp[0].minor.yy0; + yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, "qualified table names are not allowed on INSERT, UPDATE, and DELETE " "statements within triggers"); } break; - case 282: /* tridxby ::= INDEXED BY nm */ + case 235: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 283: /* tridxby ::= NOT INDEXED */ + case 236: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } + case 237: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{yymsp[-6].minor.yy327 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy442, yymsp[0].minor.yy122, yymsp[-5].minor.yy392);} break; - case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ -{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} + case 238: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ +{yymsp[-4].minor.yy327 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy180, yymsp[0].minor.yy159, yymsp[-4].minor.yy392);/*A-overwrites-R*/} break; - case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} + case 239: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yymsp[-4].minor.yy327 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy122);} break; - case 287: /* trigger_cmd ::= select */ -{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } + case 240: /* trigger_cmd ::= select */ +{yymsp[0].minor.yy327 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy159); /*A-overwrites-X*/} break; - case 288: /* expr ::= RAISE LP IGNORE RP */ + case 241: /* expr ::= RAISE LP IGNORE RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yygotominor.yy346.pExpr ){ - yygotominor.yy346.pExpr->affinity = OE_Ignore; + spanSet(&yymsp[-3].minor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yymsp[-3].minor.yy342.pExpr ){ + yymsp[-3].minor.yy342.pExpr->affinity = OE_Ignore; } - yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 242: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yygotominor.yy346.pExpr ) { - yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328; + spanSet(&yymsp[-5].minor.yy342,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ + yymsp[-5].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yymsp[-5].minor.yy342.pExpr ) { + yymsp[-5].minor.yy342.pExpr->affinity = (char)yymsp[-3].minor.yy392; } - yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z; - yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 290: /* raisetype ::= ROLLBACK */ -{yygotominor.yy328 = OE_Rollback;} + case 243: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy392 = OE_Rollback;} break; - case 292: /* raisetype ::= FAIL */ -{yygotominor.yy328 = OE_Fail;} + case 245: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy392 = OE_Fail;} break; - case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 246: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy347,yymsp[-1].minor.yy392); } break; - case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 247: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); + sqlite3Attach(pParse, yymsp[-3].minor.yy342.pExpr, yymsp[-1].minor.yy342.pExpr, yymsp[0].minor.yy122); } break; - case 295: /* cmd ::= DETACH database_kw_opt expr */ + case 248: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); + sqlite3Detach(pParse, yymsp[0].minor.yy342.pExpr); } break; - case 300: /* cmd ::= REINDEX */ + case 251: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 301: /* cmd ::= REINDEX nm dbnm */ + case 252: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 302: /* cmd ::= ANALYZE */ + case 253: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 303: /* cmd ::= ANALYZE nm dbnm */ + case 254: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 255: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy347,&yymsp[0].minor.yy0); } break; - case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ + case 256: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { - sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); + yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; + sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 306: /* add_column_fullname ::= fullname */ + case 257: /* add_column_fullname ::= fullname */ { - pParse->db->lookaside.bEnabled = 0; - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy347); } break; - case 309: /* cmd ::= create_vtab */ + case 258: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 310: /* cmd ::= create_vtab LP vtabarglist RP */ + case 259: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 260: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy392); } break; - case 314: /* vtabarg ::= */ + case 261: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 316: /* vtabargtoken ::= ANY */ - case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); - case 318: /* lp ::= LP */ yytestcase(yyruleno==318); + case 262: /* vtabargtoken ::= ANY */ + case 263: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==263); + case 264: /* lp ::= LP */ yytestcase(yyruleno==264); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 322: /* with ::= */ -{yygotominor.yy59 = 0;} + case 265: /* with ::= */ +{yymsp[1].minor.yy331 = 0;} + break; + case 266: /* with ::= WITH wqlist */ +{ yymsp[-1].minor.yy331 = yymsp[0].minor.yy331; } break; - case 323: /* with ::= WITH wqlist */ - case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); -{ yygotominor.yy59 = yymsp[0].minor.yy59; } + case 267: /* with ::= WITH RECURSIVE wqlist */ +{ yymsp[-2].minor.yy331 = yymsp[0].minor.yy331; } break; - case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ + case 268: /* wqlist ::= nm eidlist_opt AS LP select RP */ { - yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); + yymsp[-5].minor.yy331 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy442, yymsp[-1].minor.yy159); /*A-overwrites-X*/ } break; - case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ + case 269: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { - yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); + yymsp[-7].minor.yy331 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy331, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy442, yymsp[-1].minor.yy159); } break; default: - /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); - /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); - /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); - /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); - /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); - /* (10) trans_opt ::= */ yytestcase(yyruleno==10); - /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); - /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); - /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); - /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); - /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); - /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36); - /* (37) columnlist ::= column */ yytestcase(yyruleno==37); - /* (43) type ::= */ yytestcase(yyruleno==43); - /* (50) signed ::= plus_num */ yytestcase(yyruleno==50); - /* (51) signed ::= minus_num */ yytestcase(yyruleno==51); - /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52); - /* (53) carglist ::= */ yytestcase(yyruleno==53); - /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60); - /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); - /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); - /* (91) tconscomma ::= */ yytestcase(yyruleno==91); - /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); - /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); - /* (281) tridxby ::= */ yytestcase(yyruleno==281); - /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); - /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); - /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); - /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); - /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); - /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); - /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); - /* (319) anylist ::= */ yytestcase(yyruleno==319); - /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); - /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + /* (270) input ::= cmdlist */ yytestcase(yyruleno==270); + /* (271) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==271); + /* (272) cmdlist ::= ecmd */ yytestcase(yyruleno==272); + /* (273) ecmd ::= SEMI */ yytestcase(yyruleno==273); + /* (274) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==274); + /* (275) explain ::= */ yytestcase(yyruleno==275); + /* (276) trans_opt ::= */ yytestcase(yyruleno==276); + /* (277) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==277); + /* (278) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==278); + /* (279) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==279); + /* (280) savepoint_opt ::= */ yytestcase(yyruleno==280); + /* (281) cmd ::= create_table create_table_args */ yytestcase(yyruleno==281); + /* (282) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==282); + /* (283) columnlist ::= columnname carglist */ yytestcase(yyruleno==283); + /* (284) nm ::= ID|INDEXED */ yytestcase(yyruleno==284); + /* (285) nm ::= STRING */ yytestcase(yyruleno==285); + /* (286) nm ::= JOIN_KW */ yytestcase(yyruleno==286); + /* (287) typetoken ::= typename */ yytestcase(yyruleno==287); + /* (288) typename ::= ID|STRING */ yytestcase(yyruleno==288); + /* (289) signed ::= plus_num */ yytestcase(yyruleno==289); + /* (290) signed ::= minus_num */ yytestcase(yyruleno==290); + /* (291) carglist ::= carglist ccons */ yytestcase(yyruleno==291); + /* (292) carglist ::= */ yytestcase(yyruleno==292); + /* (293) ccons ::= NULL onconf */ yytestcase(yyruleno==293); + /* (294) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==294); + /* (295) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==295); + /* (296) conslist ::= tcons */ yytestcase(yyruleno==296); + /* (297) tconscomma ::= */ yytestcase(yyruleno==297); + /* (298) defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==298); + /* (299) resolvetype ::= raisetype */ yytestcase(yyruleno==299); + /* (300) selectnowith ::= oneselect */ yytestcase(yyruleno==300); + /* (301) oneselect ::= values */ yytestcase(yyruleno==301); + /* (302) sclp ::= selcollist COMMA */ yytestcase(yyruleno==302); + /* (303) as ::= ID|STRING */ yytestcase(yyruleno==303); + /* (304) expr ::= term */ yytestcase(yyruleno==304); + /* (305) exprlist ::= nexprlist */ yytestcase(yyruleno==305); + /* (306) nmnum ::= plus_num */ yytestcase(yyruleno==306); + /* (307) nmnum ::= nm */ yytestcase(yyruleno==307); + /* (308) nmnum ::= ON */ yytestcase(yyruleno==308); + /* (309) nmnum ::= DELETE */ yytestcase(yyruleno==309); + /* (310) nmnum ::= DEFAULT */ yytestcase(yyruleno==310); + /* (311) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==311); + /* (312) foreach_clause ::= */ yytestcase(yyruleno==312); + /* (313) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==313); + /* (314) trnm ::= nm */ yytestcase(yyruleno==314); + /* (315) tridxby ::= */ yytestcase(yyruleno==315); + /* (316) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==316); + /* (317) database_kw_opt ::= */ yytestcase(yyruleno==317); + /* (318) kwcolumn_opt ::= */ yytestcase(yyruleno==318); + /* (319) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==319); + /* (320) vtabarglist ::= vtabarg */ yytestcase(yyruleno==320); + /* (321) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==321); + /* (322) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==322); + /* (323) anylist ::= */ yytestcase(yyruleno==323); + /* (324) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==324); + /* (325) anylist ::= anylist ANY */ yytestcase(yyruleno==325); break; +/********** End reduce actions ************************************************/ }; - assert( yyruleno>=0 && yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) ); + assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) ); yygoto = yyRuleInfo[yyruleno].lhs; yysize = yyRuleInfo[yyruleno].nrhs; - yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least - ** one element off the stack, then we can push the new element back - ** onto the stack here, and skip the stack overflow test in yy_shift(). - ** That gives a significant speed improvement. */ - if( yysize ){ - yypParser->yyidx++; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; - }else -#endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); - } - }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yypParser->yyidx -= yysize - 1; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact); + }else{ + assert( yyact == YY_ACCEPT_ACTION ); + yypParser->yyidx -= yysize; yy_accept(yypParser); } } @@ -128031,6 +131711,8 @@ static void yy_parse_failed( while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -128041,14 +131723,16 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - YYMINORTYPE yyminor /* The minor type of the error token */ + sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ ){ sqlite3ParserARG_FETCH; -#define TOKEN (yyminor.yy0) +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); +/************ End %syntax_error code ******************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -128067,6 +131751,8 @@ static void yy_accept( while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -128096,7 +131782,7 @@ SQLITE_PRIVATE void sqlite3Parser( sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - int yyact; /* The parser action. */ + unsigned int yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif @@ -128110,18 +131796,23 @@ SQLITE_PRIVATE void sqlite3Parser( if( yypParser->yyidx<0 ){ #if YYSTACKDEPTH<=0 if( yypParser->yystksz <=0 ){ - /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ - yyminorunion = yyzerominor; - yyStackOverflow(yypParser, &yyminorunion); + yyStackOverflow(yypParser); return; } #endif yypParser->yyidx = 0; +#ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; +#endif yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n", + yyTracePrompt); + } +#endif } - yyminorunion.yy0 = yyminor; #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif @@ -128129,20 +131820,24 @@ SQLITE_PRIVATE void sqlite3Parser( #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); + fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); } #endif do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact<YYNSTATE ){ - yy_shift(yypParser,yyact,yymajor,&yyminorunion); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,yyminor); +#ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; +#endif yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx; #endif @@ -128172,7 +131867,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** */ if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor,yyminor); } yymx = yypParser->yystack[yypParser->yyidx].major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ @@ -128182,15 +131877,15 @@ SQLITE_PRIVATE void sqlite3Parser( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( + while( yypParser->yyidx >= 0 && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -128199,9 +131894,7 @@ SQLITE_PRIVATE void sqlite3Parser( yy_parse_failed(yypParser); yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ - YYMINORTYPE u2; - u2.YYERRSYMDT = 0; - yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); } } yypParser->yyerrcnt = 3; @@ -128214,7 +131907,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** Applications can set this macro (for example inside %include) if ** they intend to abandon the parse upon the first syntax error seen. */ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; @@ -128229,7 +131922,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** three input tokens have been successfully shifted. */ if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_syntax_error(yypParser,yymajor, yyminor); } yypParser->yyerrcnt = 3; yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); @@ -128240,6 +131933,16 @@ SQLITE_PRIVATE void sqlite3Parser( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + int i; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', + yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"]\n"); + } +#endif return; } @@ -128265,12 +131968,92 @@ SQLITE_PRIVATE void sqlite3Parser( /* #include "sqliteInt.h" */ /* #include <stdlib.h> */ +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +#ifdef SQLITE_ASCII +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +#endif +#ifdef SQLITE_EBCDIC +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10, +/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, +/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7, +/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, +/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, +#endif +}; + /* -** The charMap() macro maps alphabetic characters into their +** The charMap() macro maps alphabetic characters (only) into their ** lower-case ASCII equivalent. On ASCII machines, this is just ** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. Only alphabetic characters and underscores -** need to be translated. +** to adjust the encoding. The mapping is only valid for alphabetics +** which are the only characters for which this feature is used. +** +** Used by keywordhash.h */ #ifdef SQLITE_ASCII # define charMap(X) sqlite3UpperToLower[(unsigned char)X] @@ -128304,7 +132087,7 @@ const unsigned char ebcdicToAscii[] = { ** returned. If the input is not a keyword, TK_ID is returned. ** ** The implementation of this routine was generated by a program, -** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** mkkeywordhash.c, located in the tool subdirectory of the distribution. ** The output of the mkkeywordhash.c program is written into a file ** named keywordhash.h and then included into this source file by ** the #include below. @@ -128325,7 +132108,7 @@ const unsigned char ebcdicToAscii[] = { ** on platforms with limited memory. */ /* Hash score: 182 */ -static int keywordCode(const char *z, int n){ +static int keywordCode(const char *z, int n, int *pType){ /* zText[] encodes 834 bytes of keywords in 554 bytes */ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ @@ -128445,13 +132228,21 @@ static int keywordCode(const char *z, int n){ TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, }; - int h, i; - if( n<2 ) return TK_ID; - h = ((charMap(z[0])*4) ^ - (charMap(z[n-1])*3) ^ - n) % 127; - for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ - if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + int i, j; + const char *zKW; + if( n>=2 ){ + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; + for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){ + if( aLen[i]!=n ) continue; + j = 0; + zKW = &zText[aOffset[i]]; +#ifdef SQLITE_ASCII + while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; } +#endif +#ifdef SQLITE_EBCDIC + while( j<n && toupper(z[j])==zKW[j] ){ j++; } +#endif + if( j<n ) continue; testcase( i==0 ); /* REINDEX */ testcase( i==1 ); /* INDEXED */ testcase( i==2 ); /* INDEX */ @@ -128576,13 +132367,16 @@ static int keywordCode(const char *z, int n){ testcase( i==121 ); /* VIEW */ testcase( i==122 ); /* INITIALLY */ testcase( i==123 ); /* ALL */ - return aCode[i]; + *pType = aCode[i]; + break; } } - return TK_ID; + return n; } SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ - return keywordCode((char*)z, n); + int id = TK_ID; + keywordCode((char*)z, n, &id); + return id; } #define SQLITE_N_KEYWORD 124 @@ -128635,13 +132429,15 @@ SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* -** Return the length of the token that begins at z[0]. +** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ int i, c; - switch( *z ){ - case ' ': case '\t': case '\n': case '\f': case '\r': { + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { testcase( z[0]==' ' ); testcase( z[0]=='\t' ); testcase( z[0]=='\n' ); @@ -128651,7 +132447,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SPACE; return i; } - case '-': { + case CC_MINUS: { if( z[1]=='-' ){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ @@ -128660,27 +132456,27 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_MINUS; return 1; } - case '(': { + case CC_LP: { *tokenType = TK_LP; return 1; } - case ')': { + case CC_RP: { *tokenType = TK_RP; return 1; } - case ';': { + case CC_SEMI: { *tokenType = TK_SEMI; return 1; } - case '+': { + case CC_PLUS: { *tokenType = TK_PLUS; return 1; } - case '*': { + case CC_STAR: { *tokenType = TK_STAR; return 1; } - case '/': { + case CC_SLASH: { if( z[1]!='*' || z[2]==0 ){ *tokenType = TK_SLASH; return 1; @@ -128690,15 +132486,15 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; } - case '%': { + case CC_PERCENT: { *tokenType = TK_REM; return 1; } - case '=': { + case CC_EQ: { *tokenType = TK_EQ; return 1 + (z[1]=='='); } - case '<': { + case CC_LT: { if( (c=z[1])=='=' ){ *tokenType = TK_LE; return 2; @@ -128713,7 +132509,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 1; } } - case '>': { + case CC_GT: { if( (c=z[1])=='=' ){ *tokenType = TK_GE; return 2; @@ -128725,16 +132521,16 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 1; } } - case '!': { + case CC_BANG: { if( z[1]!='=' ){ *tokenType = TK_ILLEGAL; - return 2; + return 1; }else{ *tokenType = TK_NE; return 2; } } - case '|': { + case CC_PIPE: { if( z[1]!='|' ){ *tokenType = TK_BITOR; return 1; @@ -128743,21 +132539,19 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return 2; } } - case ',': { + case CC_COMMA: { *tokenType = TK_COMMA; return 1; } - case '&': { + case CC_AND: { *tokenType = TK_BITAND; return 1; } - case '~': { + case CC_TILDA: { *tokenType = TK_BITNOT; return 1; } - case '`': - case '\'': - case '"': { + case CC_QUOTE: { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); @@ -128782,7 +132576,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } } - case '.': { + case CC_DOT: { #ifndef SQLITE_OMIT_FLOATING_POINT if( !sqlite3Isdigit(z[1]) ) #endif @@ -128793,8 +132587,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ /* If the next character is a digit, this is a floating point ** number that begins with ".". Fall thru into the next case */ } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { + case CC_DIGIT: { testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); @@ -128829,22 +132622,18 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } return i; } - case '[': { + case CC_QUOTE2: { for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} *tokenType = c==']' ? TK_ID : TK_ILLEGAL; return i; } - case '?': { + case CC_VARNUM: { *tokenType = TK_VARIABLE; for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } -#ifndef SQLITE_OMIT_TCL_VARIABLE - case '$': -#endif - case '@': /* For compatibility with MS SQL Server */ - case '#': - case ':': { + case CC_DOLLAR: + case CC_VARALPHA: { int n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); @@ -128873,8 +132662,20 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( n==0 ) *tokenType = TK_ILLEGAL; return i; } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); + } + case CC_X: { #ifndef SQLITE_OMIT_BLOB_LITERAL - case 'x': case 'X': { testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; @@ -128886,20 +132687,22 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( z[i] ) i++; return i; } - /* Otherwise fall through to the next case */ - } #endif + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + } + case CC_ID: { + i = 1; + break; + } default: { - if( !IdChar(*z) ){ - break; - } - for(i=1; IdChar(z[i]); i++){} - *tokenType = keywordCode((char*)z, i); - return i; + *tokenType = TK_ILLEGAL; + return 1; } } - *tokenType = TK_ILLEGAL; - return 1; + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; } /* @@ -128915,7 +132718,6 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr void *pEngine; /* The LEMON-generated LALR(1) parser */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ @@ -128928,19 +132730,18 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); + /* sqlite3ParserTrace(stdout, "parser: "); */ pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ - db->mallocFailed = 1; - return SQLITE_NOMEM; + sqlite3OomFault(db); + return SQLITE_NOMEM_BKPT; } assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->nzVar==0 ); assert( pParse->azVar==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; - while( !db->mallocFailed && zSql[i]!=0 ){ + while( zSql[i]!=0 ){ assert( i>=0 ); pParse->sLastToken.z = &zSql[i]; pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); @@ -128949,41 +132750,29 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr pParse->rc = SQLITE_TOOBIG; break; } - switch( tokenType ){ - case TK_SPACE: { - if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); - pParse->rc = SQLITE_INTERRUPT; - goto abort_parse; - } + if( tokenType>=TK_SPACE ){ + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + if( db->u1.isInterrupted ){ + pParse->rc = SQLITE_INTERRUPT; break; } - case TK_ILLEGAL: { + if( tokenType==TK_ILLEGAL ){ sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &pParse->sLastToken); - goto abort_parse; - } - case TK_SEMI: { - pParse->zTail = &zSql[i]; - /* Fall thru into the default case */ - } - default: { - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK ){ - goto abort_parse; - } break; } + }else{ + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } } -abort_parse: assert( nErr==0 ); + pParse->zTail = &zSql[i]; if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ assert( zSql[i]==0 ); if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; } if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); @@ -128991,15 +132780,14 @@ abort_parse: } #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, sqlite3ParserStackPeak(pEngine) ); sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + pParse->rc = SQLITE_NOMEM_BKPT; } if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); @@ -129034,7 +132822,7 @@ abort_parse: sqlite3DeleteTable(db, pParse->pNewTable); } - if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); + sqlite3WithDelete(db, pParse->pWithToFree); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); @@ -129337,7 +133125,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ if( zSql8 ){ rc = sqlite3_complete(zSql8); }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3ValueFree(pVal); return rc & 0xff; @@ -129465,6 +133253,12 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); /************** End of sqliteicu.h *******************************************/ /************** Continuing where we left off in main.c ***********************/ #endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif #ifndef SQLITE_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant @@ -129621,7 +133415,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ sqlite3GlobalConfig.pInitMutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -129652,10 +133446,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); sqlite3GlobalConfig.inProgress = 1; - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); if( sqlite3GlobalConfig.isPCacheInit==0 ){ rc = sqlite3PcacheInitialize(); } @@ -129873,9 +133672,10 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_PAGECACHE: { - /* EVIDENCE-OF: R-31408-40510 There are three arguments to - ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size - ** of each page buffer (sz), and the number of pages (N). */ + /* EVIDENCE-OF: R-18761-36601 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), + ** the size of each page cache line (sz), and the number of cache lines + ** (N). */ sqlite3GlobalConfig.pPage = va_arg(ap, void*); sqlite3GlobalConfig.szPage = va_arg(ap, int); sqlite3GlobalConfig.nPage = va_arg(ap, int); @@ -130061,6 +133861,11 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -130125,12 +133930,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ p = (LookasideSlot*)&((u8*)p)[sz]; } db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; + db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; }else{ db->lookaside.pStart = db; db->lookaside.pEnd = db; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable = 1; db->lookaside.bMalloced = 0; } #endif /* SQLITE_OMIT_LOOKASIDE */ @@ -130175,6 +133980,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ } /* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + +/* ** Configuration settings for an individual database connection */ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ @@ -130194,8 +134029,9 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ int op; /* The opcode */ u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -130355,7 +134191,7 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ ** with SQLITE_ANY as the encoding. */ static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; + FuncDestructor *pDestructor = p->u.pDestructor; if( pDestructor ){ pDestructor->nRef--; if( pDestructor->nRef==0 ){ @@ -130372,17 +134208,23 @@ static void functionDestroy(sqlite3 *db, FuncDef *p){ static void disconnectAllVtab(sqlite3 *db){ #ifndef SQLITE_OMIT_VIRTUALTABLE int i; + HashElem *p; sqlite3BtreeEnterAll(db); for(i=0; i<db->nDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( db->aDb[i].pSchema ){ - HashElem *p; for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Table *pTab = (Table *)sqliteHashData(p); if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); } } } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else @@ -130531,18 +134373,17 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ */ sqlite3ConnectionClosed(db); - for(j=0; j<ArraySize(db->aFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } - } + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); } + sqlite3HashClear(&db->aFunc); for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ CollSeq *pColl = (CollSeq *)sqliteHashData(i); /* Invoke any destructors registered for collation sequence user data. */ @@ -130560,6 +134401,7 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ if( pMod->xDestroy ){ pMod->xDestroy(pMod->pAux); } + sqlite3VtabEponymousTableClear(db, pMod); sqlite3DbFree(db, pMod); } sqlite3HashClear(&db->aModule); @@ -130965,7 +134807,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( int nArg, int enc, void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), FuncDestructor *pDestructor @@ -130976,9 +134818,9 @@ SQLITE_PRIVATE int sqlite3CreateFunc( assert( sqlite3_mutex_held(db->mutex) ); if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || + (xSFunc && (xFinal || xStep)) || + (!xSFunc && (xFinal && !xStep)) || + (!xSFunc && (!xFinal && xStep)) || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || (255<(nName = sqlite3Strlen30( zFunctionName))) ){ return SQLITE_MISUSE_BKPT; @@ -131001,10 +134843,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; @@ -131020,7 +134862,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( ** is being overridden/deleted but there are no active VMs, allow the ** operation to continue but invalidate all precompiled statements. */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ if( db->nVdbeActive ){ sqlite3ErrorWithMsg(db, SQLITE_BUSY, @@ -131032,10 +134874,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( } } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); assert(p || db->mallocFailed); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* If an older version of the function with a configured destructor is @@ -131045,11 +134887,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( if( pDestructor ){ pDestructor->nRef++; } - p->pDestructor = pDestructor; + p->u.pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xFunc = xFunc; - p->xStep = xStep; + p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; p->pUserData = pUserData; p->nArg = (u16)nArg; @@ -131065,11 +134906,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function( int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, 0); } @@ -131079,7 +134920,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), void (*xDestroy)(void *) @@ -131102,7 +134943,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); if( pArg && pArg->nRef==0 ){ assert( rc!=SQLITE_OK ); xDestroy(p); @@ -131122,7 +134963,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( int nArg, int eTextRep, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ){ @@ -131135,7 +134976,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -131161,7 +135002,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( const char *zName, int nArg ){ - int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; #ifdef SQLITE_ENABLE_API_ARMOR @@ -131170,7 +135010,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( } #endif sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + if( sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, 0, sqlite3InvalidFunction, 0, 0, 0); } @@ -131540,14 +135380,14 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ const char *z; if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); } if( !sqlite3SafetyCheckSickOrOk(db) ){ return sqlite3ErrStr(SQLITE_MISUSE_BKPT); } sqlite3_mutex_enter(db->mutex); if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); z = (char*)sqlite3_value_text(db->pErr); @@ -131599,7 +135439,7 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ ** be cleared before returning. Do this directly, instead of via ** sqlite3ApiExit(), to avoid setting the database handle error message. */ - db->mallocFailed = 0; + sqlite3OomClear(db); } sqlite3_mutex_leave(db->mutex); return z; @@ -131615,7 +135455,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode & db->errMask; } @@ -131624,10 +135464,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode; } +SQLITE_API int SQLITE_STDCALL sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} /* ** Return a string that describes the kind of error specified in the @@ -131704,7 +135547,7 @@ static int createCollation( } pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; + if( pColl==0 ) return SQLITE_NOMEM_BKPT; pColl->xCmp = xCompare; pColl->pUser = pCtx; pColl->xDel = xDel; @@ -131752,8 +135595,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -131883,7 +135726,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( for(iIn=0; iIn<nUri; iIn++) nByte += (zUri[iIn]=='&'); zFile = sqlite3_malloc64(nByte); - if( !zFile ) return SQLITE_NOMEM; + if( !zFile ) return SQLITE_NOMEM_BKPT; iIn = 5; #ifdef SQLITE_ALLOW_URI_AUTHORITY @@ -132049,7 +135892,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( }else{ zFile = sqlite3_malloc64(nUri+2); - if( !zFile ) return SQLITE_NOMEM; + if( !zFile ) return SQLITE_NOMEM_BKPT; memcpy(zFile, zUri, nUri); zFile[nUri] = '\0'; zFile[nUri+1] = '\0'; @@ -132206,6 +136049,9 @@ static int openDatabase( #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) | SQLITE_CellSizeCk #endif +#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) + | SQLITE_Fts3Tokenizer +#endif ; sqlite3HashInit(&db->aCollSeq); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -132219,9 +136065,9 @@ static int openDatabase( ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating ** functions: */ - createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); if( db->mallocFailed ){ @@ -132230,14 +136076,14 @@ static int openDatabase( /* EVIDENCE-OF: R-08308-17224 The default collating function for all ** strings is BINARY. */ - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, sqlite3StrBINARY, 0); assert( db->pDfltColl!=0 ); /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; @@ -132248,7 +136094,7 @@ static int openDatabase( flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3Error(db, rc); goto opendb_out; @@ -132259,13 +136105,13 @@ static int openDatabase( sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. */ db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; db->magic = SQLITE_MAGIC_OPEN; if( db->mallocFailed ){ @@ -132277,7 +136123,7 @@ static int openDatabase( ** is accessed. */ sqlite3Error(db, SQLITE_OK); - sqlite3RegisterBuiltinFunctions(db); + sqlite3RegisterPerConnectionBuiltinFunctions(db); /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. @@ -132305,12 +136151,18 @@ static int openDatabase( } #endif -#ifdef SQLITE_ENABLE_FTS3 +#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } #endif +#ifdef SQLITE_ENABLE_FTS5 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif + #ifdef SQLITE_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); @@ -132329,6 +136181,12 @@ static int openDatabase( } #endif +#ifdef SQLITE_ENABLE_JSON1 + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3Json1Init(db); + } +#endif + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. @@ -132348,7 +136206,6 @@ static int openDatabase( sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: - sqlite3_free(zOpen); if( db ){ assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); @@ -132370,6 +136227,22 @@ opendb_out: sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } #endif +#if defined(SQLITE_HAS_CODEC) + if( rc==SQLITE_OK ){ + const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey"); + if( zHexKey && zHexKey[0] ){ + u8 iByte; + int i; + char zKey[40]; + for(i=0, iByte=0; i<sizeof(zKey)*2 && sqlite3Isxdigit(zHexKey[i]); i++){ + iByte = (iByte<<4) + sqlite3HexToInt(zHexKey[i]); + if( (i&1)!=0 ) zKey[i/2] = iByte; + } + sqlite3_key_v2(db, 0, zKey, i/2); + } + } +#endif + sqlite3_free(zOpen); return rc & 0xff; } @@ -132424,7 +136297,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_open16( SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; } }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3ValueFree(pVal); @@ -132569,7 +136442,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ /* ** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error ** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger @@ -132578,28 +136451,33 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ ** 2. Invoke sqlite3_log() to provide the source code location where ** a low-level error is first detected. */ +static int reportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; + return reportError(SQLITE_CORRUPT, lineno, "database corruption"); } SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; + return reportError(SQLITE_MISUSE, lineno, "misuse"); } SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; + return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); } - +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_NOMEM, lineno, "OOM"); +} +SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); +} +#endif #ifndef SQLITE_OMIT_DEPRECATED /* @@ -132693,7 +136571,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( ** explicitly declared column. Copy meta information from *pCol. */ if( pCol ){ - zDataType = pCol->zType; + zDataType = sqlite3ColumnType(pCol,0); zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; @@ -132703,7 +136581,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( primarykey = 1; } if( !zCollSeq ){ - zCollSeq = "BINARY"; + zCollSeq = sqlite3StrBINARY; } error_out: @@ -132784,6 +136662,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbN if( op==SQLITE_FCNTL_FILE_POINTER ){ *(sqlite3_file**)pArg = fd; rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_VFS_POINTER ){ + *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ @@ -132924,7 +136808,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_ASSERT: { volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); rc = x; break; } @@ -133227,6 +137111,88 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbNa return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Obtain a snapshot handle for the snapshot of database zDb currently +** being read by handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_get( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot **ppSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Open a read-transaction on the snapshot idendified by pSnapshot. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( db->autoCommit==0 ){ + int iDb; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Free a snapshot handle obtained from sqlite3_snapshot_get(). +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ + sqlite3_free(pSnapshot); +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + /************** End of main.c ************************************************/ /************** Begin file notify.c ******************************************/ /* @@ -133876,6 +137842,12 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ # define NDEBUG 1 #endif +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + /* ** FTS4 is really an extension for FTS3. It is enabled using the ** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all @@ -134403,6 +138375,7 @@ struct Fts3Table { int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control @@ -135978,6 +139951,19 @@ static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ #endif } +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: @@ -136068,6 +140054,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } } + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + iIdx = 1; if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; @@ -141718,125 +145707,151 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ rc = SQLITE_ERROR; } - if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ - Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); - if( 0==apLeaf ){ - rc = SQLITE_NOMEM; - }else{ - memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); - } - - if( rc==SQLITE_OK ){ - int i; - Fts3Expr *p; - - /* Set $p to point to the left-most leaf in the tree of eType nodes. */ - for(p=pRoot; p->eType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); } - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){ - if( apLeaf[iLvl]==0 ){ - apLeaf[iLvl] = p; - p = 0; + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; }else{ - assert( pFree ); - pFree->pLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; + pRoot = 0; } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } - - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; - - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; - }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } - - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; - } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; - if( rc==SQLITE_OK ){ - p = 0; - for(i=0; i<nMaxDepth; i++){ - if( apLeaf[i] ){ - if( p==0 ){ - p = apLeaf[i]; - p->pParent = 0; + for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){ + if( apLeaf[iLvl]==0 ){ + apLeaf[iLvl] = p; + p = 0; }else{ - assert( pFree!=0 ); + assert( pFree ); + pFree->pLeft = apLeaf[iLvl]; pFree->pRight = p; - pFree->pLeft = apLeaf[i]; pFree->pLeft->pParent = pFree; pFree->pRight->pParent = pFree; p = pFree; pFree = pFree->pParent; p->pParent = 0; + apLeaf[iLvl] = 0; } } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; } - pRoot = p; - }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; i<nMaxDepth; i++){ - sqlite3Fts3ExprFree(apLeaf[i]); - } - while( (pDel=pFree)!=0 ){ - pFree = pDel->pParent; - sqlite3_free(pDel); + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; i<nMaxDepth; i++){ + if( apLeaf[i] ){ + if( p==0 ){ + p = apLeaf[i]; + p->pParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; i<nMaxDepth; i++){ + sqlite3Fts3ExprFree(apLeaf[i]); + } + while( (pDel=pFree)!=0 ){ + pFree = pDel->pParent; + sqlite3_free(pDel); + } } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; - assert( pFree==0 ); - sqlite3_free( apLeaf ); + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } } } - + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(pRoot); pRoot = 0; @@ -143292,6 +147307,18 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( /* #include <string.h> */ /* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + +/* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** @@ -143311,7 +147338,7 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** is a blob containing the pointer stored as the hash data corresponding ** to string <key-name> (after the hash-table is updated, if applicable). */ -static void scalarFunc( +static void fts3TokenizerFunc( sqlite3_context *context, int argc, sqlite3_value **argv @@ -143329,16 +147356,20 @@ static void scalarFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ - void *pOld; - int n = sqlite3_value_bytes(argv[1]); - if( zName==0 || n!=sizeof(pPtr) ){ - sqlite3_result_error(context, "argument type mismatch", -1); - return; - } - pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); - if( pOld==pPtr ){ - sqlite3_result_error(context, "out of memory", -1); + if( fts3TokenizerEnabled(context) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); return; } }else{ @@ -143352,7 +147383,6 @@ static void scalarFunc( return; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } @@ -143612,6 +147642,7 @@ int registerTokenizer( return sqlite3_finalize(pStmt); } + static int queryTokenizer( sqlite3 *db, @@ -143682,11 +147713,13 @@ static void intTestFunc( assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ - rc = registerTokenizer(db, "nosuchtokenizer", p1); - assert( rc==SQLITE_OK ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_OK ); - assert( p2==p1 ); + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } @@ -143702,7 +147735,7 @@ static void intTestFunc( ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above -** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. @@ -143731,10 +147764,10 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( #endif if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); } if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ @@ -144786,7 +148819,8 @@ static int fts3SqlStmt( ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment @@ -145313,10 +149347,12 @@ static int fts3PendingTermsAdd( */ static int fts3PendingTermsDocid( Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ int iLangid, /* Language id of row being written */ sqlite_int64 iDocid /* Docid of row being written */ ){ assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if @@ -145324,7 +149360,8 @@ static int fts3PendingTermsDocid( ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid + if( iDocid<p->iPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) || p->iPrevLangid!=iLangid || p->nPendingData>p->nMaxPendingData ){ @@ -145333,6 +149370,7 @@ static int fts3PendingTermsDocid( } p->iPrevDocid = iDocid; p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; return SQLITE_OK; } @@ -145522,7 +149560,8 @@ static void fts3DeleteTerms( if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; int iLangid = langidFromSelect(p, pSelect); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ int iCol = i-1; if( p->abNotindexed[iCol]==0 ){ @@ -145770,14 +149809,19 @@ static int fts3SegReaderNext( if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); - if( pElem==0 ){ - pReader->aNode = 0; - }else{ + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; pReader->zTerm = (char *)fts3HashKey(pElem); pReader->nTerm = fts3HashKeysize(pElem); - pReader->nNode = pReader->nDoclist = pList->nData + 1; - pReader->aNode = pReader->aDoclist = pList->aData; + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; pReader->ppNextElem++; assert( pReader->aNode ); } @@ -146017,12 +150061,14 @@ SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( ** second argument. */ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ - if( pReader && !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); - sqlite3_blob_close(pReader->pBlob); } + sqlite3_blob_close(pReader->pBlob); } sqlite3_free(pReader); } @@ -147635,7 +151681,7 @@ static int fts3SegmentMerge( ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ - if( csr.nSegment==1 ){ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ rc = SQLITE_DONE; goto finished; } @@ -147965,7 +152011,7 @@ static int fts3DoRebuild(Fts3Table *p){ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; int iLangid = langidFromSelect(p, pStmt); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){ if( p->abNotindexed[iCol]==0 ){ @@ -149277,10 +153323,11 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = nMin; + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); }else{ nSeg = -1; } @@ -150070,7 +154117,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, iLangid, *pRowid); + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); @@ -154334,7 +158381,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_NOMEM; } - nRow = pRtree->nRowEst / (iIdx + 1); + nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; setEstimatedRows(pIdxInfo, nRow); @@ -156169,6 +160216,38 @@ static void xFree(void *p){ } /* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* ** Compare two UTF-8 strings for equality where the first string is ** a "LIKE" expression. Return true (1) if they are the same and ** false (0) if they are different. @@ -156181,16 +160260,14 @@ static int icuLikeCompare( static const int MATCH_ONE = (UChar32)'_'; static const int MATCH_ALL = (UChar32)'%'; - int iPattern = 0; /* Current byte index in zPattern */ - int iString = 0; /* Current byte index in zString */ - int prevEscape = 0; /* True if the previous character was uEsc */ - while( zPattern[iPattern]!=0 ){ + while( 1 ){ /* Read (and consume) the next character from the input pattern. */ UChar32 uPattern; - U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; /* There are now 4 possibilities: ** @@ -156207,28 +160284,28 @@ static int icuLikeCompare( ** MATCH_ALL. For each MATCH_ONE, skip one character in the ** test string. */ - while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ if( c==MATCH_ONE ){ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); } - iPattern++; + zPattern++; } - if( zPattern[iPattern]==0 ) return 1; + if( *zPattern==0 ) return 1; - while( zString[iString] ){ - if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ return 1; } - U8_FWD_1_UNSAFE(zString, iString); + SQLITE_ICU_SKIP_UTF8(zString); } return 0; }else if( !prevEscape && uPattern==MATCH_ONE ){ /* Case 2. */ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); }else if( !prevEscape && uPattern==uEsc){ /* Case 3. */ @@ -156237,7 +160314,7 @@ static int icuLikeCompare( }else{ /* Case 4. */ UChar32 uString; - U8_NEXT_UNSAFE(zString, iString, uString); + SQLITE_ICU_READ_UTF8(zString, uString); uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); if( uString!=uPattern ){ @@ -156247,7 +160324,7 @@ static int icuLikeCompare( } } - return zString[iString]==0; + return *zString==0; } /* @@ -156433,11 +160510,11 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ */ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ const UChar *zInput; - UChar *zOutput; + UChar *zOutput = 0; int nInput; - int nOutput; - - UErrorCode status = U_ZERO_ERROR; + int nOut; + int cnt; + UErrorCode status; const char *zLocale = 0; assert(nArg==1 || nArg==2); @@ -156449,26 +160526,34 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ if( !zInput ){ return; } - nInput = sqlite3_value_bytes16(apArg[0]); - - nOutput = nInput * 2 + 2; - zOutput = sqlite3_malloc(nOutput); - if( !zOutput ){ + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); return; } - if( sqlite3_user_data(p) ){ - u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - }else{ - u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - } - - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "u_strToLower()/u_strToUpper", status); - return; + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( sqlite3_user_data(p) ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } + if( !U_SUCCESS(status) ){ + if( status==U_BUFFER_OVERFLOW_ERROR ) continue; + icuFunctionError(p, + sqlite3_user_data(p) ? "u_strToUpper" : "u_strToLower", status); + return; + } } - - sqlite3_result_text16(p, zOutput, -1, xFree); + sqlite3_result_text16(p, zOutput, nOut, xFree); } /* @@ -156964,10 +161049,6 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( /* #include <string.h> */ /* #include <stdio.h> */ -#if !defined(_WIN32) -/* # include <unistd.h> */ -#endif - /* #include "sqlite3.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) @@ -157072,6 +161153,18 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** The order of the columns in the data_% table does not matter. ** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_<target> naming scheme. +** +** Instead of the plain data_<target> naming scheme, RBU database tables +** may also be named data<integer>_<target>, where <integer> is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the the <integer> portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column ** named "rbu_rowid". This column is mapped to the tables implicit primary @@ -157152,6 +161245,14 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; ** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** ** If the target database table is a virtual table or a table with no PRIMARY ** KEY, the rbu_control value should not include a character corresponding ** to the rbu_rowid value. For example, this: @@ -157224,6 +161325,10 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( /* #include "sqlite3.h" ** Required for error code definitions ** */ +#if 0 +extern "C" { +#endif + typedef struct sqlite3rbu sqlite3rbu; /* @@ -157292,6 +161397,9 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( ** If an error has occurred, either while opening or stepping the RBU object, ** this function may return NULL. The error code and message may be collected ** when sqlite3rbu_close() is called. +** +** Database handles returned by this function remain valid until the next +** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). */ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu); @@ -157310,6 +161418,18 @@ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu); SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *pRbu); /* +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu); + +/* ** Close an RBU handle. ** ** If the RBU update has been completely applied, mark the RBU database @@ -157337,6 +161457,48 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); /* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + +/* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, ** then the new RBU VFS uses the default system VFS to access the file-system. @@ -157390,11 +161552,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha */ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); +#if 0 +} /* end of the 'extern "C"' block */ +#endif + #endif /* _SQLITE3RBU_H */ /************** End of sqlite3rbu.h ******************************************/ /************** Continuing where we left off in sqlite3rbu.c *****************/ +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif + /* Maximum number of prepared UPDATE statements held by this module */ #define SQLITE_RBU_UPDATE_CACHESIZE 16 @@ -157449,14 +161619,15 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -157502,6 +161673,7 @@ struct RbuState { i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { @@ -157541,17 +161713,19 @@ struct RbuObjIter { /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ const char *zIdx; /* Name of target db index (or null) */ int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ - sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zTbl */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ /* Last UPDATE used (for PK b-tree updates only), or NULL. */ RbuUpdateStmt *pRbuUpdate; @@ -157581,10 +161755,11 @@ struct RbuObjIter { */ #define RBU_INSERT 1 /* Insert on a main table b-tree */ #define RBU_DELETE 2 /* Delete a row from a main table b-tree */ -#define RBU_IDX_DELETE 3 /* Delete a row from an aux. index b-tree */ -#define RBU_IDX_INSERT 4 /* Insert on an aux. index b-tree */ -#define RBU_UPDATE 5 /* Update a row in a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ /* ** A single step of an incremental checkpoint - frame iWalFrame of the wal @@ -157597,6 +161772,43 @@ struct RbuFrame { /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ @@ -157614,6 +161826,7 @@ struct sqlite3rbu { const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -157662,6 +161875,252 @@ struct rbu_file { }; +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return @@ -157828,7 +162287,8 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - rc = pIter->zTbl ? SQLITE_OK : SQLITE_NOMEM; + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; } }else{ if( pIter->zIdx==0 ){ @@ -157859,6 +162319,40 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ return rc; } + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one argument - the name of a table in the RBU database. If the +** table name matches the pattern: +** +** data[0-9]_<name> +** +** where <name> is any sequence of 1 or more characters, <name> is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +*/ +static void rbuTargetNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zIn; + assert( argc==1 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + } + } +} + /* ** Initialize the iterator structure passed as the second argument. ** @@ -157872,8 +162366,9 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT substr(name, 6) FROM sqlite_master " - "WHERE type IN ('table', 'view') AND name LIKE 'data_%'" + "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + "ORDER BY name" ); if( rc==SQLITE_OK ){ @@ -157954,7 +162449,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ p->rc = SQLITE_NOMEM; }else{ @@ -158000,8 +162495,8 @@ static char *rbuStrndup(const char *zStr, int *pRc){ assert( *pRc==SQLITE_OK ); if( zStr ){ - int nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc(nCopy); + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); if( zRet ){ memcpy(zRet, zStr, nCopy); }else{ @@ -158140,7 +162635,7 @@ static void rbuTableType( } rbuTableType_end: { - int i; + unsigned int i; for(i=0; i<sizeof(aStmt)/sizeof(aStmt[0]); i++){ rbuFinalize(p, aStmt[i]); } @@ -158162,6 +162657,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; @@ -158175,6 +162671,12 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; } rbuFinalize(p, pList); @@ -158219,7 +162721,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ ** of the input table. Ignore any input table columns that begin with ** "rbu_". */ p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT * FROM 'data_%q'", pIter->zTbl) + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) ); if( p->rc==SQLITE_OK ){ nCol = sqlite3_column_count(pStmt); @@ -158244,7 +162746,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf( - "table data_%q %s rbu_rowid column", pIter->zTbl, + "table %q %s rbu_rowid column", pIter->zDataTbl, (bRbuRowid ? "may not have" : "requires") ); } @@ -158265,8 +162767,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ } if( i==pIter->nTblCol ){ p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("column missing from data_%q: %s", - pIter->zTbl, zName + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName ); }else{ int iPk = sqlite3_column_int(pStmt, 5); @@ -158288,6 +162790,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ rbuFinalize(p, pStmt); rbuObjIterCacheIndexedCols(p, pIter); assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); } return p->rc; @@ -158553,7 +163056,7 @@ static char *rbuObjIterGetSetlist( if( p->rc==SQLITE_OK ){ int i; - if( strlen(zMask)!=pIter->nTblCol ){ + if( (int)strlen(zMask)!=pIter->nTblCol ){ rbuBadControlError(p); }else{ const char *zSep = ""; @@ -158565,12 +163068,18 @@ static char *rbuObjIterGetSetlist( ); zSep = ", "; } - if( c=='d' ){ + else if( c=='d' ){ zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } } } } @@ -158821,7 +163330,7 @@ static void rbuObjIterPrepareTmpInsert( p->rc = prepareFreeAndCollectError( p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", - p->zStateDb, pIter->zTbl, zCollist, zRbuRowid, zBind + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind )); } } @@ -158835,6 +163344,14 @@ static void rbuTmpInsertFunc( int rc = SQLITE_OK; int i; + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + for(i=0; rc==SQLITE_OK && i<nVal; i++){ rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]); } @@ -158917,18 +163434,18 @@ static int rbuObjIterPrepareAll( if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM 'data_%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " - "UNION ALL " "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "UNION ALL " + "SELECT %s, rbu_control FROM '%q' " + "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " "ORDER BY %s%s", - zCollist, pIter->zTbl, - zCollist, p->zStateDb, pIter->zTbl, + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, pIter->zDataTbl, zCollist, zLimit ); } @@ -158952,16 +163469,6 @@ static int rbuObjIterPrepareAll( zCollist = rbuObjIterGetCollist(p, pIter); pIter->nCol = pIter->nTblCol; - /* Create the SELECT statement to read keys from data_xxx */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM 'data_%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), zTbl, zLimit - ) - ); - } - /* Create the imposter table or tables (if required). */ rbuCreateImposterTable(p, pIter); rbuCreateImposterTable2(p, pIter); @@ -158995,26 +163502,26 @@ static int rbuObjIterPrepareAll( /* Create the rbu_tmp_xxx table and the triggers to populate it. */ rbuMPrintfExec(p, p->dbRbu, "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " - "SELECT *%s FROM 'data_%q' WHERE 0;" - , p->zStateDb - , zTbl, (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") - , zTbl + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl ); rbuMPrintfExec(p, p->dbMain, "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(2, %s);" + " SELECT rbu_tmp_insert(3, %s);" "END;" "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(2, %s);" + " SELECT rbu_tmp_insert(3, %s);" "END;" "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" + " SELECT rbu_tmp_insert(4, %s);" "END;", zWrite, zTbl, zOldlist, zWrite, zTbl, zOldlist, @@ -159034,6 +163541,17 @@ static int rbuObjIterPrepareAll( rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); } + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s, rbu_control%s FROM '%q'%s", + zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + pIter->zDataTbl, zLimit + ) + ); + } + sqlite3_free(zWhere); sqlite3_free(zOldlist); sqlite3_free(zNewlist); @@ -159165,6 +163683,18 @@ static void rbuOpenDatabase(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); @@ -159330,7 +163860,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; RbuFrame *aNew; - aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); if( aNew==0 ) return SQLITE_NOMEM; pRbu->aFrame = aNew; pRbu->nFrameAlloc = nNew; @@ -159386,6 +163916,30 @@ static void rbuLockDatabase(sqlite3rbu *p){ } } +#if defined(_WIN32_WCE) +static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ + int nChar; + LPWSTR zWideFilename; + + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) ); + if( zWideFilename==0 ){ + return 0; + } + memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0])); + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + } + return zWideFilename; +} +#endif + /* ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock ** on the database file. This proc moves the *-oal file to the *-wal path, @@ -159420,10 +163974,37 @@ static void rbuMoveOalFile(sqlite3rbu *p){ rbuObjIterFinalize(&p->objiter); sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + p->dbMain = 0; + p->dbRbu = 0; + +#if defined(_WIN32_WCE) + { + LPWSTR zWideOal; + LPWSTR zWideWal; + + zWideOal = rbuWinUtf8ToUnicode(zOal); + if( zWideOal ){ + zWideWal = rbuWinUtf8ToUnicode(zWal); + if( zWideWal ){ + if( MoveFileW(zWideOal, zWideWal) ){ + p->rc = SQLITE_OK; + }else{ + p->rc = SQLITE_IOERR; + } + sqlite3_free(zWideWal); + }else{ + p->rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOal); + }else{ + p->rc = SQLITE_IOERR_NOMEM; + } + } +#else p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; +#endif + if( p->rc==SQLITE_OK ){ - p->dbMain = 0; - p->dbRbu = 0; rbuOpenDatabase(p); rbuSetupCheckpoint(p, 0); } @@ -159458,14 +164039,12 @@ static int rbuStepType(sqlite3rbu *p, const char **pzMask){ switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){ case SQLITE_INTEGER: { int iVal = sqlite3_column_int(p->objiter.pSelect, iCol); - if( iVal==0 ){ - res = RBU_INSERT; - }else if( iVal==1 ){ - res = RBU_DELETE; - }else if( iVal==2 ){ - res = RBU_IDX_DELETE; - }else if( iVal==3 ){ - res = RBU_IDX_INSERT; + switch( iVal ){ + case 0: res = RBU_INSERT; break; + case 1: res = RBU_DELETE; break; + case 2: res = RBU_REPLACE; break; + case 3: res = RBU_IDX_DELETE; break; + case 4: res = RBU_IDX_INSERT; break; } break; } @@ -159505,6 +164084,78 @@ static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ #endif /* +** Argument eType must be one of RBU_INSERT, RBU_DELETE, RBU_IDX_INSERT or +** RBU_IDX_DELETE. This function performs the work of a single +** sqlite3rbu_step() call for the type of operation specified by eType. +*/ +static void rbuStepOneOp(sqlite3rbu *p, int eType){ + RbuObjIter *pIter = &p->objiter; + sqlite3_value *pVal; + sqlite3_stmt *pWriter; + int i; + + assert( p->rc==SQLITE_OK ); + assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } + + if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ + pWriter = pIter->pDelete; + }else{ + pWriter = pIter->pInsert; + } + + for(i=0; i<pIter->nCol; i++){ + /* If this is an INSERT into a table b-tree and the table has an + ** explicit INTEGER PRIMARY KEY, check that this is not an attempt + ** to write a NULL into the IPK column. That is not permitted. */ + if( eType==RBU_INSERT + && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] + && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL + ){ + p->rc = SQLITE_MISMATCH; + p->zErrmsg = sqlite3_mprintf("datatype mismatch"); + return; + } + + if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){ + continue; + } + + pVal = sqlite3_column_value(pIter->pSelect, i); + p->rc = sqlite3_bind_value(pWriter, i+1, pVal); + if( p->rc ) return; + } + if( pIter->zIdx==0 + && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT <cols>, rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } + if( p->rc==SQLITE_OK ){ + sqlite3_step(pWriter); + p->rc = resetAndCollectError(pWriter, &p->zErrmsg); + } +} + +/* ** This function does the work for an sqlite3rbu_step() call. ** ** The object-iterator (p->objiter) currently points to a valid object, @@ -159518,82 +164169,40 @@ static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ static int rbuStep(sqlite3rbu *p){ RbuObjIter *pIter = &p->objiter; const char *zMask = 0; - int i; int eType = rbuStepType(p, &zMask); if( eType ){ + assert( eType==RBU_INSERT || eType==RBU_DELETE + || eType==RBU_REPLACE || eType==RBU_IDX_DELETE + || eType==RBU_IDX_INSERT || eType==RBU_UPDATE + ); assert( eType!=RBU_UPDATE || pIter->zIdx==0 ); - if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){ + if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){ rbuBadControlError(p); } - else if( - eType==RBU_INSERT - || eType==RBU_DELETE - || eType==RBU_IDX_DELETE - || eType==RBU_IDX_INSERT - ){ - sqlite3_value *pVal; - sqlite3_stmt *pWriter; - - assert( eType!=RBU_UPDATE ); - assert( eType!=RBU_DELETE || pIter->zIdx==0 ); - - if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ - pWriter = pIter->pDelete; - }else{ - pWriter = pIter->pInsert; - } - - for(i=0; i<pIter->nCol; i++){ - /* If this is an INSERT into a table b-tree and the table has an - ** explicit INTEGER PRIMARY KEY, check that this is not an attempt - ** to write a NULL into the IPK column. That is not permitted. */ - if( eType==RBU_INSERT - && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] - && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL - ){ - p->rc = SQLITE_MISMATCH; - p->zErrmsg = sqlite3_mprintf("datatype mismatch"); - goto step_out; - } - - if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){ - continue; - } - - pVal = sqlite3_column_value(pIter->pSelect, i); - p->rc = sqlite3_bind_value(pWriter, i+1, pVal); - if( p->rc ) goto step_out; - } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT <cols>, rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); - } - if( p->rc==SQLITE_OK ){ - sqlite3_step(pWriter); - p->rc = resetAndCollectError(pWriter, &p->zErrmsg); + else if( eType==RBU_REPLACE ){ + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); } - }else{ + if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); + } + else if( eType!=RBU_UPDATE ){ + rbuStepOneOp(p, eType); + } + else{ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ + int i; for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){ char c = zMask[pIter->aiSrcOrder[i]]; pVal = sqlite3_column_value(pIter->pSelect, i); - if( pIter->abTblPk[i] || c=='x' || c=='d' ){ + if( pIter->abTblPk[i] || c!='.' ){ p->rc = sqlite3_bind_value(pUpdate, i+1, pVal); } } @@ -159612,8 +164221,6 @@ static int rbuStep(sqlite3rbu *p){ } } } - - step_out: return p->rc; } @@ -159666,6 +164273,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, @@ -159675,7 +164283,8 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -159705,7 +164314,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){ ** But the contents can be deleted. */ if( pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, - "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zTbl + "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); } }else{ @@ -159862,6 +164471,10 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); break; + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + default: rc = SQLITE_CORRUPT; break; @@ -159928,10 +164541,13 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ ** leave an error code and error message in the rbu handle. */ static void rbuDeleteOalFile(sqlite3rbu *p){ - char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); - assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); - unlink(zOal); - sqlite3_free(zOal); + char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget); + if( zOal ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 ); + pVfs->xDelete(pVfs, zOal, 0); + sqlite3_free(zOal); + } } /* @@ -159967,6 +164583,100 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } /* +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. +*/ +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} + +/* ** Open and return a new RBU handle. */ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( @@ -159975,11 +164685,12 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( const char *zState ){ sqlite3rbu *p; - int nTarget = strlen(zTarget); - int nRbu = strlen(zRbu); - int nState = zState ? strlen(zState) : 0; + size_t nTarget = strlen(zTarget); + size_t nRbu = strlen(zRbu); + size_t nState = zState ? strlen(zState) : 0; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; - p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1); + p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ RbuState *pState = 0; @@ -160010,9 +164721,11 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; + p->nPhaseOneStep = pState->nPhaseOneStep; } p->nProgress = pState->nProgress; p->iOalSz = pState->iOalSz; @@ -160044,14 +164757,25 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ + sqlite3 *db = p->dbMain; /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ - p->rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); } - + + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = rbuObjIterFirst(p, &p->objiter); @@ -160105,7 +164829,7 @@ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ int i; - int nErrmsg = strlen(p->zErrmsg); + size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ int nDel = 8; @@ -160165,6 +164889,68 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} + +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ + int rc = p->rc; + + if( rc==SQLITE_DONE ) return SQLITE_OK; + + assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); + if( p->eStage==RBU_STAGE_OAL ){ + assert( rc!=SQLITE_DONE ); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); + } + + p->rc = rc; + rbuSaveState(p, p->eStage); + rc = p->rc; + + if( p->eStage==RBU_STAGE_OAL ){ + assert( rc!=SQLITE_DONE ); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, 0); + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0); + } + + p->rc = rc; + return rc; +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: @@ -160543,7 +165329,7 @@ static int rbuVfsShmMap( if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); - char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); if( apNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -160554,7 +165340,7 @@ static int rbuVfsShmMap( } if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ - char *pNew = (char*)sqlite3_malloc(szRegion); + char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -160612,7 +165398,7 @@ static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ rbu_file *pDb; sqlite3_mutex_enter(pRbuVfs->mutex); - for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext); + for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){} sqlite3_mutex_leave(pRbuVfs->mutex); return pDb; } @@ -160644,7 +165430,8 @@ static int rbuVfsOpen( rbuVfsShmMap, /* xShmMap */ rbuVfsShmLock, /* xShmLock */ rbuVfsShmBarrier, /* xShmBarrier */ - rbuVfsShmUnmap /* xShmUnmap */ + rbuVfsShmUnmap, /* xShmUnmap */ + 0, 0 /* xFetch, xUnfetch */ }; rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs; @@ -160663,7 +165450,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = strlen(zName); + int n = (int)strlen(zName); const char *z = &zName[n]; if( flags & SQLITE_OPEN_URI ){ int odd = 0; @@ -160689,8 +165476,8 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - int nCopy = strlen(zName); - char *zCopy = sqlite3_malloc(nCopy+2); + size_t nCopy = strlen(zName); + char *zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ memcpy(zCopy, zName, nCopy); zCopy[nCopy-3] = 'o'; @@ -160919,13 +165706,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha }; rbu_vfs *pNew = 0; /* Newly allocated VFS */ - int nName; int rc = SQLITE_OK; + size_t nName; + size_t nByte; - int nByte; nName = strlen(zName); nByte = sizeof(rbu_vfs) + nName + 1; - pNew = (rbu_vfs*)sqlite3_malloc(nByte); + pNew = (rbu_vfs*)sqlite3_malloc64(nByte); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -160986,6 +165773,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha ** information from an SQLite database in order to implement the ** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script ** for an example implementation. +** +** Additional information is available on the "dbstat.html" page of the +** official SQLite documentation. */ /* #include "sqliteInt.h" ** Requires access to internal data structures ** */ @@ -161034,7 +165824,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha " unused INTEGER, /* Bytes of unused space on this page */" \ " mx_payload INTEGER, /* Largest payload size of all cells */" \ " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER /* Size of the page */" \ + " pgsize INTEGER, /* Size of the page */" \ + " schema TEXT HIDDEN /* Database schema being analyzed */" \ ");" @@ -161072,6 +165863,7 @@ struct StatCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Iterates through set of root pages */ int isEof; /* After pStmt has returned SQLITE_DONE */ + int iDb; /* Schema used for this query */ StatPage aPage[32]; int iPage; /* Current entry in aPage[] */ @@ -161114,7 +165906,9 @@ static int statConnect( int iDb; if( argc>=4 ){ - iDb = sqlite3FindDbName(db, argv[3]); + Token nm; + sqlite3TokenInit(&nm, (char*)argv[3]); + iDb = sqlite3FindDb(db, &nm); if( iDb<0 ){ *pzErr = sqlite3_mprintf("no such database: %s", argv[3]); return SQLITE_ERROR; @@ -161125,7 +165919,7 @@ static int statConnect( rc = sqlite3_declare_vtab(db, VTAB_SCHEMA); if( rc==SQLITE_OK ){ pTab = (StatTable *)sqlite3_malloc64(sizeof(StatTable)); - if( pTab==0 ) rc = SQLITE_NOMEM; + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; } assert( rc==SQLITE_OK || pTab==0 ); @@ -161149,9 +165943,32 @@ static int statDisconnect(sqlite3_vtab *pVtab){ /* ** There is no "best-index". This virtual table always does a linear -** scan of the binary VFS log file. +** scan. However, a schema=? constraint should cause this table to +** operate on a different database schema, so check for it. +** +** idxNum is normally 0, but will be 1 if a schema=? constraint exists. */ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + + /* Look for a valid schema=? constraint. If found, change the idxNum to + ** 1 and request the value of that constraint be sent to xFilter. And + ** lower the cost estimate to encourage the constrained version to be + ** used. + */ + for(i=0; i<pIdxInfo->nConstraint; i++){ + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + /* Records are always returned in ascending order of (name, path). ** If this will satisfy the client, set the orderByConsumed flag so that @@ -161171,7 +165988,6 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = 10.0; return SQLITE_OK; } @@ -161181,36 +165997,18 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ StatTable *pTab = (StatTable *)pVTab; StatCursor *pCsr; - int rc; pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor)); if( pCsr==0 ){ - rc = SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ - char *zSql; memset(pCsr, 0, sizeof(StatCursor)); pCsr->base.pVtab = pVTab; - - zSql = sqlite3_mprintf( - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type" - " FROM \"%w\".sqlite_master WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pTab->iDb].zName); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - sqlite3_free(pCsr); - pCsr = 0; - } + pCsr->iDb = pTab->iDb; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; - return rc; + return SQLITE_OK; } static void statClearPage(StatPage *p){ @@ -161235,6 +166033,7 @@ static void statResetCsr(StatCursor *pCsr){ pCsr->iPage = 0; sqlite3_free(pCsr->zPath); pCsr->zPath = 0; + pCsr->isEof = 0; } /* @@ -161307,7 +166106,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt); sqlite3BtreeLeave(pBt); p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell)); - if( p->aCell==0 ) return SQLITE_NOMEM; + if( p->aCell==0 ) return SQLITE_NOMEM_BKPT; memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell)); for(i=0; i<p->nCell; i++){ @@ -161340,13 +166139,13 @@ static int statDecodePage(Btree *pBt, StatPage *p){ pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); pCell->nOvfl = nOvfl; pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl); - if( pCell->aOvfl==0 ) return SQLITE_NOMEM; + if( pCell->aOvfl==0 ) return SQLITE_NOMEM_BKPT; pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]); for(j=1; j<nOvfl; j++){ int rc; u32 iPrev = pCell->aOvfl[j-1]; DbPage *pPg = 0; - rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg); + rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg, 0); if( rc!=SQLITE_OK ){ assert( pPg==0 ); return rc; @@ -161397,7 +166196,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ char *z; StatCursor *pCsr = (StatCursor *)pCursor; StatTable *pTab = (StatTable *)pCursor->pVtab; - Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Btree *pBt = pTab->db->aDb[pCsr->iDb].pBt; Pager *pPager = sqlite3BtreePager(pBt); sqlite3_free(pCsr->zPath); @@ -161414,12 +166213,12 @@ statNextRestart: pCsr->isEof = 1; return sqlite3_reset(pCsr->pStmt); } - rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg); + rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg, 0); pCsr->aPage[0].iPgno = iRoot; pCsr->aPage[0].iCell = 0; pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); pCsr->iPage = 0; - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; }else{ pCsr->isEof = 1; return sqlite3_reset(pCsr->pStmt); @@ -161454,7 +166253,7 @@ statNextRestart: } pCell->iOvfl++; statSizeAndOffset(pCsr); - return z==0 ? SQLITE_NOMEM : SQLITE_OK; + return z==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; } if( p->iRightChildPg ) break; p->iCell++; @@ -161474,11 +166273,11 @@ statNextRestart: }else{ p[1].iPgno = p->aCell[p->iCell].iChildPg; } - rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg); + rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg, 0); p[1].iCell = 0; p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); p->iCell++; - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; } @@ -161512,7 +166311,7 @@ statNextRestart: pCsr->nUnused = p->nUnused; pCsr->nMxPayload = p->nMxPayload; pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; nPayload = 0; for(i=0; i<p->nCell; i++){ nPayload += p->aCell[i].nLocal; @@ -161535,9 +166334,43 @@ static int statFilter( int argc, sqlite3_value **argv ){ StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable*)(pCursor->pVtab); + char *zSql; + int rc = SQLITE_OK; + char *zMaster; + if( idxNum==1 ){ + const char *zDbase = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); + if( pCsr->iDb<0 ){ + sqlite3_free(pCursor->pVtab->zErrMsg); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase); + return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM_BKPT; + } + }else{ + pCsr->iDb = pTab->iDb; + } statResetCsr(pCsr); - return statNext(pCursor); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + zMaster = pCsr->iDb==1 ? "sqlite_temp_master" : "sqlite_master"; + zSql = sqlite3_mprintf( + "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" + " UNION ALL " + "SELECT name, rootpage, type" + " FROM \"%w\".%s WHERE rootpage!=0" + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + if( zSql==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + rc = statNext(pCursor); + } + return rc; } static int statColumn( @@ -161574,10 +166407,15 @@ static int statColumn( case 8: /* pgoffset */ sqlite3_result_int64(ctx, pCsr->iOffset); break; - default: /* pgsize */ - assert( i==9 ); + case 9: /* pgsize */ sqlite3_result_int(ctx, pCsr->szPage); break; + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + int iDb = pCsr->iDb; + sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + break; + } } return SQLITE_OK; } @@ -161621,3 +166459,22846 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; } #endif /* SQLITE_ENABLE_DBSTAT_VTAB */ /************** End of dbstat.c **********************************************/ +/************** Begin file json1.c *******************************************/ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) +#if !defined(_SQLITEINT_H_) +/* #include "sqlite3ext.h" */ +#endif +SQLITE_EXTENSION_INIT1 +/* #include <assert.h> */ +/* #include <string.h> */ +/* #include <stdlib.h> */ +/* #include <stdarg.h> */ + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +#else + /* Use the standard library for separate compilation */ +#include <ctype.h> /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) + +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned char u8; +#endif + +/* Objects */ +typedef struct JsonString JsonString; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. +*/ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 bErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* JSON type values +*/ +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ + + +/* A single node of parsed JSON +*/ +struct JsonNode { + u8 eType; /* One of the JSON_ type values */ + u8 jnFlags; /* JNODE flags */ + u8 iVal; /* Replacement value when JNODE_REPLACE */ + u32 n; /* Bytes of content, or number of sub-nodes */ + union { + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ + } u; +}; + +/* A completely parsed JSON string +*/ +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ + u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ +}; + +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ + +/* Set the JsonString object to an empty string +*/ +static void jsonZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the JsonString object +*/ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->bErr = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the JsonString object back to its +** initial state. +*/ +static void jsonReset(JsonString *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(JsonString *p){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(JsonString *p, u32 N){ + u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->bErr ) return 1; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the JsonString string. +*/ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + +/* Append a single character +*/ +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; i<N; i++){ + unsigned char c = ((unsigned const char*)zIn)[i]; + if( c=='"' || c=='\\' ){ + json_simple_escape: + if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; + } + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsed<p->nAlloc ); +} + +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 2; + jsonReset(p); + } + break; + } + } +} + + +/* Make the JSON in p the result of the SQL function. +*/ +static void jsonResult(JsonString *p){ + if( p->bErr==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. +*/ +static void jsonRenderNode( + JsonNode *pNode, /* The node to render */ + JsonString *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + break; + } + /* Fall through into the next case */ + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ + if( pNode[j].jnFlags & JNODE_REPLACE ){ + jsonAppendSeparator(pOut); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + } + }else{ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + if( pNode[j+1].jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + }else{ + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, '}'); + break; + } + } +} + +/* +** Return a JsonNode and all its descendents as a JSON string. +*/ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; + } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); + int_done: + break; + int_as_real: /* fall through to real */; + } + case JSON_REAL: { + double r; +#ifdef SQLITE_AMALGAMATION + const char *z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); +#else + r = strtod(pNode->u.zJContent, 0); +#endif + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { +#if 0 /* Never happens because JNODE_RAW is only set by json_set(), + ** json_insert() and json_replace() and those routines do not + ** call jsonReturn() */ + if( pNode->jnFlags & JNODE_RAW ){ + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else +#endif + assert( (pNode->jnFlags & JNODE_RAW)==0 ); + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->u.zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i<n-1; i++){ + char c = z[i]; + if( c!='\\' ){ + zOut[j++] = c; + }else{ + c = z[++i]; + if( c=='u' ){ + u32 v = 0, k; + for(k=0; k<4 && i<n-2; i++, k++){ + c = z[i+1]; + if( c>='0' && c<='9' ) v = v*16 + c - '0'; + else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; + else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; + else break; + } + if( v==0 ) break; + if( v<=0x7f ){ + zOut[j++] = (char)v; + }else if( v<=0x7ff ){ + zOut[j++] = (char)(0xc0 | (v>>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = (char)(0xe0 | (v>>12)); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pNode, pCtx, aReplace); + break; + } + } +} + +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define JSON_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif + + +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNode<pParse->nAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); +} + +/* +** Create a new JsonNode instance based on the arguments and append that +** instance to the JsonParse. Return the index in pParse->aNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + return jsonParseAddNodeExpand(pParse, eType, n, zContent); + } + p = &pParse->aNode[pParse->nNode]; + p->eType = (u8)eType; + p->jnFlags = 0; + p->iVal = 0; + p->n = n; + p->u.zJContent = zContent; + return pParse->nNode++; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + JsonNode *pNode; + while( safe_isspace(pParse->zJson[i]) ){ i++; } + if( (c = pParse->zJson[i])=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + if( pParse->oom ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; + j = x; + while( safe_isspace(pParse->zJson[j]) ){ j++; } + if( pParse->zJson[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + if( x<0 ) return -1; + j = x; + while( safe_isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + j = x; + while( safe_isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + u8 jnFlags = 0; + j = i+1; + for(;;){ + c = pParse->zJson[j]; + if( c==0 ) return -1; + if( c=='\\' ){ + c = pParse->zJson[++j]; + if( c==0 ) return -1; + jnFlags = JNODE_ESCAPE; + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + return j+1; + }else if( c=='n' + && strncmp(pParse->zJson+i,"null",4)==0 + && !safe_isalnum(pParse->zJson[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(pParse->zJson+i,"true",4)==0 + && !safe_isalnum(pParse->zJson[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(pParse->zJson+i,"false",5)==0 + && !safe_isalnum(pParse->zJson[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + j = i+1; + for(;; j++){ + c = pParse->zJson[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( pParse->zJson[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( pParse->zJson[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = pParse->zJson[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = pParse->zJson[j+1]; + } + if( c<'0' || c>'9' ) return -1; + continue; + } + break; + } + if( pParse->zJson[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &pParse->zJson[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ +){ + int i; + memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + while( safe_isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + +/* +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. +*/ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; + }else{ + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + } +} + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. +*/ +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot = &pParse->aNode[iRoot]; + if( zPath[0]==0 ) return pRoot; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' && safe_isdigit(zPath[1]) ){ + if( pRoot->eType!=JSON_ARRAY ) return 0; + i = 0; + j = 1; + while( safe_isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} + +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} + +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; + +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#ifdef SQLITE_DEBUG +/* +** The json_parse(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + + assert( argc==1 ); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); + jsonInit(&s, ctx); + for(i=0; i<x.nNode; i++){ + const char *zType; + if( x.aNode[i].jnFlags & JNODE_LABEL ){ + assert( x.aNode[i].eType==JSON_STRING ); + zType = "label"; + }else{ + zType = jsonType[x.aNode[i].eType]; + } + jsonPrintf(100, &s,"node %3u: %7s n=%-4d up=%-4d", + i, zType, x.aNode[i].n, x.aUp[i]); + if( x.aNode[i].u.zJContent!=0 ){ + jsonAppendRaw(&s, " ", 1); + jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); + } + jsonAppendRaw(&s, "\n", 1); + } + jsonParseReset(&x); + jsonResult(&s); +} + +/* +** The json_test1(JSON) function return true (1) if the input is JSON +** text generated by another json function. It returns (0) if the input +** is not known to be JSON. +*/ +static void jsonTest1Func( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAM(argc); + sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); +} +#endif /* SQLITE_DEBUG */ + +/**************************************************************************** +** Scalar SQL function implementations +****************************************************************************/ + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=0; i<argc; i++){ + jsonAppendSeparator(&jx); + jsonAppendValue(&jx, argv[i]); + } + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + + +/* +** json_array_length(JSON) +** json_array_length(JSON, PATH) +** +** Return the number of elements in the top-level JSON array. +** Return 0 if the input is not a well-formed JSON array. +*/ +static void jsonArrayLengthFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + sqlite3_int64 n = 0; + u32 i; + JsonNode *pNode; + + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(&x, zPath, 0, ctx); + }else{ + pNode = x.aNode; + } + if( pNode==0 ){ + x.nErr = 1; + }else if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + } + if( x.nErr==0 ) sqlite3_result_int64(ctx, n); + jsonParseReset(&x); +} + +/* +** json_extract(JSON, PATH, ...) +** +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + JsonString jx; + int i; + + if( argc<2 ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; i<argc; i++){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) break; + if( argc>2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); + } + } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); + jsonParseReset(&x); +} + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(ctx, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '{'); + for(i=0; i<argc; i+=2){ + if( sqlite3_value_type(argv[i])!=SQLITE_TEXT ){ + sqlite3_result_error(ctx, "json_object() labels must be TEXT", -1); + jsonReset(&jx); + return; + } + jsonAppendSeparator(&jx); + z = (const char*)sqlite3_value_text(argv[i]); + n = (u32)sqlite3_value_bytes(argv[i]); + jsonAppendString(&jx, z, n); + jsonAppendChar(&jx, ':'); + jsonAppendValue(&jx, argv[i+1]); + } + jsonAppendChar(&jx, '}'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + + +/* +** json_remove(JSON, PATH, ...) +** +** Remove the named elements from JSON and return the result. malformed +** JSON or PATH arguments result in an error. +*/ +static void jsonRemoveFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i++){ + zPath = (const char*)sqlite3_value_text(argv[i]); + if( zPath==0 ) goto remove_done; + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto remove_done; + if( pNode ) pNode->jnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(x.aNode, ctx, 0); + } +remove_done: + jsonParseReset(&x); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto replace_err; + if( pNode ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +replace_err: + jsonParseReset(&x); +} + +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(ctx); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + bApnd = 0; + pNode = jsonLookup(&x, zPath, &bApnd, ctx); + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +jsonSetDone: + jsonParseReset(&x); +} + +/* +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. +*/ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + const char *zPath; + JsonNode *pNode; + + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(&x, zPath, 0, ctx); + }else{ + pNode = x.aNode; + } + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + } + jsonParseReset(&x); +} + +/* +** json_valid(JSON) +** +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + int rc = 0; + + UNUSED_PARAM(argc); + if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){ + rc = 1; + } + jsonParseReset(&x); + sqlite3_result_int(ctx, rc); +} + + +/**************************************************************************** +** Aggregate SQL function implementations +****************************************************************************/ +/* +** json_group_array(VALUE) +** +** Return a JSON array composed of all values in the aggregate. +*/ +static void jsonArrayStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else{ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + jsonAppendValue(pStr, argv[0]); + } +} +static void jsonArrayFinal(sqlite3_context *ctx){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else{ + sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + +/* +** json_group_obj(NAME,VALUE) +** +** Return a JSON object composed of all names and values in the aggregate. +*/ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + const char *z; + u32 n; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else{ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + z = (const char*)sqlite3_value_text(argv[0]); + n = (u32)sqlite3_value_bytes(argv[0]); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendValue(pStr, argv[1]); + } +} +static void jsonObjectFinal(sqlite3_context *ctx){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + jsonAppendChar(pStr, '}'); + if( pStr->bErr ){ + if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else{ + sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zRoot; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAM(pzErr); + UNUSED_PARAM(argv); + UNUSED_PARAM(argc); + UNUSED_PARAM(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + + UNUSED_PARAM(p); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zRoot); + jsonParseReset(&p->sParse); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->zJson = 0; + p->zRoot = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->i<p->iEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } + } + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + } + return SQLITE_OK; +} + +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); + } +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( i ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else{ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + } + case JEACH_ROOT: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + assert( i==JEACH_JSON ); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int jsonIdx = -1; + int rootIdx = -1; + const struct sqlite3_index_constraint *pConstraint; + + UNUSED_PARAM(tab); + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case JEACH_JSON: jsonIdx = i; break; + case JEACH_ROOT: rootIdx = i; break; + default: /* no-op */ break; + } + } + if( jsonIdx<0 ){ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = 1e99; + }else{ + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; + if( rootIdx<0 ){ + pIdxInfo->idxNum = 1; + }else{ + pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[rootIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zRoot = 0; + sqlite3_int64 n; + + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc64( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, (size_t)n+1); + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } + jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; + }else{ + JsonNode *pNode = 0; + if( idxNum==3 ){ + const char *zErr = 0; + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + zErr = zRoot; + }else{ + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + } + if( zErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; + if( p->bRecursive ){ + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/**************************************************************************** +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ + +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ + int rc = SQLITE_OK; + unsigned int i; + static const struct { + const char *zName; + int nArg; + int flag; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, + +#if SQLITE_DEBUG + /* DEBUG and TESTING functions */ + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, +#endif + }; + static const struct { + const char *zName; + int nArg; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + } aAgg[] = { + { "json_group_array", 1, jsonArrayStep, jsonArrayFinal }, + { "json_group_object", 2, jsonObjectStep, jsonObjectFinal }, + }; +#ifndef SQLITE_OMIT_VIRTUALTABLE + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; +#endif + for(i=0; i<sizeof(aFunc)/sizeof(aFunc[0]) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zName, aFunc[i].nArg, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + (void*)&aFunc[i].flag, + aFunc[i].xFunc, 0, 0); + } + for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, + 0, aAgg[i].xStep, aAgg[i].xFinal); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0); + } +#endif + return rc; +} + + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_json_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return sqlite3Json1Init(db); +} +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) */ + +/************** End of json1.c ***********************************************/ +/************** Begin file fts5.c ********************************************/ + + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. The exception is if the table was created +** with the offsets=0 option specified. In this case *piOff is always +** set to -1. +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. For each row visited, the callback function +** passed as the fourth argument is invoked. The context and API objects +** passed to the callback function may be used to access the properties of +** each matched row. Invoking Api.xUserData() returns a copy of the pointer +** passed as the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and inititalize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +** </ul> +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +** <ol><li> By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +** <li> By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** <codeblock> +** ... MATCH 'first place'</codeblock> +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** <codeblock> +** ... MATCH '(first OR 1st) place'</codeblock> +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +** <li> By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +** </ol> +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** <codeblock> +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +**</codeblock> +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** <codeblock> +** ... MATCH '1s*'</codeblock> +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTS5INT_H +#define _FTS5INT_H + +/* #include "fts5.h" */ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 + +/* #include <string.h> */ +/* #include <assert.h> */ + +#ifndef SQLITE_AMALGAMATION + +typedef unsigned char u8; +typedef unsigned int u32; +typedef unsigned short u16; +typedef short i16; +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + +#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) + +#define testcase(x) +#define ALWAYS(x) 1 +#define NEVER(x) 0 + +#define MIN(x,y) (((x) < (y)) ? (x) : (y)) +#define MAX(x,y) (((x) > (y)) ? (x) : (y)) + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +*/ +# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#endif + +/* Truncate very long tokens to this many bytes. Hard limit is +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset +** field that occurs at the start of each leaf page (see fts5_index.c). */ +#define FTS5_MAX_TOKEN_SIZE 32768 + +/* +** Maximum number of prefix indexes on single FTS5 table. This must be +** less than 32. If it is set to anything large than that, an #error +** directive in fts5_index.c will cause the build to fail. +*/ +#define FTS5_MAX_PREFIX_INDEXES 31 + +#define FTS5_DEFAULT_NEARDIST 10 +#define FTS5_DEFAULT_RANK "bm25" + +/* Name of rank and rowid columns */ +#define FTS5_RANK_NAME "rank" +#define FTS5_ROWID_NAME "rowid" + +#ifdef SQLITE_DEBUG +# define FTS5_CORRUPT sqlite3Fts5Corrupt() +static int sqlite3Fts5Corrupt(void); +#else +# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB +#endif + +/* +** The assert_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +SQLITE_API extern int sqlite3_fts5_may_be_corrupt; +# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) +#else +# define assert_nc(x) assert(x) +#endif + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef UNUSED_PARAM2 +# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y) +#endif + +typedef struct Fts5Global Fts5Global; +typedef struct Fts5Colset Fts5Colset; + +/* If a NEAR() clump or phrase may only match a specific set of columns, +** then an object of the following type is used to record the set of columns. +** Each entry in the aiCol[] array is a column that may be matched. +** +** This object is used by fts5_expr.c and fts5_index.c. +*/ +struct Fts5Colset { + int nCol; + int aiCol[1]; +}; + + + +/************************************************************************** +** Interface to code in fts5_config.c. fts5_config.c contains contains code +** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. +*/ + +typedef struct Fts5Config Fts5Config; + +/* +** An instance of the following structure encodes all information that can +** be gleaned from the CREATE VIRTUAL TABLE statement. +** +** And all information loaded from the %_config table. +** +** nAutomerge: +** The minimum number of segments that an auto-merge operation should +** attempt to merge together. A value of 1 sets the object to use the +** compile time default. Zero disables auto-merge altogether. +** +** zContent: +** +** zContentRowid: +** The value of the content_rowid= option, if one was specified. Or +** the string "rowid" otherwise. This text is not quoted - if it is +** used as part of an SQL statement it needs to be quoted appropriately. +** +** zContentExprlist: +** +** pzErrmsg: +** This exists in order to allow the fts5_index.c module to return a +** decent error message if it encounters a file-format version it does +** not understand. +** +** bColumnsize: +** True if the %_docsize table is created. +** +** bPrefixIndex: +** This is only used for debugging. If set to false, any prefix indexes +** are ignored. This value is configured using: +** +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); +** +*/ +struct Fts5Config { + sqlite3 *db; /* Database handle */ + char *zDb; /* Database holding FTS index (e.g. "main") */ + char *zName; /* Name of FTS index */ + int nCol; /* Number of columns */ + char **azCol; /* Column names */ + u8 *abUnindexed; /* True for unindexed columns */ + int nPrefix; /* Number of prefix indexes */ + int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ + int eContent; /* An FTS5_CONTENT value */ + char *zContent; /* content table */ + char *zContentRowid; /* "content_rowid=" option value */ + int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int eDetail; /* FTS5_DETAIL_XXX value */ + char *zContentExprlist; + Fts5Tokenizer *pTok; + fts5_tokenizer *pTokApi; + + /* Values loaded from the %_config table */ + int iCookie; /* Incremented when %_config is modified */ + int pgsz; /* Approximate page size used in %_data */ + int nAutomerge; /* 'automerge' setting */ + int nCrisisMerge; /* Maximum allowed segments per level */ + int nUsermerge; /* 'usermerge' setting */ + int nHashSize; /* Bytes of memory for in-memory hash */ + char *zRank; /* Name of rank function */ + char *zRankArgs; /* Arguments to rank function */ + + /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ + char **pzErrmsg; + +#ifdef SQLITE_DEBUG + int bPrefixIndex; /* True to use prefix-indexes */ +#endif +}; + +/* Current expected value of %_config table 'version' field */ +#define FTS5_CURRENT_VERSION 4 + +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 + +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 + + + +static int sqlite3Fts5ConfigParse( + Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** +); +static void sqlite3Fts5ConfigFree(Fts5Config*); + +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); + +static int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +); + +static void sqlite3Fts5Dequote(char *z); + +/* Load the contents of the %_config table */ +static int sqlite3Fts5ConfigLoad(Fts5Config*, int); + +/* Set the value of a single config attribute */ +static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); + +static int sqlite3Fts5ConfigParseRank(const char*, char**, char**); + +/* +** End of interface to code in fts5_config.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_buffer.c. +*/ + +/* +** Buffer object for the incremental building of string data. +*/ +typedef struct Fts5Buffer Fts5Buffer; +struct Fts5Buffer { + u8 *p; + int n; + int nSpace; +}; + +static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); +static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); +static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); +static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); +static void sqlite3Fts5BufferFree(Fts5Buffer*); +static void sqlite3Fts5BufferZero(Fts5Buffer*); +static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); +static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); + +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); + +#define fts5BufferZero(x) sqlite3Fts5BufferZero(x) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferFree(a) sqlite3Fts5BufferFree(a) +#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) +#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) + +#define fts5BufferGrow(pRc,pBuf,nn) ( \ + (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ + sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ +) + +/* Write and decode big-endian 32-bit integer values */ +static void sqlite3Fts5Put32(u8*, int); +static int sqlite3Fts5Get32(const u8*); + +#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF) + +typedef struct Fts5PoslistReader Fts5PoslistReader; +struct Fts5PoslistReader { + /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ + const u8 *a; /* Position list to iterate through */ + int n; /* Size of buffer at a[] in bytes */ + int i; /* Current offset in a[] */ + + u8 bFlag; /* For client use (any custom purpose) */ + + /* Output variables */ + u8 bEof; /* Set to true at EOF */ + i64 iPos; /* (iCol<<32) + iPos */ +}; +static int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +); +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); + +typedef struct Fts5PoslistWriter Fts5PoslistWriter; +struct Fts5PoslistWriter { + i64 iPrev; +}; +static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); +static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); + +static int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +); + +/* Malloc utility */ +static void *sqlite3Fts5MallocZero(int *pRc, int nByte); +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); + +/* Character set tests (like isspace(), isalpha() etc.) */ +static int sqlite3Fts5IsBareword(char t); + + +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ +typedef struct Fts5Termset Fts5Termset; +static int sqlite3Fts5TermsetNew(Fts5Termset**); +static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); +static void sqlite3Fts5TermsetFree(Fts5Termset*); + +/* +** End of interface to code in fts5_buffer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_index.c. fts5_index.c contains contains code +** to access the data stored in the %_data table. +*/ + +typedef struct Fts5Index Fts5Index; +typedef struct Fts5IndexIter Fts5IndexIter; + +struct Fts5IndexIter { + i64 iRowid; + const u8 *pData; + int nData; + u8 bEof; +}; + +#define sqlite3Fts5IterEof(x) ((x)->bEof) + +/* +** Values used as part of the flags argument passed to IndexQuery(). +*/ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ + +/* The following are used internally by the fts5_index.c module. They are +** defined here only to make it easier to avoid clashes with the flags +** above. */ +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 + +/* +** Create/destroy an Fts5Index object. +*/ +static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); +static int sqlite3Fts5IndexClose(Fts5Index *p); + +/* +** Return a simple checksum value based on the arguments. +*/ +static u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +); + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +static int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +); + +/* +** Open a new iterator to iterate though all rowids that match the +** specified token or token prefix. +*/ +static int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +); + +/* +** The various operations on open token or token prefix iterators opened +** using sqlite3Fts5IndexQuery(). +*/ +static int sqlite3Fts5IterNext(Fts5IndexIter*); +static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); + +/* +** Close an iterator opened by sqlite3Fts5IndexQuery(). +*/ +static void sqlite3Fts5IterClose(Fts5IndexIter*); + +/* +** This interface is used by the fts5vocab module. +*/ +static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); +static int sqlite3Fts5IterNextScan(Fts5IndexIter*); + + +/* +** Insert or remove data to or from the index. Each time a document is +** added to or removed from the index, this function is called one or more +** times. +** +** For an insert, it must be called once for each token in the new document. +** If the operation is a delete, it must be called (at least) once for each +** unique token in the document with an iCol value less than zero. The iPos +** argument is ignored for a delete. +*/ +static int sqlite3Fts5IndexWrite( + Fts5Index *p, /* Index to write to */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to +** document iDocid. +*/ +static int sqlite3Fts5IndexBeginWrite( + Fts5Index *p, /* Index to write to */ + int bDelete, /* True if current operation is a delete */ + i64 iDocid /* Docid to add or remove data from */ +); + +/* +** Flush any data stored in the in-memory hash tables to the database. +** If the bCommit flag is true, also close any open blob handles. +*/ +static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit); + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +static int sqlite3Fts5IndexRollback(Fts5Index *p); + +/* +** Get or set the "averages" values. +*/ +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); + +/* +** Functions called by the storage module as part of integrity-check. +*/ +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); + +/* +** Called during virtual module initialization to register UDF +** fts5_decode() with SQLite +*/ +static int sqlite3Fts5IndexInit(sqlite3*); + +static int sqlite3Fts5IndexSetCookie(Fts5Index*, int); + +/* +** Return the total number of entries read from the %_data table by +** this connection since it was created. +*/ +static int sqlite3Fts5IndexReads(Fts5Index *p); + +static int sqlite3Fts5IndexReinit(Fts5Index *p); +static int sqlite3Fts5IndexOptimize(Fts5Index *p); +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); +static int sqlite3Fts5IndexReset(Fts5Index *p); + +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); + +/* +** End of interface to code in fts5_index.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_varint.c. +*/ +static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); +static int sqlite3Fts5GetVarintLen(u32 iVal); +static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); + +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) +#define fts5GetVarint sqlite3Fts5GetVarint + +#define fts5FastGetVarint32(a, iOff, nVal) { \ + nVal = (a)[iOff++]; \ + if( nVal & 0x80 ){ \ + iOff--; \ + iOff += fts5GetVarint32(&(a)[iOff], nVal); \ + } \ +} + + +/* +** End of interface to code in fts5_varint.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5.c. +*/ + +static int sqlite3Fts5GetTokenizer( + Fts5Global*, + const char **azArg, + int nArg, + Fts5Tokenizer**, + fts5_tokenizer**, + char **pzErr +); + +static Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **); + +/* +** End of interface to code in fts5.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_hash.c. +*/ +typedef struct Fts5Hash Fts5Hash; + +/* +** Create a hash table, free a hash table. +*/ +static int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); +static void sqlite3Fts5HashFree(Fts5Hash*); + +static int sqlite3Fts5HashWrite( + Fts5Hash*, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Empty (but do not delete) a hash table. +*/ +static void sqlite3Fts5HashClear(Fts5Hash*); + +static int sqlite3Fts5HashQuery( + Fts5Hash*, /* Hash table to query */ + const char *pTerm, int nTerm, /* Query term */ + const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +); + +static int sqlite3Fts5HashScanInit( + Fts5Hash*, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +); +static void sqlite3Fts5HashScanNext(Fts5Hash*); +static int sqlite3Fts5HashScanEof(Fts5Hash*); +static void sqlite3Fts5HashScanEntry(Fts5Hash *, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +); + + +/* +** End of interface to code in fts5_hash.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_storage.c. fts5_storage.c contains contains +** code to access the data stored in the %_content and %_docsize tables. +*/ + +#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */ +#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */ +#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */ + +typedef struct Fts5Storage Fts5Storage; + +static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); +static int sqlite3Fts5StorageClose(Fts5Storage *p); +static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); + +static int sqlite3Fts5DropAll(Fts5Config*); +static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); + +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); +static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); + +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p); + +static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); +static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); + +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); + +static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); +static int sqlite3Fts5StorageRollback(Fts5Storage *p); + +static int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, const char*, sqlite3_value*, int +); + +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); +static int sqlite3Fts5StorageRebuild(Fts5Storage *p); +static int sqlite3Fts5StorageOptimize(Fts5Storage *p); +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); +static int sqlite3Fts5StorageReset(Fts5Storage *p); + +/* +** End of interface to code in fts5_storage.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5_expr.c. +*/ +typedef struct Fts5Expr Fts5Expr; +typedef struct Fts5ExprNode Fts5ExprNode; +typedef struct Fts5Parse Fts5Parse; +typedef struct Fts5Token Fts5Token; +typedef struct Fts5ExprPhrase Fts5ExprPhrase; +typedef struct Fts5ExprNearset Fts5ExprNearset; + +struct Fts5Token { + const char *p; /* Token text (not NULL terminated) */ + int n; /* Size of buffer p in bytes */ +}; + +/* Parse a MATCH expression. */ +static int sqlite3Fts5ExprNew( + Fts5Config *pConfig, + const char *zExpr, + Fts5Expr **ppNew, + char **pzErr +); + +/* +** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); +** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr); +** rc = sqlite3Fts5ExprNext(pExpr) +** ){ +** // The document with rowid iRowid matches the expression! +** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); +** } +*/ +static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); +static int sqlite3Fts5ExprEof(Fts5Expr*); +static i64 sqlite3Fts5ExprRowid(Fts5Expr*); + +static void sqlite3Fts5ExprFree(Fts5Expr*); + +/* Called during startup to register a UDF with SQLite */ +static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); + +static int sqlite3Fts5ExprPhraseCount(Fts5Expr*); +static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); +static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); + +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; +static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); +static int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int +); +static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); +static void sqlite3Fts5ExprClearEof(Fts5Expr*); + +static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); + +static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); + +/******************************************* +** The fts5_expr.c API above this point is used by the other hand-written +** C code in this module. The interfaces below this point are called by +** the parser code in fts5parse.y. */ + +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); + +static Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, + int eType, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight, + Fts5ExprNearset *pNear +); + +static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight +); + +static Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, + Fts5ExprPhrase *pPhrase, + Fts5Token *pToken, + int bPrefix +); + +static Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse*, + Fts5ExprNearset*, + Fts5ExprPhrase* +); + +static Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse*, + Fts5Colset*, + Fts5Token * +); + +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); + +static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); +static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); + +/* +** End of interface to code in fts5_expr.c. +**************************************************************************/ + + + +/************************************************************************** +** Interface to code in fts5_aux.c. +*/ + +static int sqlite3Fts5AuxInit(fts5_api*); +/* +** End of interface to code in fts5_aux.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_tokenizer.c. +*/ + +static int sqlite3Fts5TokenizerInit(fts5_api*); +/* +** End of interface to code in fts5_tokenizer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_vocab.c. +*/ + +static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); + +/* +** End of interface to code in fts5_vocab.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to automatically generated code in fts5_unicode2.c. +*/ +static int sqlite3Fts5UnicodeIsalnum(int c); +static int sqlite3Fts5UnicodeIsdiacritic(int c); +static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); +/* +** End of interface to code in fts5_unicode2.c. +**************************************************************************/ + +#endif + +#define FTS5_OR 1 +#define FTS5_AND 2 +#define FTS5_NOT 3 +#define FTS5_TERM 4 +#define FTS5_COLON 5 +#define FTS5_LP 6 +#define FTS5_RP 7 +#define FTS5_LCP 8 +#define FTS5_RCP 9 +#define FTS5_STRING 10 +#define FTS5_COMMA 11 +#define FTS5_PLUS 12 +#define FTS5_STAR 13 + +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +/* #include <stdio.h> */ +/************ Begin %include sections from the grammar ************************/ + +/* #include "fts5Int.h" */ +/* #include "fts5parse.h" */ + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define fts5YYNOERRORRECOVERY 1 + +/* +** Make fts5yytestcase() the same as testcase() +*/ +#define fts5yytestcase(X) testcase(X) + +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define fts5YYPARSEFREENOTNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define fts5YYMALLOCARGTYPE u64 + +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** fts5YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** fts5YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** fts5YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** fts5YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union +** for terminal symbols is called "fts5yy0". +** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser +** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser +** fts5YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** fts5YYNSTATE the combined number of states. +** fts5YYNRULE the number of rules in the grammar +** fts5YY_MAX_SHIFT Maximum value for shift actions +** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** fts5YY_MIN_REDUCE Maximum value for reduce actions +** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error +** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept +** fts5YY_NO_ACTION The fts5yy_action[] code for no-op +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define fts5YYCODETYPE unsigned char +#define fts5YYNOCODE 27 +#define fts5YYACTIONTYPE unsigned char +#define sqlite3Fts5ParserFTS5TOKENTYPE Fts5Token +typedef union { + int fts5yyinit; + sqlite3Fts5ParserFTS5TOKENTYPE fts5yy0; + Fts5Colset* fts5yy3; + Fts5ExprPhrase* fts5yy11; + Fts5ExprNode* fts5yy18; + int fts5yy20; + Fts5ExprNearset* fts5yy26; +} fts5YYMINORTYPE; +#ifndef fts5YYSTACKDEPTH +#define fts5YYSTACKDEPTH 100 +#endif +#define sqlite3Fts5ParserARG_SDECL Fts5Parse *pParse; +#define sqlite3Fts5ParserARG_PDECL ,Fts5Parse *pParse +#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse = fts5yypParser->pParse +#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse = pParse +#define fts5YYNSTATE 26 +#define fts5YYNRULE 24 +#define fts5YY_MAX_SHIFT 25 +#define fts5YY_MIN_SHIFTREDUCE 40 +#define fts5YY_MAX_SHIFTREDUCE 63 +#define fts5YY_MIN_REDUCE 64 +#define fts5YY_MAX_REDUCE 87 +#define fts5YY_ERROR_ACTION 88 +#define fts5YY_ACCEPT_ACTION 89 +#define fts5YY_NO_ACTION 90 +/************* End control #defines *******************************************/ + +/* Define the fts5yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define fts5yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the fts5yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef fts5yytestcase +# define fts5yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE. +** +** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE +** and fts5YY_MAX_REDUCE + +** N == fts5YY_ERROR_ACTION A syntax error has occurred. +** +** N == fts5YY_ACCEPT_ACTION The parser accepts its input. +** +** N == fts5YY_NO_ACTION No such action. Denotes unused +** slots in the fts5yy_action[] table. +** +** The action table is constructed as a single large table named fts5yy_action[]. +** Given state S and lookahead X, the action is computed as +** +** fts5yy_action[ fts5yy_shift_ofst[S] + X ] +** +** If the index value fts5yy_shift_ofst[S]+X is out of range or if the value +** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X or if fts5yy_shift_ofst[S] +** is equal to fts5YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that fts5yy_default[S] should be used instead. +** +** The formula above is for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of +** the fts5yy_shift_ofst[] array and fts5YY_REDUCE_USE_DFLT is used in place of +** fts5YY_SHIFT_USE_DFLT. +** +** The following are the tables generated in this section: +** +** fts5yy_action[] A single table containing all actions. +** fts5yy_lookahead[] A table containing the lookahead for each entry in +** fts5yy_action. Used to detect hash collisions. +** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for +** shifting terminals. +** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for +** shifting non-terminals after a reduce. +** fts5yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define fts5YY_ACTTAB_COUNT (78) +static const fts5YYACTIONTYPE fts5yy_action[] = { + /* 0 */ 89, 15, 46, 5, 48, 24, 12, 19, 23, 14, + /* 10 */ 46, 5, 48, 24, 20, 21, 23, 43, 46, 5, + /* 20 */ 48, 24, 6, 18, 23, 17, 46, 5, 48, 24, + /* 30 */ 75, 7, 23, 25, 46, 5, 48, 24, 62, 47, + /* 40 */ 23, 48, 24, 7, 11, 23, 9, 3, 4, 2, + /* 50 */ 62, 50, 52, 44, 64, 3, 4, 2, 49, 4, + /* 60 */ 2, 1, 23, 11, 16, 9, 12, 2, 10, 61, + /* 70 */ 53, 59, 62, 60, 22, 13, 55, 8, +}; +static const fts5YYCODETYPE fts5yy_lookahead[] = { + /* 0 */ 15, 16, 17, 18, 19, 20, 10, 11, 23, 16, + /* 10 */ 17, 18, 19, 20, 23, 24, 23, 16, 17, 18, + /* 20 */ 19, 20, 22, 23, 23, 16, 17, 18, 19, 20, + /* 30 */ 5, 6, 23, 16, 17, 18, 19, 20, 13, 17, + /* 40 */ 23, 19, 20, 6, 8, 23, 10, 1, 2, 3, + /* 50 */ 13, 9, 10, 7, 0, 1, 2, 3, 19, 2, + /* 60 */ 3, 6, 23, 8, 21, 10, 10, 3, 10, 25, + /* 70 */ 10, 10, 13, 25, 12, 10, 7, 5, +}; +#define fts5YY_SHIFT_USE_DFLT (-5) +#define fts5YY_SHIFT_COUNT (25) +#define fts5YY_SHIFT_MIN (-4) +#define fts5YY_SHIFT_MAX (72) +static const signed char fts5yy_shift_ofst[] = { + /* 0 */ 55, 55, 55, 55, 55, 36, -4, 56, 58, 25, + /* 10 */ 37, 60, 59, 59, 46, 54, 42, 57, 62, 61, + /* 20 */ 62, 69, 65, 62, 72, 64, +}; +#define fts5YY_REDUCE_USE_DFLT (-16) +#define fts5YY_REDUCE_COUNT (13) +#define fts5YY_REDUCE_MIN (-15) +#define fts5YY_REDUCE_MAX (48) +static const signed char fts5yy_reduce_ofst[] = { + /* 0 */ -15, -7, 1, 9, 17, 22, -9, 0, 39, 44, + /* 10 */ 44, 43, 44, 48, +}; +static const fts5YYACTIONTYPE fts5yy_default[] = { + /* 0 */ 88, 88, 88, 88, 88, 69, 82, 88, 88, 87, + /* 10 */ 87, 88, 87, 87, 88, 88, 88, 66, 80, 88, + /* 20 */ 81, 88, 88, 78, 88, 65, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef fts5YYFALLBACK +static const fts5YYCODETYPE fts5yyFallback[] = { +}; +#endif /* fts5YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct fts5yyStackEntry { + fts5YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + fts5YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + fts5YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct fts5yyStackEntry fts5yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct fts5yyParser { + int fts5yyidx; /* Index of top element in stack */ +#ifdef fts5YYTRACKMAXSTACKDEPTH + int fts5yyidxMax; /* Maximum value of fts5yyidx */ +#endif +#ifndef fts5YYNOERRORRECOVERY + int fts5yyerrcnt; /* Shifts left before out of the error */ +#endif + sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ +#if fts5YYSTACKDEPTH<=0 + int fts5yystksz; /* Current side of the stack */ + fts5yyStackEntry *fts5yystack; /* The parser's stack */ +#else + fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ +#endif +}; +typedef struct fts5yyParser fts5yyParser; + +#ifndef NDEBUG +/* #include <stdio.h> */ +static FILE *fts5yyTraceFILE = 0; +static char *fts5yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +** <ul> +** <li> A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +** <li> A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +** </ul> +** +** Outputs: +** None. +*/ +static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + fts5yyTraceFILE = TraceFILE; + fts5yyTracePrompt = zTracePrompt; + if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0; + else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const fts5yyTokenName[] = { + "$", "OR", "AND", "NOT", + "TERM", "COLON", "LP", "RP", + "LCP", "RCP", "STRING", "COMMA", + "PLUS", "STAR", "error", "input", + "expr", "cnearset", "exprlist", "nearset", + "colset", "colsetlist", "nearphrases", "phrase", + "neardist_opt", "star_opt", +}; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const fts5yyRuleName[] = { + /* 0 */ "input ::= expr", + /* 1 */ "expr ::= expr AND expr", + /* 2 */ "expr ::= expr OR expr", + /* 3 */ "expr ::= expr NOT expr", + /* 4 */ "expr ::= LP expr RP", + /* 5 */ "expr ::= exprlist", + /* 6 */ "exprlist ::= cnearset", + /* 7 */ "exprlist ::= exprlist cnearset", + /* 8 */ "cnearset ::= nearset", + /* 9 */ "cnearset ::= colset COLON nearset", + /* 10 */ "colset ::= LCP colsetlist RCP", + /* 11 */ "colset ::= STRING", + /* 12 */ "colsetlist ::= colsetlist STRING", + /* 13 */ "colsetlist ::= STRING", + /* 14 */ "nearset ::= phrase", + /* 15 */ "nearset ::= STRING LP nearphrases neardist_opt RP", + /* 16 */ "nearphrases ::= phrase", + /* 17 */ "nearphrases ::= nearphrases phrase", + /* 18 */ "neardist_opt ::=", + /* 19 */ "neardist_opt ::= COMMA STRING", + /* 20 */ "phrase ::= phrase PLUS STRING star_opt", + /* 21 */ "phrase ::= STRING star_opt", + /* 22 */ "star_opt ::= STAR", + /* 23 */ "star_opt ::=", +}; +#endif /* NDEBUG */ + + +#if fts5YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. +*/ +static void fts5yyGrowStack(fts5yyParser *p){ + int newSize; + fts5yyStackEntry *pNew; + + newSize = p->fts5yystksz*2 + 100; + pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew ){ + p->fts5yystack = pNew; + p->fts5yystksz = newSize; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows to %d entries!\n", + fts5yyTracePrompt, p->fts5yystksz); + } +#endif + } +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef fts5YYMALLOCARGTYPE +# define fts5YYMALLOCARGTYPE size_t +#endif + +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Fts5Parser and sqlite3Fts5ParserFree. +*/ +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE)){ + fts5yyParser *pParser; + pParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) ); + if( pParser ){ + pParser->fts5yyidx = -1; +#ifdef fts5YYTRACKMAXSTACKDEPTH + pParser->fts5yyidxMax = 0; +#endif +#if fts5YYSTACKDEPTH<=0 + pParser->fts5yystack = NULL; + pParser->fts5yystksz = 0; + fts5yyGrowStack(pParser); +#endif + } + return pParser; +} + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void fts5yy_destructor( + fts5yyParser *fts5yypParser, /* The parser */ + fts5YYCODETYPE fts5yymajor, /* Type code for object to destroy */ + fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */ +){ + sqlite3Fts5ParserARG_FETCH; + switch( fts5yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + case 15: /* input */ +{ + (void)pParse; +} + break; + case 16: /* expr */ + case 17: /* cnearset */ + case 18: /* exprlist */ +{ + sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy18)); +} + break; + case 19: /* nearset */ + case 22: /* nearphrases */ +{ + sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy26)); +} + break; + case 20: /* colset */ + case 21: /* colsetlist */ +{ + sqlite3_free((fts5yypminor->fts5yy3)); +} + break; + case 23: /* phrase */ +{ + sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy11)); +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ + fts5yyStackEntry *fts5yytos; + assert( pParser->fts5yyidx>=0 ); + fts5yytos = &pParser->fts5yystack[pParser->fts5yyidx--]; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); +} + +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +static void sqlite3Fts5ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ + fts5yyParser *pParser = (fts5yyParser*)p; +#ifndef fts5YYPARSEFREENEVERNULL + if( pParser==0 ) return; +#endif + while( pParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(pParser); +#if fts5YYSTACKDEPTH<=0 + free(pParser->fts5yystack); +#endif + (*freeProc)((void*)pParser); +} + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef fts5YYTRACKMAXSTACKDEPTH +static int sqlite3Fts5ParserStackPeak(void *p){ + fts5yyParser *pParser = (fts5yyParser*)p; + return pParser->fts5yyidxMax; +} +#endif + +/* +** Find the appropriate action for a parser given the terminal +** look-ahead token iLookAhead. +*/ +static unsigned int fts5yy_find_shift_action( + fts5yyParser *pParser, /* The parser */ + fts5YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; + int stateno = pParser->fts5yystack[pParser->fts5yyidx].stateno; + + if( stateno>=fts5YY_MIN_REDUCE ) return stateno; + assert( stateno <= fts5YY_SHIFT_COUNT ); + do{ + i = fts5yy_shift_ofst[stateno]; + if( i==fts5YY_SHIFT_USE_DFLT ) return fts5yy_default[stateno]; + assert( iLookAhead!=fts5YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ +#ifdef fts5YYFALLBACK + fts5YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead<sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0]) + && (iFallback = fts5yyFallback[iLookAhead])!=0 ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]); + } +#endif + assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef fts5YYWILDCARD + { + int j = i - iLookAhead + fts5YYWILDCARD; + if( +#if fts5YY_SHIFT_MIN+fts5YYWILDCARD<0 + j>=0 && +#endif +#if fts5YY_SHIFT_MAX+fts5YYWILDCARD>=fts5YY_ACTTAB_COUNT + j<fts5YY_ACTTAB_COUNT && +#endif + fts5yy_lookahead[j]==fts5YYWILDCARD + ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], + fts5yyTokenName[fts5YYWILDCARD]); + } +#endif /* NDEBUG */ + return fts5yy_action[j]; + } + } +#endif /* fts5YYWILDCARD */ + } + return fts5yy_default[stateno]; + }else{ + return fts5yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static int fts5yy_find_reduce_action( + int stateno, /* Current state number */ + fts5YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef fts5YYERRORSYMBOL + if( stateno>fts5YY_REDUCE_COUNT ){ + return fts5yy_default[stateno]; + } +#else + assert( stateno<=fts5YY_REDUCE_COUNT ); +#endif + i = fts5yy_reduce_ofst[stateno]; + assert( i!=fts5YY_REDUCE_USE_DFLT ); + assert( iLookAhead!=fts5YYNOCODE ); + i += iLookAhead; +#ifdef fts5YYERRORSYMBOL + if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ + return fts5yy_default[stateno]; + } +#else + assert( i>=0 && i<fts5YY_ACTTAB_COUNT ); + assert( fts5yy_lookahead[i]==iLookAhead ); +#endif + return fts5yy_action[i]; +} + +/* +** The following routine is called if the stack overflows. +*/ +static void fts5yyStackOverflow(fts5yyParser *fts5yypParser){ + sqlite3Fts5ParserARG_FETCH; + fts5yypParser->fts5yyidx--; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt); + } +#endif + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ + + sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); +/******** End %stack_overflow code ********************************************/ + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState){ + if( fts5yyTraceFILE ){ + if( fts5yyNewState<fts5YYNSTATE ){ + fprintf(fts5yyTraceFILE,"%sShift '%s', go to state %d\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major], + fts5yyNewState); + }else{ + fprintf(fts5yyTraceFILE,"%sShift '%s'\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major]); + } + } +} +#else +# define fts5yyTraceShift(X,Y) +#endif + +/* +** Perform a shift action. +*/ +static void fts5yy_shift( + fts5yyParser *fts5yypParser, /* The parser to be shifted */ + int fts5yyNewState, /* The new state to shift in */ + int fts5yyMajor, /* The major token to shift in */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyMinor /* The minor token to shift in */ +){ + fts5yyStackEntry *fts5yytos; + fts5yypParser->fts5yyidx++; +#ifdef fts5YYTRACKMAXSTACKDEPTH + if( fts5yypParser->fts5yyidx>fts5yypParser->fts5yyidxMax ){ + fts5yypParser->fts5yyidxMax = fts5yypParser->fts5yyidx; + } +#endif +#if fts5YYSTACKDEPTH>0 + if( fts5yypParser->fts5yyidx>=fts5YYSTACKDEPTH ){ + fts5yyStackOverflow(fts5yypParser); + return; + } +#else + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){ + fts5yyGrowStack(fts5yypParser); + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){ + fts5yyStackOverflow(fts5yypParser); + return; + } + } +#endif + fts5yytos = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx]; + fts5yytos->stateno = (fts5YYACTIONTYPE)fts5yyNewState; + fts5yytos->major = (fts5YYCODETYPE)fts5yyMajor; + fts5yytos->minor.fts5yy0 = fts5yyMinor; + fts5yyTraceShift(fts5yypParser, fts5yyNewState); +} + +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + fts5YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} fts5yyRuleInfo[] = { + { 15, 1 }, + { 16, 3 }, + { 16, 3 }, + { 16, 3 }, + { 16, 3 }, + { 16, 1 }, + { 18, 1 }, + { 18, 2 }, + { 17, 1 }, + { 17, 3 }, + { 20, 3 }, + { 20, 1 }, + { 21, 2 }, + { 21, 1 }, + { 19, 1 }, + { 19, 5 }, + { 22, 1 }, + { 22, 2 }, + { 24, 0 }, + { 24, 2 }, + { 23, 4 }, + { 23, 2 }, + { 25, 1 }, + { 25, 0 }, +}; + +static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +*/ +static void fts5yy_reduce( + fts5yyParser *fts5yypParser, /* The parser */ + unsigned int fts5yyruleno /* Number of the rule by which to reduce */ +){ + int fts5yygoto; /* The next state */ + int fts5yyact; /* The next action */ + fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */ + int fts5yysize; /* Amount to pop the stack */ + sqlite3Fts5ParserARG_FETCH; + fts5yymsp = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx]; +#ifndef NDEBUG + if( fts5yyTraceFILE && fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ){ + fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs; + fprintf(fts5yyTraceFILE, "%sReduce [%s], go to state %d.\n", fts5yyTracePrompt, + fts5yyRuleName[fts5yyruleno], fts5yymsp[-fts5yysize].stateno); + } +#endif /* NDEBUG */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( fts5yyRuleInfo[fts5yyruleno].nrhs==0 ){ +#ifdef fts5YYTRACKMAXSTACKDEPTH + if( fts5yypParser->fts5yyidx>fts5yypParser->fts5yyidxMax ){ + fts5yypParser->fts5yyidxMax = fts5yypParser->fts5yyidx; + } +#endif +#if fts5YYSTACKDEPTH>0 + if( fts5yypParser->fts5yyidx>=fts5YYSTACKDEPTH-1 ){ + fts5yyStackOverflow(fts5yypParser); + return; + } +#else + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz-1 ){ + fts5yyGrowStack(fts5yypParser); + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz-1 ){ + fts5yyStackOverflow(fts5yypParser); + return; + } + } +#endif + } + + switch( fts5yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line <lineno> <grammarfile> + ** { ... } // User supplied code + ** #line <lineno> <thisfile> + ** break; + */ +/********** Begin reduce actions **********************************************/ + fts5YYMINORTYPE fts5yylhsminor; + case 0: /* input ::= expr */ +{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy18); } + break; + case 1: /* expr ::= expr AND expr */ +{ + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); +} + fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 2: /* expr ::= expr OR expr */ +{ + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); +} + fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 3: /* expr ::= expr NOT expr */ +{ + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); +} + fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 4: /* expr ::= LP expr RP */ +{fts5yymsp[-2].minor.fts5yy18 = fts5yymsp[-1].minor.fts5yy18;} + break; + case 5: /* expr ::= exprlist */ + case 6: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==6); +{fts5yylhsminor.fts5yy18 = fts5yymsp[0].minor.fts5yy18;} + fts5yymsp[0].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 7: /* exprlist ::= exprlist cnearset */ +{ + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18); +} + fts5yymsp[-1].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 8: /* cnearset ::= nearset */ +{ + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26); +} + fts5yymsp[0].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 9: /* cnearset ::= colset COLON nearset */ +{ + sqlite3Fts5ParseSetColset(pParse, fts5yymsp[0].minor.fts5yy26, fts5yymsp[-2].minor.fts5yy3); + fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26); +} + fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + break; + case 10: /* colset ::= LCP colsetlist RCP */ +{ fts5yymsp[-2].minor.fts5yy3 = fts5yymsp[-1].minor.fts5yy3; } + break; + case 11: /* colset ::= STRING */ +{ + fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); +} + fts5yymsp[0].minor.fts5yy3 = fts5yylhsminor.fts5yy3; + break; + case 12: /* colsetlist ::= colsetlist STRING */ +{ + fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy3, &fts5yymsp[0].minor.fts5yy0); } + fts5yymsp[-1].minor.fts5yy3 = fts5yylhsminor.fts5yy3; + break; + case 13: /* colsetlist ::= STRING */ +{ + fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); +} + fts5yymsp[0].minor.fts5yy3 = fts5yylhsminor.fts5yy3; + break; + case 14: /* nearset ::= phrase */ +{ fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11); } + fts5yymsp[0].minor.fts5yy26 = fts5yylhsminor.fts5yy26; + break; + case 15: /* nearset ::= STRING LP nearphrases neardist_opt RP */ +{ + sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0); + sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy26, &fts5yymsp[-1].minor.fts5yy0); + fts5yylhsminor.fts5yy26 = fts5yymsp[-2].minor.fts5yy26; +} + fts5yymsp[-4].minor.fts5yy26 = fts5yylhsminor.fts5yy26; + break; + case 16: /* nearphrases ::= phrase */ +{ + fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11); +} + fts5yymsp[0].minor.fts5yy26 = fts5yylhsminor.fts5yy26; + break; + case 17: /* nearphrases ::= nearphrases phrase */ +{ + fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy26, fts5yymsp[0].minor.fts5yy11); +} + fts5yymsp[-1].minor.fts5yy26 = fts5yylhsminor.fts5yy26; + break; + case 18: /* neardist_opt ::= */ +{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; } + break; + case 19: /* neardist_opt ::= COMMA STRING */ +{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; } + break; + case 20: /* phrase ::= phrase PLUS STRING star_opt */ +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy11, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20); +} + fts5yymsp[-3].minor.fts5yy11 = fts5yylhsminor.fts5yy11; + break; + case 21: /* phrase ::= STRING star_opt */ +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20); +} + fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11; + break; + case 22: /* star_opt ::= STAR */ +{ fts5yymsp[0].minor.fts5yy20 = 1; } + break; + case 23: /* star_opt ::= */ +{ fts5yymsp[1].minor.fts5yy20 = 0; } + break; + default: + break; +/********** End reduce actions ************************************************/ + }; + assert( fts5yyruleno<sizeof(fts5yyRuleInfo)/sizeof(fts5yyRuleInfo[0]) ); + fts5yygoto = fts5yyRuleInfo[fts5yyruleno].lhs; + fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs; + fts5yyact = fts5yy_find_reduce_action(fts5yymsp[-fts5yysize].stateno,(fts5YYCODETYPE)fts5yygoto); + if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ + if( fts5yyact>fts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; + fts5yypParser->fts5yyidx -= fts5yysize - 1; + fts5yymsp -= fts5yysize-1; + fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact; + fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto; + fts5yyTraceShift(fts5yypParser, fts5yyact); + }else{ + assert( fts5yyact == fts5YY_ACCEPT_ACTION ); + fts5yypParser->fts5yyidx -= fts5yysize; + fts5yy_accept(fts5yypParser); + } +} + +/* +** The following code executes when the parse fails +*/ +#ifndef fts5YYNOERRORRECOVERY +static void fts5yy_parse_failed( + fts5yyParser *fts5yypParser /* The parser */ +){ + sqlite3Fts5ParserARG_FETCH; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt); + } +#endif + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} +#endif /* fts5YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void fts5yy_syntax_error( + fts5yyParser *fts5yypParser, /* The parser */ + int fts5yymajor, /* The major type of the error token */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The minor type of the error token */ +){ + sqlite3Fts5ParserARG_FETCH; +#define FTS5TOKEN fts5yyminor +/************ Begin %syntax_error code ****************************************/ + + UNUSED_PARAM(fts5yymajor); /* Silence a compiler warning */ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKEN.n,FTS5TOKEN.p + ); +/************ End %syntax_error code ******************************************/ + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* +** The following is executed when the parser accepts +*/ +static void fts5yy_accept( + fts5yyParser *fts5yypParser /* The parser */ +){ + sqlite3Fts5ParserARG_FETCH; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt); + } +#endif + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3Fts5ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +** <ul> +** <li> A pointer to the parser (an opaque structure.) +** <li> The major token number. +** <li> The minor token number. +** <li> An option argument of a grammar-specified type. +** </ul> +** +** Outputs: +** None. +*/ +static void sqlite3Fts5Parser( + void *fts5yyp, /* The parser */ + int fts5yymajor, /* The major token code number */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The value for the token */ + sqlite3Fts5ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + fts5YYMINORTYPE fts5yyminorunion; + unsigned int fts5yyact; /* The parser action. */ +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) + int fts5yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef fts5YYERRORSYMBOL + int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */ +#endif + fts5yyParser *fts5yypParser; /* The parser */ + + /* (re)initialize the parser, if necessary */ + fts5yypParser = (fts5yyParser*)fts5yyp; + if( fts5yypParser->fts5yyidx<0 ){ +#if fts5YYSTACKDEPTH<=0 + if( fts5yypParser->fts5yystksz <=0 ){ + fts5yyStackOverflow(fts5yypParser); + return; + } +#endif + fts5yypParser->fts5yyidx = 0; +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + fts5yypParser->fts5yystack[0].stateno = 0; + fts5yypParser->fts5yystack[0].major = 0; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sInitialize. Empty stack. State 0\n", + fts5yyTracePrompt); + } +#endif + } +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) + fts5yyendofinput = (fts5yymajor==0); +#endif + sqlite3Fts5ParserARG_STORE; + +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sInput '%s'\n",fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); + } +#endif + + do{ + fts5yyact = fts5yy_find_shift_action(fts5yypParser,(fts5YYCODETYPE)fts5yymajor); + if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ + if( fts5yyact > fts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; + fts5yy_shift(fts5yypParser,fts5yyact,fts5yymajor,fts5yyminor); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt--; +#endif + fts5yymajor = fts5YYNOCODE; + }else if( fts5yyact <= fts5YY_MAX_REDUCE ){ + fts5yy_reduce(fts5yypParser,fts5yyact-fts5YY_MIN_REDUCE); + }else{ + assert( fts5yyact == fts5YY_ERROR_ACTION ); + fts5yyminorunion.fts5yy0 = fts5yyminor; +#ifdef fts5YYERRORSYMBOL + int fts5yymx; +#endif +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt); + } +#endif +#ifdef fts5YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( fts5yypParser->fts5yyerrcnt<0 ){ + fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor); + } + fts5yymx = fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major; + if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); + } +#endif + fts5yy_destructor(fts5yypParser, (fts5YYCODETYPE)fts5yymajor, &fts5yyminorunion); + fts5yymajor = fts5YYNOCODE; + }else{ + while( + fts5yypParser->fts5yyidx >= 0 && + fts5yymx != fts5YYERRORSYMBOL && + (fts5yyact = fts5yy_find_reduce_action( + fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].stateno, + fts5YYERRORSYMBOL)) >= fts5YY_MIN_REDUCE + ){ + fts5yy_pop_parser_stack(fts5yypParser); + } + if( fts5yypParser->fts5yyidx < 0 || fts5yymajor==0 ){ + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + fts5yy_parse_failed(fts5yypParser); + fts5yymajor = fts5YYNOCODE; + }else if( fts5yymx!=fts5YYERRORSYMBOL ){ + fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor); + } + } + fts5yypParser->fts5yyerrcnt = 3; + fts5yyerrorhit = 1; +#elif defined(fts5YYNOERRORRECOVERY) + /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + fts5yymajor = fts5YYNOCODE; + +#else /* fts5YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( fts5yypParser->fts5yyerrcnt<=0 ){ + fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); + } + fts5yypParser->fts5yyerrcnt = 3; + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + if( fts5yyendofinput ){ + fts5yy_parse_failed(fts5yypParser); + } + fts5yymajor = fts5YYNOCODE; +#endif + } + }while( fts5yymajor!=fts5YYNOCODE && fts5yypParser->fts5yyidx>=0 ); +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + int i; + fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt); + for(i=1; i<=fts5yypParser->fts5yyidx; i++) + fprintf(fts5yyTraceFILE,"%c%s", i==1 ? '[' : ' ', + fts5yyTokenName[fts5yypParser->fts5yystack[i].major]); + fprintf(fts5yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + +/* #include "fts5Int.h" */ +#include <math.h> /* amalgamator: keep */ + +/* +** Object used to iterate through all "coalesced phrase instances" in +** a single column of the current row. If the phrase instances in the +** column being considered do not overlap, this object simply iterates +** through them. Or, if they do overlap (share one or more tokens in +** common), each set of overlapping instances is treated as a single +** match. See documentation for the highlight() auxiliary function for +** details. +** +** Usage is: +** +** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter); +** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter); +** rc = fts5CInstIterNext(&iter) +** ){ +** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd); +** } +** +*/ +typedef struct CInstIter CInstIter; +struct CInstIter { + const Fts5ExtensionApi *pApi; /* API offered by current FTS version */ + Fts5Context *pFts; /* First arg to pass to pApi functions */ + int iCol; /* Column to search */ + int iInst; /* Next phrase instance index */ + int nInst; /* Total number of phrase instances */ + + /* Output variables */ + int iStart; /* First token in coalesced phrase instance */ + int iEnd; /* Last token in coalesced phrase instance */ +}; + +/* +** Advance the iterator to the next coalesced phrase instance. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +static int fts5CInstIterNext(CInstIter *pIter){ + int rc = SQLITE_OK; + pIter->iStart = -1; + pIter->iEnd = -1; + + while( rc==SQLITE_OK && pIter->iInst<pIter->nInst ){ + int ip; int ic; int io; + rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + if( ic==pIter->iCol ){ + int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip); + if( pIter->iStart<0 ){ + pIter->iStart = io; + pIter->iEnd = iEnd; + }else if( io<=pIter->iEnd ){ + if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd; + }else{ + break; + } + } + pIter->iInst++; + } + } + + return rc; +} + +/* +** Initialize the iterator object indicated by the final parameter to +** iterate through coalesced phrase instances in column iCol. +*/ +static int fts5CInstIterInit( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + int iCol, + CInstIter *pIter +){ + int rc; + + memset(pIter, 0, sizeof(CInstIter)); + pIter->pApi = pApi; + pIter->pFts = pFts; + pIter->iCol = iCol; + rc = pApi->xInstCount(pFts, &pIter->nInst); + + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(pIter); + } + + return rc; +} + + + +/************************************************************************* +** Start of highlight() implementation. +*/ +typedef struct HighlightContext HighlightContext; +struct HighlightContext { + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iRangeStart; /* First token to include */ + int iRangeEnd; /* If non-zero, last token to include */ + const char *zOpen; /* Opening highlight */ + const char *zClose; /* Closing highlight */ + const char *zIn; /* Input text */ + int nIn; /* Size of input text in bytes */ + int iOff; /* Current offset within zIn[] */ + char *zOut; /* Output value */ +}; + +/* +** Append text to the HighlightContext output string - p->zOut. Argument +** z points to a buffer containing n bytes of text to append. If n is +** negative, everything up until the first '\0' is appended to the output. +** +** If *pRc is set to any value other than SQLITE_OK when this function is +** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, +** *pRc is set to an error code before returning. +*/ +static void fts5HighlightAppend( + int *pRc, + HighlightContext *p, + const char *z, int n +){ + if( *pRc==SQLITE_OK ){ + if( n<0 ) n = (int)strlen(z); + p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); + if( p->zOut==0 ) *pRc = SQLITE_NOMEM; + } +} + +/* +** Tokenizer callback used by implementation of highlight() function. +*/ +static int fts5HighlightCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + HighlightContext *p = (HighlightContext*)pContext; + int rc = SQLITE_OK; + int iPos; + + UNUSED_PARAM2(pToken, nToken); + + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; + + if( p->iRangeEnd>0 ){ + if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; + if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; + } + + if( iPos==p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); + fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->iOff = iStartOff; + } + + if( iPos==p->iter.iEnd ){ + if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){ + fts5HighlightAppend(&rc, p, p->zOpen, -1); + } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->iOff = iEndOff; + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(&p->iter); + } + } + + if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + if( iPos<p->iter.iEnd ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + } + } + + return rc; +} + +/* +** Implementation of highlight() function. +*/ +static void fts5HighlightFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc; + int iCol; + + if( nVal!=3 ){ + const char *zErr = "wrong number of arguments to function highlight()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + memset(&ctx, 0, sizeof(HighlightContext)); + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); + + if( ctx.zIn ){ + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); + } + + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + } + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + } + sqlite3_free(ctx.zOut); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} +/* +** End of highlight() implementation. +**************************************************************************/ + +/* +** Implementation of snippet() function. +*/ +static void fts5SnippetFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc = SQLITE_OK; /* Return code */ + int iCol; /* 1st argument to snippet() */ + const char *zEllips; /* 4th argument to snippet() */ + int nToken; /* 5th argument to snippet() */ + int nInst = 0; /* Number of instance matches this row */ + int i; /* Used to iterate through instances */ + int nPhrase; /* Number of phrases in query */ + unsigned char *aSeen; /* Array of "seen instance" flags */ + int iBestCol; /* Column containing best snippet */ + int iBestStart = 0; /* First token of best snippet */ + int iBestLast; /* Last token of best snippet */ + int nBestScore = 0; /* Score of best snippet */ + int nColSize = 0; /* Total size of iBestCol in tokens */ + + if( nVal!=5 ){ + const char *zErr = "wrong number of arguments to function snippet()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + memset(&ctx, 0, sizeof(HighlightContext)); + iCol = sqlite3_value_int(apVal[0]); + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + zEllips = (const char*)sqlite3_value_text(apVal[3]); + nToken = sqlite3_value_int(apVal[4]); + iBestLast = nToken-1; + + iBestCol = (iCol>=0 ? iCol : 0); + nPhrase = pApi->xPhraseCount(pFts); + aSeen = sqlite3_malloc(nPhrase); + if( aSeen==0 ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + rc = pApi->xInstCount(pFts, &nInst); + } + for(i=0; rc==SQLITE_OK && i<nInst; i++){ + int ip, iSnippetCol, iStart; + memset(aSeen, 0, nPhrase); + rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart); + if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ + int nScore = 1000; + int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); + int j; + aSeen[ip] = 1; + + for(j=i+1; rc==SQLITE_OK && j<nInst; j++){ + int ic; int io; int iFinal; + rc = pApi->xInst(pFts, j, &ip, &ic, &io); + iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; + if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){ + nScore += aSeen[ip] ? 1000 : 1; + aSeen[ip] = 1; + if( iFinal>iLast ) iLast = iFinal; + } + } + + if( rc==SQLITE_OK && nScore>nBestScore ){ + iBestCol = iSnippetCol; + iBestStart = iStart; + iBestLast = iLast; + nBestScore = nScore; + } + } + } + + if( rc==SQLITE_OK ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } + if( rc==SQLITE_OK ){ + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); + } + if( ctx.zIn ){ + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); + } + + if( (iBestStart+nToken-1)>iBestLast ){ + iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; + } + if( iBestStart+nToken>nColSize ){ + iBestStart = nColSize - nToken; + } + if( iBestStart<0 ) iBestStart = 0; + + ctx.iRangeStart = iBestStart; + ctx.iRangeEnd = iBestStart + nToken - 1; + + if( iBestStart>0 ){ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + } + if( ctx.iRangeEnd>=(nColSize-1) ){ + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + }else{ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); + } + sqlite3_free(aSeen); +} + +/************************************************************************/ + +/* +** The first time the bm25() function is called for a query, an instance +** of the following structure is allocated and populated. +*/ +typedef struct Fts5Bm25Data Fts5Bm25Data; +struct Fts5Bm25Data { + int nPhrase; /* Number of phrases in query */ + double avgdl; /* Average number of tokens in each row */ + double *aIDF; /* IDF for each phrase */ + double *aFreq; /* Array used to calculate phrase freq. */ +}; + +/* +** Callback used by fts5Bm25GetData() to count the number of rows in the +** table matched by each individual phrase within the query. +*/ +static int fts5CountCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData /* Pointer to sqlite3_int64 variable */ +){ + sqlite3_int64 *pn = (sqlite3_int64*)pUserData; + UNUSED_PARAM2(pApi, pFts); + (*pn)++; + return SQLITE_OK; +} + +/* +** Set *ppData to point to the Fts5Bm25Data object for the current query. +** If the object has not already been allocated, allocate and populate it +** now. +*/ +static int fts5Bm25GetData( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Bm25Data *p; /* Object to return */ + + p = pApi->xGetAuxdata(pFts, 0); + if( p==0 ){ + int nPhrase; /* Number of phrases in query */ + sqlite3_int64 nRow = 0; /* Number of rows in table */ + sqlite3_int64 nToken = 0; /* Number of tokens in table */ + int nByte; /* Bytes of space to allocate */ + int i; + + /* Allocate the Fts5Bm25Data object */ + nPhrase = pApi->xPhraseCount(pFts); + nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double); + p = (Fts5Bm25Data*)sqlite3_malloc(nByte); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(p, 0, nByte); + p->nPhrase = nPhrase; + p->aIDF = (double*)&p[1]; + p->aFreq = &p->aIDF[nPhrase]; + } + + /* Calculate the average document length for this FTS5 table */ + if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow); + if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken); + if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow; + + /* Calculate an IDF for each phrase in the query */ + for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ + sqlite3_int64 nHit = 0; + rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb); + if( rc==SQLITE_OK ){ + /* Calculate the IDF (Inverse Document Frequency) for phrase i. + ** This is done using the standard BM25 formula as found on wikipedia: + ** + ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) ) + ** + ** where "N" is the total number of documents in the set and nHit + ** is the number that contain at least one instance of the phrase + ** under consideration. + ** + ** The problem with this is that if (N < 2*nHit), the IDF is + ** negative. Which is undesirable. So the mimimum allowable IDF is + ** (1e-6) - roughly the same as a term that appears in just over + ** half of set of 5,000,000 documents. */ + double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); + if( idf<=0.0 ) idf = 1e-6; + p->aIDF[i] = idf; + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + if( rc!=SQLITE_OK ) p = 0; + } + *ppData = p; + return rc; +} + +/* +** Implementation of bm25() function. +*/ +static void fts5Bm25Function( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const double k1 = 1.2; /* Constant "k1" from BM25 formula */ + const double b = 0.75; /* Constant "b" from BM25 formula */ + int rc = SQLITE_OK; /* Error code */ + double score = 0.0; /* SQL function return value */ + Fts5Bm25Data *pData; /* Values allocated/calculated once only */ + int i; /* Iterator variable */ + int nInst = 0; /* Value returned by xInstCount() */ + double D = 0.0; /* Total number of tokens in row */ + double *aFreq = 0; /* Array of phrase freq. for current row */ + + /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation) + ** for each phrase in the query for the current row. */ + rc = fts5Bm25GetData(pApi, pFts, &pData); + if( rc==SQLITE_OK ){ + aFreq = pData->aFreq; + memset(aFreq, 0, sizeof(double) * pData->nPhrase); + rc = pApi->xInstCount(pFts, &nInst); + } + for(i=0; rc==SQLITE_OK && i<nInst; i++){ + int ip; int ic; int io; + rc = pApi->xInst(pFts, i, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0; + aFreq[ip] += w; + } + } + + /* Figure out the total size of the current row in tokens. */ + if( rc==SQLITE_OK ){ + int nTok; + rc = pApi->xColumnSize(pFts, -1, &nTok); + D = (double)nTok; + } + + /* Determine the BM25 score for the current row. */ + for(i=0; rc==SQLITE_OK && i<pData->nPhrase; i++){ + score += pData->aIDF[i] * ( + ( aFreq[i] * (k1 + 1.0) ) / + ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) + ); + } + + /* If no error has occurred, return the calculated score. Otherwise, + ** throw an SQL exception. */ + if( rc==SQLITE_OK ){ + sqlite3_result_double(pCtx, -1.0 * score); + }else{ + sqlite3_result_error_code(pCtx, rc); + } +} + +static int sqlite3Fts5AuxInit(fts5_api *pApi){ + struct Builtin { + const char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc;/* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + } aBuiltin [] = { + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + }; + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ + rc = pApi->xCreateFunction(pApi, + aBuiltin[i].zFunc, + aBuiltin[i].pUserData, + aBuiltin[i].xFunc, + aBuiltin[i].xDestroy + ); + } + + return rc; +} + + + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + + +/* #include "fts5Int.h" */ + +static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ + if( (u32)pBuf->nSpace<nByte ){ + u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64; + u8 *pNew; + while( nNew<nByte ){ + nNew = nNew * 2; + } + pNew = sqlite3_realloc(pBuf->p, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = nNew; + pBuf->p = pNew; + } + } + return 0; +} + + +/* +** Encode value iVal as an SQLite varint and append it to the buffer object +** pBuf. If an OOM error occurs, set the error code in p. +*/ +static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ + if( fts5BufferGrow(pRc, pBuf, 9) ) return; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal); +} + +static void sqlite3Fts5Put32(u8 *aBuf, int iVal){ + aBuf[0] = (iVal>>24) & 0x00FF; + aBuf[1] = (iVal>>16) & 0x00FF; + aBuf[2] = (iVal>> 8) & 0x00FF; + aBuf[3] = (iVal>> 0) & 0x00FF; +} + +static int sqlite3Fts5Get32(const u8 *aBuf){ + return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3]; +} + +/* +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static void sqlite3Fts5BufferAppendBlob( + int *pRc, + Fts5Buffer *pBuf, + u32 nData, + const u8 *pData +){ + assert_nc( *pRc || nData>=0 ); + if( fts5BufferGrow(pRc, pBuf, nData) ) return; + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; +} + +/* +** Append the nul-terminated string zStr to the buffer pBuf. This function +** ensures that the byte following the buffer data is set to 0x00, even +** though this byte is not included in the pBuf->n count. +*/ +static void sqlite3Fts5BufferAppendString( + int *pRc, + Fts5Buffer *pBuf, + const char *zStr +){ + int nStr = (int)strlen(zStr); + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr); + pBuf->n--; +} + +/* +** Argument zFmt is a printf() style format string. This function performs +** the printf() style processing, then appends the results to buffer pBuf. +** +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte +** following the buffer data is set to 0x00, even though this byte is not +** included in the pBuf->n count. +*/ +static void sqlite3Fts5BufferAppendPrintf( + int *pRc, + Fts5Buffer *pBuf, + char *zFmt, ... +){ + if( *pRc==SQLITE_OK ){ + char *zTmp; + va_list ap; + va_start(ap, zFmt); + zTmp = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + if( zTmp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); + sqlite3_free(zTmp); + } + } +} + +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Free any buffer allocated by pBuf. Zero the structure before returning. +*/ +static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ + sqlite3_free(pBuf->p); + memset(pBuf, 0, sizeof(Fts5Buffer)); +} + +/* +** Zero the contents of the buffer object. But do not free the associated +** memory allocation. +*/ +static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ + pBuf->n = 0; +} + +/* +** Set the buffer to contain nData/pData. If an OOM error occurs, leave an +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static void sqlite3Fts5BufferSet( + int *pRc, + Fts5Buffer *pBuf, + int nData, + const u8 *pData +){ + pBuf->n = 0; + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData); +} + +static int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +){ + int i = *pi; + if( i>=n ){ + /* EOF */ + *piOff = -1; + return 1; + }else{ + i64 iOff = *piOff; + int iVal; + fts5FastGetVarint32(a, i, iVal); + if( iVal==1 ){ + fts5FastGetVarint32(a, i, iVal); + iOff = ((i64)iVal) << 32; + fts5FastGetVarint32(a, i, iVal); + } + *piOff = iOff + (iVal-2); + *pi = i; + return 0; + } +} + + +/* +** Advance the iterator object passed as the only argument. Return true +** if the iterator reaches EOF, or false otherwise. +*/ +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ + pIter->bEof = 1; + } + return pIter->bEof; +} + +static int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +){ + memset(pIter, 0, sizeof(*pIter)); + pIter->a = a; + pIter->n = n; + sqlite3Fts5PoslistReaderNext(pIter); + return pIter->bEof; +} + +/* +** Append position iPos to the position list being accumulated in buffer +** pBuf, which must be already be large enough to hold the new data. +** The previous position written to this list is *piPrev. *piPrev is set +** to iPos before returning. +*/ +static void sqlite3Fts5PoslistSafeAppend( + Fts5Buffer *pBuf, + i64 *piPrev, + i64 iPos +){ + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; + if( (iPos & colmask) != (*piPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + *piPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); + *piPrev = iPos; +} + +static int sqlite3Fts5PoslistWriterAppend( + Fts5Buffer *pBuf, + Fts5PoslistWriter *pWriter, + i64 iPos +){ + int rc = 0; /* Initialized only to suppress erroneous warning from Clang */ + if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; + sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); + return SQLITE_OK; +} + +static void *sqlite3Fts5MallocZero(int *pRc, int nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet==0 && nByte>0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +/* +** Return a nul-terminated copy of the string indicated by pIn. If nIn +** is non-negative, then it is the length of the string in bytes. Otherwise, +** the length of the string is determined using strlen(). +** +** It is the responsibility of the caller to eventually free the returned +** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. +*/ +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + if( nIn<0 ){ + nIn = (int)strlen(pIn); + } + zRet = (char*)sqlite3_malloc(nIn+1); + if( zRet ){ + memcpy(zRet, pIn, nIn); + zRet[nIn] = '\0'; + }else{ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Return true if character 't' may be part of an FTS5 bareword, or false +** otherwise. Characters that may be part of barewords: +** +** * All non-ASCII characters, +** * The 52 upper and lower case ASCII characters, and +** * The 10 integer ASCII characters. +** * The underscore character "_" (0x5F). +** * The unicode "subsitute" character (0x1A). +*/ +static int sqlite3Fts5IsBareword(char t){ + u8 aBareword[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ + }; + + return (t & 0x80) || aBareword[(int)t]; +} + + +/************************************************************************* +*/ +typedef struct Fts5TermsetEntry Fts5TermsetEntry; +struct Fts5TermsetEntry { + char *pTerm; + int nTerm; + int iIdx; /* Index (main or aPrefix[] entry) */ + Fts5TermsetEntry *pNext; +}; + +struct Fts5Termset { + Fts5TermsetEntry *apHash[512]; +}; + +static int sqlite3Fts5TermsetNew(Fts5Termset **pp){ + int rc = SQLITE_OK; + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); + return rc; +} + +static int sqlite3Fts5TermsetAdd( + Fts5Termset *p, + int iIdx, + const char *pTerm, int nTerm, + int *pbPresent +){ + int rc = SQLITE_OK; + *pbPresent = 0; + if( p ){ + int i; + u32 hash = 13; + Fts5TermsetEntry *pEntry; + + /* Calculate a hash value for this term. This is the same hash checksum + ** used by the fts5_hash.c module. This is not important for correct + ** operation of the module, but is necessary to ensure that some tests + ** designed to produce hash table collisions really do work. */ + for(i=nTerm-1; i>=0; i--){ + hash = (hash << 3) ^ hash ^ pTerm[i]; + } + hash = (hash << 3) ^ hash ^ iIdx; + hash = hash % ArraySize(p->apHash); + + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + pEntry->iIdx = iIdx; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } + } + } + + return rc; +} + +static void sqlite3Fts5TermsetFree(Fts5Termset *p){ + if( p ){ + u32 i; + for(i=0; i<ArraySize(p->apHash); i++){ + Fts5TermsetEntry *pEntry = p->apHash[i]; + while( pEntry ){ + Fts5TermsetEntry *pDel = pEntry; + pEntry = pEntry->pNext; + sqlite3_free(pDel); + } + } + sqlite3_free(p); + } +} + +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + + +/* #include "fts5Int.h" */ + +#define FTS5_DEFAULT_PAGE_SIZE 4050 +#define FTS5_DEFAULT_AUTOMERGE 4 +#define FTS5_DEFAULT_USERMERGE 4 +#define FTS5_DEFAULT_CRISISMERGE 16 +#define FTS5_DEFAULT_HASHSIZE (1024*1024) + +/* Maximum allowed page size */ +#define FTS5_MAX_PAGE_SIZE (128*1024) + +static int fts5_iswhitespace(char x){ + return (x==' '); +} + +static int fts5_isopenquote(char x){ + return (x=='"' || x=='\'' || x=='[' || x=='`'); +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a white-space character. +*/ +static const char *fts5ConfigSkipWhitespace(const char *pIn){ + const char *p = pIn; + if( p ){ + while( fts5_iswhitespace(*p) ){ p++; } + } + return p; +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a "bareword" character. +*/ +static const char *fts5ConfigSkipBareword(const char *pIn){ + const char *p = pIn; + while ( sqlite3Fts5IsBareword(*p) ) p++; + if( p==pIn ) p = 0; + return p; +} + +static int fts5_isdigit(char a){ + return (a>='0' && a<='9'); +} + + + +static const char *fts5ConfigSkipLiteral(const char *pIn){ + const char *p = pIn; + switch( *p ){ + case 'n': case 'N': + if( sqlite3_strnicmp("null", p, 4)==0 ){ + p = &p[4]; + }else{ + p = 0; + } + break; + + case 'x': case 'X': + p++; + if( *p=='\'' ){ + p++; + while( (*p>='a' && *p<='f') + || (*p>='A' && *p<='F') + || (*p>='0' && *p<='9') + ){ + p++; + } + if( *p=='\'' && 0==((p-pIn)%2) ){ + p++; + }else{ + p = 0; + } + }else{ + p = 0; + } + break; + + case '\'': + p++; + while( p ){ + if( *p=='\'' ){ + p++; + if( *p!='\'' ) break; + } + p++; + if( *p==0 ) p = 0; + } + break; + + default: + /* maybe a number */ + if( *p=='+' || *p=='-' ) p++; + while( fts5_isdigit(*p) ) p++; + + /* At this point, if the literal was an integer, the parse is + ** finished. Or, if it is a floating point value, it may continue + ** with either a decimal point or an 'E' character. */ + if( *p=='.' && fts5_isdigit(p[1]) ){ + p += 2; + while( fts5_isdigit(*p) ) p++; + } + if( p==pIn ) p = 0; + + break; + } + + return p; +} + +/* +** The first character of the string pointed to by argument z is guaranteed +** to be an open-quote character (see function fts5_isopenquote()). +** +** This function searches for the corresponding close-quote character within +** the string and, if found, dequotes the string in place and adds a new +** nul-terminator byte. +** +** If the close-quote is found, the value returned is the byte offset of +** the character immediately following it. Or, if the close-quote is not +** found, -1 is returned. If -1 is returned, the buffer is left in an +** undefined state. +*/ +static int fts5Dequote(char *z){ + char q; + int iIn = 1; + int iOut = 0; + q = z[0]; + + /* Set stack variable q to the close-quote character */ + assert( q=='[' || q=='\'' || q=='"' || q=='`' ); + if( q=='[' ) q = ']'; + + while( ALWAYS(z[iIn]) ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + + z[iOut] = '\0'; + return iIn; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static void sqlite3Fts5Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + assert( 0==fts5_iswhitespace(z[0]) ); + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + fts5Dequote(z); + } +} + + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = (int)strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** Parse a "special" CREATE VIRTUAL TABLE directive and update +** configuration object pConfig as appropriate. +** +** If successful, object pConfig is updated and SQLITE_OK returned. If +** an error occurs, an SQLite error code is returned and an error message +** may be left in *pzErr. It is the responsibility of the caller to +** eventually free any such error message using sqlite3_free(). +*/ +static int fts5ConfigParseSpecial( + Fts5Global *pGlobal, + Fts5Config *pConfig, /* Configuration object to update */ + const char *zCmd, /* Special command to parse */ + const char *zArg, /* Argument to parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + int nCmd = (int)strlen(zCmd); + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ + const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; + const char *p; + int bFirst = 1; + if( pConfig->aPrefix==0 ){ + pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte); + if( rc ) return rc; + } + + p = zArg; + while( 1 ){ + int nPre = 0; + + while( p[0]==' ' ) p++; + if( bFirst==0 && p[0]==',' ){ + p++; + while( p[0]==' ' ) p++; + }else if( p[0]=='\0' ){ + break; + } + if( p[0]<'0' || p[0]>'9' ){ + *pzErr = sqlite3_mprintf("malformed prefix=... directive"); + rc = SQLITE_ERROR; + break; + } + + if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){ + *pzErr = sqlite3_mprintf( + "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES + ); + rc = SQLITE_ERROR; + break; + } + + while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ + nPre = nPre*10 + (p[0] - '0'); + p++; + } + + if( nPre<=0 || nPre>=1000 ){ + *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); + rc = SQLITE_ERROR; + break; + } + + pConfig->aPrefix[pConfig->nPrefix] = nPre; + pConfig->nPrefix++; + bFirst = 0; + } + assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES ); + return rc; + } + + if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ + const char *p = (const char*)zArg; + int nArg = (int)strlen(zArg) + 1; + char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); + char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); + char *pSpace = pDel; + + if( azArg && pSpace ){ + if( pConfig->pTok ){ + *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); + rc = SQLITE_ERROR; + }else{ + for(nArg=0; p && *p; nArg++){ + const char *p2 = fts5ConfigSkipWhitespace(p); + if( *p2=='\'' ){ + p = fts5ConfigSkipLiteral(p2); + }else{ + p = fts5ConfigSkipBareword(p2); + } + if( p ){ + memcpy(pSpace, p2, p-p2); + azArg[nArg] = pSpace; + sqlite3Fts5Dequote(pSpace); + pSpace += (p - p2) + 1; + p = fts5ConfigSkipWhitespace(p); + } + } + if( p==0 ){ + *pzErr = sqlite3_mprintf("parse error in tokenize directive"); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5GetTokenizer(pGlobal, + (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, + pzErr + ); + } + } + } + + sqlite3_free(azArg); + sqlite3_free(pDel); + return rc; + } + + if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + *pzErr = sqlite3_mprintf("multiple content=... directives"); + rc = SQLITE_ERROR; + }else{ + if( zArg[0] ){ + pConfig->eContent = FTS5_CONTENT_EXTERNAL; + pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg); + }else{ + pConfig->eContent = FTS5_CONTENT_NONE; + } + } + return rc; + } + + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ + if( pConfig->zContentRowid ){ + *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); + rc = SQLITE_ERROR; + }else{ + pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1); + } + return rc; + } + + if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bColumnsize = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); + } + return rc; + } + + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); + return SQLITE_ERROR; +} + +/* +** Allocate an instance of the default tokenizer ("simple") at +** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error +** code if an error occurs. +*/ +static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ + assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); + return sqlite3Fts5GetTokenizer( + pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0 + ); +} + +/* +** Gobble up the first bareword or quoted word from the input buffer zIn. +** Return a pointer to the character immediately following the last in +** the gobbled word if successful, or a NULL pointer otherwise (failed +** to find close-quote character). +** +** Before returning, set pzOut to point to a new buffer containing a +** nul-terminated, dequoted copy of the gobbled word. If the word was +** quoted, *pbQuoted is also set to 1 before returning. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op (NULL is returned). Otherwise, if an OOM occurs within this +** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not* +** set if a parse error (failed to find close quote) occurs. +*/ +static const char *fts5ConfigGobbleWord( + int *pRc, /* IN/OUT: Error code */ + const char *zIn, /* Buffer to gobble string/bareword from */ + char **pzOut, /* OUT: malloc'd buffer containing str/bw */ + int *pbQuoted /* OUT: Set to true if dequoting required */ +){ + const char *zRet = 0; + + int nIn = (int)strlen(zIn); + char *zOut = sqlite3_malloc(nIn+1); + + assert( *pRc==SQLITE_OK ); + *pbQuoted = 0; + *pzOut = 0; + + if( zOut==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memcpy(zOut, zIn, nIn+1); + if( fts5_isopenquote(zOut[0]) ){ + int ii = fts5Dequote(zOut); + zRet = &zIn[ii]; + *pbQuoted = 1; + }else{ + zRet = fts5ConfigSkipBareword(zIn); + if( zRet ){ + zOut[zRet-zIn] = '\0'; + } + } + } + + if( zRet==0 ){ + sqlite3_free(zOut); + }else{ + *pzOut = zOut; + } + + return zRet; +} + +static int fts5ConfigParseColumn( + Fts5Config *p, + char *zCol, + char *zArg, + char **pzErr +){ + int rc = SQLITE_OK; + if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) + || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) + ){ + *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol); + rc = SQLITE_ERROR; + }else if( zArg ){ + if( 0==sqlite3_stricmp(zArg, "unindexed") ){ + p->abUnindexed[p->nCol] = 1; + }else{ + *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); + rc = SQLITE_ERROR; + } + } + + p->azCol[p->nCol++] = zCol; + return rc; +} + +/* +** Populate the Fts5Config.zContentExprlist string. +*/ +static int fts5ConfigMakeExprlist(Fts5Config *p){ + int i; + int rc = SQLITE_OK; + Fts5Buffer buf = {0, 0, 0}; + + sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); + if( p->eContent!=FTS5_CONTENT_NONE ){ + for(i=0; i<p->nCol; i++){ + if( p->eContent==FTS5_CONTENT_EXTERNAL ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + } + } + } + + assert( p->zContentExprlist==0 ); + p->zContentExprlist = (char*)buf.p; + return rc; +} + +/* +** Arguments nArg/azArg contain the string arguments passed to the xCreate +** or xConnect method of the virtual table. This function attempts to +** allocate an instance of Fts5Config containing the results of parsing +** those arguments. +** +** If successful, SQLITE_OK is returned and *ppOut is set to point to the +** new Fts5Config object. If an error occurs, an SQLite error code is +** returned, *ppOut is set to NULL and an error message may be left in +** *pzErr. It is the responsibility of the caller to eventually free any +** such error message using sqlite3_free(). +*/ +static int sqlite3Fts5ConfigParse( + Fts5Global *pGlobal, + sqlite3 *db, + int nArg, /* Number of arguments */ + const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */ + Fts5Config **ppOut, /* OUT: Results of parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pRet; /* New object to return */ + int i; + int nByte; + + *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + if( pRet==0 ) return SQLITE_NOMEM; + memset(pRet, 0, sizeof(Fts5Config)); + pRet->db = db; + pRet->iCookie = -1; + + nByte = nArg * (sizeof(char*) + sizeof(u8)); + pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); + pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; + pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); + pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); + pRet->bColumnsize = 1; + pRet->eDetail = FTS5_DETAIL_FULL; +#ifdef SQLITE_DEBUG + pRet->bPrefixIndex = 1; +#endif + if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ + *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); + rc = SQLITE_ERROR; + } + + for(i=3; rc==SQLITE_OK && i<nArg; i++){ + const char *zOrig = azArg[i]; + const char *z; + char *zOne = 0; + char *zTwo = 0; + int bOption = 0; + int bMustBeCol = 0; + + z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol); + z = fts5ConfigSkipWhitespace(z); + if( z && *z=='=' ){ + bOption = 1; + z++; + if( bMustBeCol ) z = 0; + } + z = fts5ConfigSkipWhitespace(z); + if( z && z[0] ){ + int bDummy; + z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy); + if( z && z[0] ) z = 0; + } + + if( rc==SQLITE_OK ){ + if( z==0 ){ + *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig); + rc = SQLITE_ERROR; + }else{ + if( bOption ){ + rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr); + }else{ + rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); + zOne = 0; + } + } + } + + sqlite3_free(zOne); + sqlite3_free(zTwo); + } + + /* If a tokenizer= option was successfully parsed, the tokenizer has + ** already been allocated. Otherwise, allocate an instance of the default + ** tokenizer (unicode61) now. */ + if( rc==SQLITE_OK && pRet->pTok==0 ){ + rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); + } + + /* If no zContent option was specified, fill in the default values. */ + if( rc==SQLITE_OK && pRet->zContent==0 ){ + const char *zTail = 0; + assert( pRet->eContent==FTS5_CONTENT_NORMAL + || pRet->eContent==FTS5_CONTENT_NONE + ); + if( pRet->eContent==FTS5_CONTENT_NORMAL ){ + zTail = "content"; + }else if( pRet->bColumnsize ){ + zTail = "docsize"; + } + + if( zTail ){ + pRet->zContent = sqlite3Fts5Mprintf( + &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail + ); + } + } + + if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ + pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1); + } + + /* Formulate the zContentExprlist text */ + if( rc==SQLITE_OK ){ + rc = fts5ConfigMakeExprlist(pRet); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts5ConfigFree(pRet); + *ppOut = 0; + } + return rc; +} + +/* +** Free the configuration object passed as the only argument. +*/ +static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ + if( pConfig ){ + int i; + if( pConfig->pTok ){ + pConfig->pTokApi->xDelete(pConfig->pTok); + } + sqlite3_free(pConfig->zDb); + sqlite3_free(pConfig->zName); + for(i=0; i<pConfig->nCol; i++){ + sqlite3_free(pConfig->azCol[i]); + } + sqlite3_free(pConfig->azCol); + sqlite3_free(pConfig->aPrefix); + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + sqlite3_free(pConfig->zContent); + sqlite3_free(pConfig->zContentRowid); + sqlite3_free(pConfig->zContentExprlist); + sqlite3_free(pConfig); + } +} + +/* +** Call sqlite3_declare_vtab() based on the contents of the configuration +** object passed as the only argument. Return SQLITE_OK if successful, or +** an SQLite error code if an error occurs. +*/ +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ + int i; + int rc = SQLITE_OK; + char *zSql; + + zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x("); + for(i=0; zSql && i<pConfig->nCol; i++){ + const char *zSep = (i==0?"":", "); + zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]); + } + zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", + zSql, pConfig->zName, FTS5_RANK_NAME + ); + + assert( zSql || rc==SQLITE_NOMEM ); + if( zSql ){ + rc = sqlite3_declare_vtab(pConfig->db, zSql); + sqlite3_free(zSql); + } + + return rc; +} + +/* +** Tokenize the text passed via the second and third arguments. +** +** The callback is invoked once for each token in the input text. The +** arguments passed to it are, in order: +** +** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize() +** const char *pToken // Pointer to buffer containing token +** int nToken // Size of token in bytes +** int iStart // Byte offset of start of token within input text +** int iEnd // Byte offset of end of token within input text +** int iPos // Position of token in input (first token is 0) +** +** If the callback returns a non-zero value the tokenization is abandoned +** and no further callbacks are issued. +** +** This function returns SQLITE_OK if successful or an SQLite error code +** if an error occurs. If the tokenization was abandoned early because +** the callback returned SQLITE_DONE, this is not an error and this function +** still returns SQLITE_OK. Or, if the tokenization was abandoned early +** because the callback returned another non-zero value, it is assumed +** to be an SQLite error code and returned to the caller. +*/ +static int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +){ + if( pText==0 ) return SQLITE_OK; + return pConfig->pTokApi->xTokenize( + pConfig->pTok, pCtx, flags, pText, nText, xToken + ); +} + +/* +** Argument pIn points to the first character in what is expected to be +** a comma-separated list of SQL literals followed by a ')' character. +** If it actually is this, return a pointer to the ')'. Otherwise, return +** NULL to indicate a parse error. +*/ +static const char *fts5ConfigSkipArgs(const char *pIn){ + const char *p = pIn; + + while( 1 ){ + p = fts5ConfigSkipWhitespace(p); + p = fts5ConfigSkipLiteral(p); + p = fts5ConfigSkipWhitespace(p); + if( p==0 || *p==')' ) break; + if( *p!=',' ){ + p = 0; + break; + } + p++; + } + + return p; +} + +/* +** Parameter zIn contains a rank() function specification. The format of +** this is: +** +** + Bareword (function name) +** + Open parenthesis - "(" +** + Zero or more SQL literals in a comma separated list +** + Close parenthesis - ")" +*/ +static int sqlite3Fts5ConfigParseRank( + const char *zIn, /* Input string */ + char **pzRank, /* OUT: Rank function name */ + char **pzRankArgs /* OUT: Rank function arguments */ +){ + const char *p = zIn; + const char *pRank; + char *zRank = 0; + char *zRankArgs = 0; + int rc = SQLITE_OK; + + *pzRank = 0; + *pzRankArgs = 0; + + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + p = fts5ConfigSkipWhitespace(p); + pRank = p; + p = fts5ConfigSkipBareword(p); + + if( p ){ + zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank); + if( zRank ) memcpy(zRank, pRank, p-pRank); + }else{ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + p = fts5ConfigSkipWhitespace(p); + if( *p!='(' ) rc = SQLITE_ERROR; + p++; + } + if( rc==SQLITE_OK ){ + const char *pArgs; + p = fts5ConfigSkipWhitespace(p); + pArgs = p; + if( *p!=')' ){ + p = fts5ConfigSkipArgs(p); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs); + if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs); + } + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRank); + assert( zRankArgs==0 ); + }else{ + *pzRank = zRank; + *pzRankArgs = zRankArgs; + } + return rc; +} + +static int sqlite3Fts5ConfigSetValue( + Fts5Config *pConfig, + const char *zKey, + sqlite3_value *pVal, + int *pbBadkey +){ + int rc = SQLITE_OK; + + if( 0==sqlite3_stricmp(zKey, "pgsz") ){ + int pgsz = 0; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + pgsz = sqlite3_value_int(pVal); + } + if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ + *pbBadkey = 1; + }else{ + pConfig->pgsz = pgsz; + } + } + + else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ + int nHashSize = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nHashSize = sqlite3_value_int(pVal); + } + if( nHashSize<=0 ){ + *pbBadkey = 1; + }else{ + pConfig->nHashSize = nHashSize; + } + } + + else if( 0==sqlite3_stricmp(zKey, "automerge") ){ + int nAutomerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nAutomerge = sqlite3_value_int(pVal); + } + if( nAutomerge<0 || nAutomerge>64 ){ + *pbBadkey = 1; + }else{ + if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nAutomerge = nAutomerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ + int nUsermerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nUsermerge = sqlite3_value_int(pVal); + } + if( nUsermerge<2 || nUsermerge>16 ){ + *pbBadkey = 1; + }else{ + pConfig->nUsermerge = nUsermerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ + int nCrisisMerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nCrisisMerge = sqlite3_value_int(pVal); + } + if( nCrisisMerge<0 ){ + *pbBadkey = 1; + }else{ + if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nCrisisMerge = nCrisisMerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "rank") ){ + const char *zIn = (const char*)sqlite3_value_text(pVal); + char *zRank; + char *zRankArgs; + rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); + if( rc==SQLITE_OK ){ + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + pConfig->zRank = zRank; + pConfig->zRankArgs = zRankArgs; + }else if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + *pbBadkey = 1; + } + }else{ + *pbBadkey = 1; + } + return rc; +} + +/* +** Load the contents of the %_config table into memory. +*/ +static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ + const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; + char *zSql; + sqlite3_stmt *p = 0; + int rc = SQLITE_OK; + int iVersion = 0; + + /* Set default values */ + pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; + pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; + pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + + zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); + sqlite3_free(zSql); + } + + assert( rc==SQLITE_OK || p==0 ); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(p) ){ + const char *zK = (const char*)sqlite3_column_text(p, 0); + sqlite3_value *pVal = sqlite3_column_value(p, 1); + if( 0==sqlite3_stricmp(zK, "version") ){ + iVersion = sqlite3_value_int(pVal); + }else{ + int bDummy = 0; + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy); + } + } + rc = sqlite3_finalize(p); + } + + if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + rc = SQLITE_ERROR; + if( pConfig->pzErrmsg ){ + assert( 0==*pConfig->pzErrmsg ); + *pConfig->pzErrmsg = sqlite3_mprintf( + "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION + ); + } + } + + if( rc==SQLITE_OK ){ + pConfig->iCookie = iCookie; + } + return rc; +} + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +/* #include "fts5Int.h" */ +/* #include "fts5parse.h" */ + +/* +** All token types in the generated fts5parse.h file are greater than 0. +*/ +#define FTS5_EOF 0 + +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + +typedef struct Fts5ExprTerm Fts5ExprTerm; + +/* +** Functions generated by lemon from fts5parse.y. +*/ +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64)); +static void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*)); +static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); +#ifndef NDEBUG +/* #include <stdio.h> */ +static void sqlite3Fts5ParserTrace(FILE*, char*); +#endif + + +struct Fts5Expr { + Fts5Index *pIndex; + Fts5Config *pConfig; + Fts5ExprNode *pRoot; + int bDesc; /* Iterate in descending rowid order */ + int nPhrase; /* Number of phrases in expression */ + Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ +}; + +/* +** eType: +** Expression node type. Always one of: +** +** FTS5_AND (nChild, apChild valid) +** FTS5_OR (nChild, apChild valid) +** FTS5_NOT (nChild, apChild valid) +** FTS5_STRING (pNear valid) +** FTS5_TERM (pNear valid) +*/ +struct Fts5ExprNode { + int eType; /* Node type */ + int bEof; /* True at EOF */ + int bNomatch; /* True if entry is not a match */ + + /* Next method for this node. */ + int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); + + i64 iRowid; /* Current rowid */ + Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ + + /* Child nodes. For a NOT node, this array always contains 2 entries. For + ** AND or OR nodes, it contains 2 or more entries. */ + int nChild; /* Number of child nodes */ + Fts5ExprNode *apChild[1]; /* Array of child nodes */ +}; + +#define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) + +/* +** Invoke the xNext method of an Fts5ExprNode object. This macro should be +** used as if it has the same signature as the xNext() methods themselves. +*/ +#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d)) + +/* +** An instance of the following structure represents a single search term +** or term prefix. +*/ +struct Fts5ExprTerm { + int bPrefix; /* True for a prefix term */ + char *zTerm; /* nul-terminated term */ + Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ +}; + +/* +** A phrase. One or more terms that must appear in a contiguous sequence +** within a document for it to match. +*/ +struct Fts5ExprPhrase { + Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ + Fts5Buffer poslist; /* Current position list */ + int nTerm; /* Number of entries in aTerm[] */ + Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ +}; + +/* +** One or more phrases that must appear within a certain token distance of +** each other within each matching document. +*/ +struct Fts5ExprNearset { + int nNear; /* NEAR parameter */ + Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ + int nPhrase; /* Number of entries in aPhrase[] array */ + Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ +}; + + +/* +** Parse context. +*/ +struct Fts5Parse { + Fts5Config *pConfig; + char *zErr; + int rc; + int nPhrase; /* Size of apPhrase array */ + Fts5ExprPhrase **apPhrase; /* Array of all phrases */ + Fts5ExprNode *pExpr; /* Result of a successful parse */ +}; + +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pParse->rc==SQLITE_OK ){ + pParse->zErr = sqlite3_vmprintf(zFmt, ap); + pParse->rc = SQLITE_ERROR; + } + va_end(ap); +} + +static int fts5ExprIsspace(char t){ + return t==' ' || t=='\t' || t=='\n' || t=='\r'; +} + +/* +** Read the first token from the nul-terminated string at *pz. +*/ +static int fts5ExprGetToken( + Fts5Parse *pParse, + const char **pz, /* IN/OUT: Pointer into buffer */ + Fts5Token *pToken +){ + const char *z = *pz; + int tok; + + /* Skip past any whitespace */ + while( fts5ExprIsspace(*z) ) z++; + + pToken->p = z; + pToken->n = 1; + switch( *z ){ + case '(': tok = FTS5_LP; break; + case ')': tok = FTS5_RP; break; + case '{': tok = FTS5_LCP; break; + case '}': tok = FTS5_RCP; break; + case ':': tok = FTS5_COLON; break; + case ',': tok = FTS5_COMMA; break; + case '+': tok = FTS5_PLUS; break; + case '*': tok = FTS5_STAR; break; + case '\0': tok = FTS5_EOF; break; + + case '"': { + const char *z2; + tok = FTS5_STRING; + + for(z2=&z[1]; 1; z2++){ + if( z2[0]=='"' ){ + z2++; + if( z2[0]!='"' ) break; + } + if( z2[0]=='\0' ){ + sqlite3Fts5ParseError(pParse, "unterminated string"); + return FTS5_EOF; + } + } + pToken->n = (z2 - z); + break; + } + + default: { + const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } + tok = FTS5_STRING; + for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); + pToken->n = (z2 - z); + if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR; + if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT; + if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND; + break; + } + } + + *pz = &pToken->p[pToken->n]; + return tok; +} + +static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); } +static void fts5ParseFree(void *p){ sqlite3_free(p); } + +static int sqlite3Fts5ExprNew( + Fts5Config *pConfig, /* FTS5 Configuration */ + const char *zExpr, /* Expression text */ + Fts5Expr **ppNew, + char **pzErr +){ + Fts5Parse sParse; + Fts5Token token; + const char *z = zExpr; + int t; /* Next token type */ + void *pEngine; + Fts5Expr *pNew; + + *ppNew = 0; + *pzErr = 0; + memset(&sParse, 0, sizeof(sParse)); + pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc); + if( pEngine==0 ){ return SQLITE_NOMEM; } + sParse.pConfig = pConfig; + + do { + t = fts5ExprGetToken(&sParse, &z, &token); + sqlite3Fts5Parser(pEngine, t, token, &sParse); + }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); + sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + + assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); + if( sParse.rc==SQLITE_OK ){ + *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + if( pNew==0 ){ + sParse.rc = SQLITE_NOMEM; + sqlite3Fts5ParseNodeFree(sParse.pExpr); + }else{ + if( !sParse.pExpr ){ + const int nByte = sizeof(Fts5ExprNode); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); + if( pNew->pRoot ){ + pNew->pRoot->bEof = 1; + } + }else{ + pNew->pRoot = sParse.pExpr; + } + pNew->pIndex = 0; + pNew->pConfig = pConfig; + pNew->apExprPhrase = sParse.apPhrase; + pNew->nPhrase = sParse.nPhrase; + sParse.apPhrase = 0; + } + }else{ + sqlite3Fts5ParseNodeFree(sParse.pExpr); + } + + sqlite3_free(sParse.apPhrase); + *pzErr = sParse.zErr; + return sParse.rc; +} + +/* +** Free the expression node object passed as the only argument. +*/ +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ + if( p ){ + int i; + for(i=0; i<p->nChild; i++){ + sqlite3Fts5ParseNodeFree(p->apChild[i]); + } + sqlite3Fts5ParseNearsetFree(p->pNear); + sqlite3_free(p); + } +} + +/* +** Free the expression object passed as the only argument. +*/ +static void sqlite3Fts5ExprFree(Fts5Expr *p){ + if( p ){ + sqlite3Fts5ParseNodeFree(p->pRoot); + sqlite3_free(p->apExprPhrase); + sqlite3_free(p); + } +} + +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = p->pIter->iRowid; + if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){ + iRet = iRowid; + bRetValid = 1; + } + } + } + + if( pbEof && bRetValid==0 ) *pbEof = 1; + return iRet; +} + +/* +** Argument pTerm must be a synonym iterator. +*/ +static int fts5ExprSynonymList( + Fts5ExprTerm *pTerm, + i64 iRowid, + Fts5Buffer *pBuf, /* Use this buffer for space if required */ + u8 **pa, int *pn +){ + Fts5PoslistReader aStatic[4]; + Fts5PoslistReader *aIter = aStatic; + int nIter = 0; + int nAlloc = 4; + int rc = SQLITE_OK; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ + if( pIter->nData==0 ) continue; + if( nIter==nAlloc ){ + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); + aIter = aNew; + } + sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; + } + } + + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + i64 iPrev = -1; + fts5BufferZero(pBuf); + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; i<nIter; i++){ + if( aIter[i].bEof==0 ){ + if( aIter[i].iPos==iPrev ){ + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue; + } + if( aIter[i].iPos<iMin ){ + iMin = aIter[i].iPos; + } + } + } + if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break; + rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin); + iPrev = iMin; + } + if( rc==SQLITE_OK ){ + *pa = pBuf->p; + *pn = pBuf->n; + } + } + + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + + +/* +** All individual term iterators in pPhrase are guaranteed to be valid and +** pointing to the same rowid when this function is called. This function +** checks if the current rowid really is a match, and if so populates +** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch +** is set to true if this is really a match, or false otherwise. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if the current rowid is +** not a match. +*/ +static int fts5ExprPhraseIsMatch( + Fts5ExprNode *pNode, /* Node pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbMatch /* OUT: Set to true if really a match */ +){ + Fts5PoslistWriter writer = {0}; + Fts5PoslistReader aStatic[4]; + Fts5PoslistReader *aIter = aStatic; + int i; + int rc = SQLITE_OK; + + fts5BufferZero(&pPhrase->poslist); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pPhrase->nTerm>ArraySize(aStatic) ){ + int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; + aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( !aIter ) return SQLITE_NOMEM; + } + memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); + + /* Initialize a term iterator for each term in the phrase */ + for(i=0; i<pPhrase->nTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + int n = 0; + int bFlag = 0; + u8 *a = 0; + if( pTerm->pSynonym ){ + Fts5Buffer buf = {0, 0, 0}; + rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n); + if( rc ){ + sqlite3_free(a); + goto ismatch_out; + } + if( a==buf.p ) bFlag = 1; + }else{ + a = (u8*)pTerm->pIter->pData; + n = pTerm->pIter->nData; + } + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); + aIter[i].bFlag = (u8)bFlag; + if( aIter[i].bEof ) goto ismatch_out; + } + + while( 1 ){ + int bMatch; + i64 iPos = aIter[0].iPos; + do { + bMatch = 1; + for(i=0; i<pPhrase->nTerm; i++){ + Fts5PoslistReader *pPos = &aIter[i]; + i64 iAdj = iPos + i; + if( pPos->iPos!=iAdj ){ + bMatch = 0; + while( pPos->iPos<iAdj ){ + if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out; + } + if( pPos->iPos>iAdj ) iPos = pPos->iPos-i; + } + } + }while( bMatch==0 ); + + /* Append position iPos to the output */ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; + + for(i=0; i<pPhrase->nTerm; i++){ + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; + } + } + + ismatch_out: + *pbMatch = (pPhrase->poslist.n>0); + for(i=0; i<pPhrase->nTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + +typedef struct Fts5LookaheadReader Fts5LookaheadReader; +struct Fts5LookaheadReader { + const u8 *a; /* Buffer containing position list */ + int n; /* Size of buffer a[] in bytes */ + int i; /* Current offset in position list */ + i64 iPos; /* Current position */ + i64 iLookahead; /* Next position */ +}; + +#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62) + +static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){ + p->iPos = p->iLookahead; + if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){ + p->iLookahead = FTS5_LOOKAHEAD_EOF; + } + return (p->iPos==FTS5_LOOKAHEAD_EOF); +} + +static int fts5LookaheadReaderInit( + const u8 *a, int n, /* Buffer to read position list from */ + Fts5LookaheadReader *p /* Iterator object to initialize */ +){ + memset(p, 0, sizeof(Fts5LookaheadReader)); + p->a = a; + p->n = n; + fts5LookaheadReaderNext(p); + return fts5LookaheadReaderNext(p); +} + +typedef struct Fts5NearTrimmer Fts5NearTrimmer; +struct Fts5NearTrimmer { + Fts5LookaheadReader reader; /* Input iterator */ + Fts5PoslistWriter writer; /* Writer context */ + Fts5Buffer *pOut; /* Output poslist */ +}; + +/* +** The near-set object passed as the first argument contains more than +** one phrase. All phrases currently point to the same row. The +** Fts5ExprPhrase.poslist buffers are populated accordingly. This function +** tests if the current row contains instances of each phrase sufficiently +** close together to meet the NEAR constraint. Non-zero is returned if it +** does, or zero otherwise. +** +** If in/out parameter (*pRc) is set to other than SQLITE_OK when this +** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM) +** occurs within this function (*pRc) is set accordingly before returning. +** The return value is undefined in both these cases. +** +** If no error occurs and non-zero (a match) is returned, the position-list +** of each phrase object is edited to contain only those entries that +** meet the constraint before returning. +*/ +static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ + Fts5NearTrimmer aStatic[4]; + Fts5NearTrimmer *a = aStatic; + Fts5ExprPhrase **apPhrase = pNear->apPhrase; + + int i; + int rc = *pRc; + int bMatch; + + assert( pNear->nPhrase>1 ); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pNear->nPhrase>ArraySize(aStatic) ){ + int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; + a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); + }else{ + memset(aStatic, 0, sizeof(aStatic)); + } + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + } + + /* Initialize a lookahead iterator for each phrase. After passing the + ** buffer and buffer size to the lookaside-reader init function, zero + ** the phrase poslist buffer. The new poslist for the phrase (containing + ** the same entries as the original with some entries removed on account + ** of the NEAR constraint) is written over the original even as it is + ** being read. This is safe as the entries for the new poslist are a + ** subset of the old, so it is not possible for data yet to be read to + ** be overwritten. */ + for(i=0; i<pNear->nPhrase; i++){ + Fts5Buffer *pPoslist = &apPhrase[i]->poslist; + fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader); + pPoslist->n = 0; + a[i].pOut = pPoslist; + } + + while( 1 ){ + int iAdv; + i64 iMin; + i64 iMax; + + /* This block advances the phrase iterators until they point to a set of + ** entries that together comprise a match. */ + iMax = a[0].reader.iPos; + do { + bMatch = 1; + for(i=0; i<pNear->nPhrase; i++){ + Fts5LookaheadReader *pPos = &a[i].reader; + iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear; + if( pPos->iPos<iMin || pPos->iPos>iMax ){ + bMatch = 0; + while( pPos->iPos<iMin ){ + if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out; + } + if( pPos->iPos>iMax ) iMax = pPos->iPos; + } + } + }while( bMatch==0 ); + + /* Add an entry to each output position list */ + for(i=0; i<pNear->nPhrase; i++){ + i64 iPos = a[i].reader.iPos; + Fts5PoslistWriter *pWriter = &a[i].writer; + if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){ + sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos); + } + } + + iAdv = 0; + iMin = a[0].reader.iLookahead; + for(i=0; i<pNear->nPhrase; i++){ + if( a[i].reader.iLookahead < iMin ){ + iMin = a[i].reader.iLookahead; + iAdv = i; + } + } + if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out; + } + + ismatch_out: { + int bRet = a[0].pOut->n>0; + *pRc = rc; + if( a!=aStatic ) sqlite3_free(a); + return bRet; + } +} + +/* +** Advance iterator pIter until it points to a value equal to or laster +** than the initial value of *piLast. If this means the iterator points +** to a value laster than *piLast, update *piLast to the new lastest value. +** +** If the iterator reaches EOF, set *pbEof to true before returning. If +** an error occurs, set *pRc to an error code. If either *pbEof or *pRc +** are set, return a non-zero value. Otherwise, return zero. +*/ +static int fts5ExprAdvanceto( + Fts5IndexIter *pIter, /* Iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + i64 iLast = *piLast; + i64 iRowid; + + iRowid = pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ + int rc = sqlite3Fts5IterNextFrom(pIter, iLast); + if( rc || sqlite3Fts5IterEof(pIter) ){ + *pRc = rc; + *pbEof = 1; + return 1; + } + iRowid = pIter->iRowid; + assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); + } + *piLast = iRowid; + + return 0; +} + +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc /* OUT: Error code */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + int bEof = 0; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = p->pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pRc = rc; + bEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); + } + return bEof; +} + + +static int fts5ExprNearTest( + int *pRc, + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ +){ + Fts5ExprNearset *pNear = pNode->pNear; + int rc = *pRc; + + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ + pPhrase->poslist.n = 1; + } + } + } + return pPhrase->poslist.n; + }else{ + int i; + + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); + } + } + + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } +} + + +/* +** Initialize all term iterators in the pNear object. If any term is found +** to match no documents at all, return immediately without initializing any +** further iterators. +*/ +static int fts5ExprNearInitAll( + Fts5Expr *pExpr, + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + int i, j; + int rc = SQLITE_OK; + + assert( pNode->bNomatch==0 ); + for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + for(j=0; j<pPhrase->nTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bEof = 1; + + for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, + &p->pIter + ); + assert( rc==SQLITE_OK || p->pIter==0 ); + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ + bEof = 0; + } + } + + if( bEof ){ + pNode->bEof = 1; + return rc; + } + } + } + + return rc; +} + +/* +** If pExpr is an ASC iterator, this function returns a value with the +** same sign as: +** +** (iLhs - iRhs) +** +** Otherwise, if this is a DESC iterator, the opposite is returned: +** +** (iRhs - iLhs) +*/ +static int fts5RowidCmp( + Fts5Expr *pExpr, + i64 iLhs, + i64 iRhs +){ + assert( pExpr->bDesc==0 || pExpr->bDesc==1 ); + if( pExpr->bDesc==0 ){ + if( iLhs<iRhs ) return -1; + return (iLhs > iRhs); + }else{ + if( iLhs>iRhs ) return -1; + return (iLhs < iRhs); + } +} + +static void fts5ExprSetEof(Fts5ExprNode *pNode){ + int i; + pNode->bEof = 1; + pNode->bNomatch = 0; + for(i=0; i<pNode->nChild; i++){ + fts5ExprSetEof(pNode->apChild[i]); + } +} + +static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + int i; + for(i=0; i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + pPhrase->poslist.n = 0; + } + }else{ + int i; + for(i=0; i<pNode->nChild; i++){ + fts5ExprNodeZeroPoslist(pNode->apChild[i]); + } + } +} + + + +/* +** Compare the values currently indicated by the two nodes as follows: +** +** res = (*p1) - (*p2) +** +** Nodes that point to values that come later in the iteration order are +** considered to be larger. Nodes at EOF are the largest of all. +** +** This means that if the iteration order is ASC, then numerically larger +** rowids are considered larger. Or if it is the default DESC, numerically +** smaller rowids are larger. +*/ +static int fts5NodeCompare( + Fts5Expr *pExpr, + Fts5ExprNode *p1, + Fts5ExprNode *p2 +){ + if( p2->bEof ) return -1; + if( p1->bEof ) return +1; + return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); +} + +/* +** All individual term iterators in pNear are guaranteed to be valid when +** this function is called. This function checks if all term iterators +** point to the same rowid, and if not, advances them until they do. +** If an EOF is reached before this happens, *pbEof is set to true before +** returning. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprNodeTest_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; + int rc = SQLITE_OK; + i64 iLast; /* Lastest rowid any iterator points to */ + int i, j; /* Phrase and token index, respectively */ + int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; + + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + ); + + /* Initialize iLast, the "lastest" rowid any iterator points to. If the + ** iterator skips through rowids in the default ascending order, this means + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it + ** means the minimum rowid. */ + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); + }else{ + iLast = pLeft->aTerm[0].pIter->iRowid; + } + + do { + bMatch = 1; + for(i=0; i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + for(j=0; j<pPhrase->nTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bNomatch = 0; + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + if( pIter->iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } + } + } + } + }while( bMatch==0 ); + + pNode->iRowid = iLast; + pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); + assert( pNode->bEof==0 || pNode->bNomatch==0 ); + + return rc; +} + +/* +** Advance the first term iterator in the first phrase of pNear. Set output +** variable *pbEof to true if it reaches EOF or if an error occurs. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5ExprNodeNext_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ + int bFromValid, + i64 iFrom +){ + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; + int rc = SQLITE_OK; + + pNode->bNomatch = 0; + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); + + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 ii = p->pIter->iRowid; + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } + if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); + }else{ + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); + } + + if( pNode->bEof==0 ){ + assert( rc==SQLITE_OK ); + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + } + + return rc; +} + + +static int fts5ExprNodeTest_TERM( + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ +){ + /* As this "NEAR" object is actually a single phrase that consists + ** of a single term only, grab pointers into the poslist managed by the + ** fts5_index.c iterator object. This is much faster than synthesizing + ** a new poslist the way we have to for more complicated phrase or NEAR + ** expressions. */ + Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + + assert( pNode->eType==FTS5_TERM ); + assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); + + pPhrase->poslist.n = pIter->nData; + if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pPhrase->poslist.p = (u8*)pIter->pData; + } + pNode->iRowid = pIter->iRowid; + pNode->bNomatch = (pPhrase->poslist.n==0); + return SQLITE_OK; +} + +/* +** xNext() method for a node of type FTS5_TERM. +*/ +static int fts5ExprNodeNext_TERM( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc; + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + + assert( pNode->bEof==0 ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + }else{ + pNode->bEof = 1; + pNode->bNomatch = 0; + } + return rc; +} + +static void fts5ExprNodeTest_OR( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + Fts5ExprNode *pNext = pNode->apChild[0]; + int i; + + for(i=1; i<pNode->nChild; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + int cmp = fts5NodeCompare(pExpr, pNext, pChild); + if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ + pNext = pChild; + } + } + pNode->iRowid = pNext->iRowid; + pNode->bEof = pNext->bEof; + pNode->bNomatch = pNext->bNomatch; +} + +static int fts5ExprNodeNext_OR( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int i; + i64 iLast = pNode->iRowid; + + for(i=0; i<pNode->nChild; i++){ + Fts5ExprNode *p1 = pNode->apChild[i]; + assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); + if( p1->bEof==0 ){ + if( (p1->iRowid==iLast) + || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) + ){ + int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + fts5ExprNodeTest_OR(pExpr, pNode); + return SQLITE_OK; +} + +/* +** Argument pNode is an FTS5_AND node. +*/ +static int fts5ExprNodeTest_AND( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pAnd /* FTS5_AND node to advance */ +){ + int iChild; + i64 iLast = pAnd->iRowid; + int rc = SQLITE_OK; + int bMatch; + + assert( pAnd->bEof==0 ); + do { + pAnd->bNomatch = 0; + bMatch = 1; + for(iChild=0; iChild<pAnd->nChild; iChild++){ + Fts5ExprNode *pChild = pAnd->apChild[iChild]; + int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); + if( cmp>0 ){ + /* Advance pChild until it points to iLast or laster */ + rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); + if( rc!=SQLITE_OK ) return rc; + } + + /* If the child node is now at EOF, so is the parent AND node. Otherwise, + ** the child node is guaranteed to have advanced at least as far as + ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the + ** new lastest rowid seen so far. */ + assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); + if( pChild->bEof ){ + fts5ExprSetEof(pAnd); + bMatch = 1; + break; + }else if( iLast!=pChild->iRowid ){ + bMatch = 0; + iLast = pChild->iRowid; + } + + if( pChild->bNomatch ){ + pAnd->bNomatch = 1; + } + } + }while( bMatch==0 ); + + if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ + fts5ExprNodeZeroPoslist(pAnd); + } + pAnd->iRowid = iLast; + return SQLITE_OK; +} + +static int fts5ExprNodeNext_AND( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_AND(pExpr, pNode); + } + return rc; +} + +static int fts5ExprNodeTest_NOT( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode /* FTS5_NOT node to advance */ +){ + int rc = SQLITE_OK; + Fts5ExprNode *p1 = pNode->apChild[0]; + Fts5ExprNode *p2 = pNode->apChild[1]; + assert( pNode->nChild==2 ); + + while( rc==SQLITE_OK && p1->bEof==0 ){ + int cmp = fts5NodeCompare(pExpr, p1, p2); + if( cmp>0 ){ + rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); + cmp = fts5NodeCompare(pExpr, p1, p2); + } + assert( rc!=SQLITE_OK || cmp<=0 ); + if( cmp || p2->bNomatch ) break; + rc = fts5ExprNodeNext(pExpr, p1, 0, 0); + } + pNode->bEof = p1->bEof; + pNode->bNomatch = p1->bNomatch; + pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); + } + return rc; +} + +static int fts5ExprNodeNext_NOT( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + } + return rc; +} + +/* +** If pNode currently points to a match, this function returns SQLITE_OK +** without modifying it. Otherwise, pNode is advanced until it does point +** to a match or EOF is reached. +*/ +static int fts5ExprNodeTest( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + int rc = SQLITE_OK; + if( pNode->bEof==0 ){ + switch( pNode->eType ){ + + case FTS5_STRING: { + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + break; + } + + case FTS5_TERM: { + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + break; + } + + case FTS5_AND: { + rc = fts5ExprNodeTest_AND(pExpr, pNode); + break; + } + + case FTS5_OR: { + fts5ExprNodeTest_OR(pExpr, pNode); + break; + } + + default: assert( pNode->eType==FTS5_NOT ); { + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + break; + } + } + } + return rc; +} + + +/* +** Set node pNode, which is part of expression pExpr, to point to the first +** match. If there are no matches, set the Node.bEof flag to indicate EOF. +** +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +** It is not an error if there are no matches. +*/ +static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ + int rc = SQLITE_OK; + pNode->bEof = 0; + pNode->bNomatch = 0; + + if( Fts5NodeIsString(pNode) ){ + /* Initialize all term iterators in the NEAR object. */ + rc = fts5ExprNearInitAll(pExpr, pNode); + }else if( pNode->xNext==0 ){ + pNode->bEof = 1; + }else{ + int i; + int nEof = 0; + for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); + assert( pChild->bEof==0 || pChild->bEof==1 ); + nEof += pChild->bEof; + } + pNode->iRowid = pNode->apChild[0]->iRowid; + + switch( pNode->eType ){ + case FTS5_AND: + if( nEof>0 ) fts5ExprSetEof(pNode); + break; + + case FTS5_OR: + if( pNode->nChild==nEof ) fts5ExprSetEof(pNode); + break; + + default: + assert( pNode->eType==FTS5_NOT ); + pNode->bEof = pNode->apChild[0]->bEof; + break; + } + } + + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest(pExpr, pNode); + } + return rc; +} + + +/* +** Begin iterating through the set of documents in index pIdx matched by +** the MATCH expression passed as the first argument. If the "bDesc" +** parameter is passed a non-zero value, iteration is in descending rowid +** order. Or, if it is zero, in ascending order. +** +** If iterating in ascending rowid order (bDesc==0), the first document +** visited is that with the smallest rowid that is larger than or equal +** to parameter iFirst. Or, if iterating in ascending order (bDesc==1), +** then the first document visited must have a rowid smaller than or +** equal to iFirst. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ + Fts5ExprNode *pRoot = p->pRoot; + int rc; /* Return code */ + + p->pIndex = pIdx; + p->bDesc = bDesc; + rc = fts5ExprNodeFirst(p, pRoot); + + /* If not at EOF but the current rowid occurs earlier than iFirst in + ** the iteration order, move to document iFirst or later. */ + if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); + } + + /* If the iterator is not at a real match, skip forward until it is. */ + while( pRoot->bNomatch ){ + assert( pRoot->bEof==0 && rc==SQLITE_OK ); + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + } + return rc; +} + +/* +** Move to the next document +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ + int rc; + Fts5ExprNode *pRoot = p->pRoot; + assert( pRoot->bEof==0 && pRoot->bNomatch==0 ); + do { + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) ); + }while( pRoot->bNomatch ); + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } + return rc; +} + +static int sqlite3Fts5ExprEof(Fts5Expr *p){ + return p->pRoot->bEof; +} + +static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ + return p->pRoot->iRowid; +} + +static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ + int rc = SQLITE_OK; + *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n); + return rc; +} + +/* +** Free the phrase object passed as the only argument. +*/ +static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ + if( pPhrase ){ + int i; + for(i=0; i<pPhrase->nTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + sqlite3_free(pTerm->zTerm); + sqlite3Fts5IterClose(pTerm->pIter); + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + fts5BufferFree((Fts5Buffer*)&pSyn[1]); + sqlite3_free(pSyn); + } + } + if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); + sqlite3_free(pPhrase); + } +} + +/* +** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated +** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is +** appended to it and the results returned. +** +** If an OOM error occurs, both the pNear and pPhrase objects are freed and +** NULL returned. +*/ +static Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNearset *pNear, /* Existing nearset, or NULL */ + Fts5ExprPhrase *pPhrase /* Recently parsed phrase */ +){ + const int SZALLOC = 8; + Fts5ExprNearset *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + if( pPhrase==0 ){ + return pNear; + } + if( pNear==0 ){ + int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + }else if( (pNear->nPhrase % SZALLOC)==0 ){ + int nNew = pNear->nPhrase + SZALLOC; + int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + + pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + } + }else{ + pRet = pNear; + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNearsetFree(pNear); + sqlite3Fts5ParsePhraseFree(pPhrase); + }else{ + if( pRet->nPhrase>0 ){ + Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); + if( pPhrase->nTerm==0 ){ + fts5ExprPhraseFree(pPhrase); + pRet->nPhrase--; + pParse->nPhrase--; + pPhrase = pLast; + }else if( pLast->nTerm==0 ){ + fts5ExprPhraseFree(pLast); + pParse->apPhrase[pParse->nPhrase-2] = pPhrase; + pParse->nPhrase--; + pRet->nPhrase--; + } + } + pRet->apPhrase[pRet->nPhrase++] = pPhrase; + } + return pRet; +} + +typedef struct TokenCtx TokenCtx; +struct TokenCtx { + Fts5ExprPhrase *pPhrase; + int rc; +}; + +/* +** Callback for tokenizing terms used by ParseTerm(). +*/ +static int fts5ParseTokenize( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int rc = SQLITE_OK; + const int SZALLOC = 8; + TokenCtx *pCtx = (TokenCtx*)pContext; + Fts5ExprPhrase *pPhrase = pCtx->pPhrase; + + UNUSED_PARAM2(iUnused1, iUnused2); + + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, nByte); + pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + + pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } + } + + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + } + } + + pCtx->rc = rc; + return rc; +} + + +/* +** Free the phrase object passed as the only argument. +*/ +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){ + fts5ExprPhraseFree(pPhrase); +} + +/* +** Free the phrase object passed as the second argument. +*/ +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){ + if( pNear ){ + int i; + for(i=0; i<pNear->nPhrase; i++){ + fts5ExprPhraseFree(pNear->apPhrase[i]); + } + sqlite3_free(pNear->pColset); + sqlite3_free(pNear); + } +} + +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){ + assert( pParse->pExpr==0 ); + pParse->pExpr = p; +} + +/* +** This function is called by the parser to process a string token. The +** string may or may not be quoted. In any case it is tokenized and a +** phrase object consisting of all tokens returned. +*/ +static Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprPhrase *pAppend, /* Phrase to append to */ + Fts5Token *pToken, /* String to tokenize */ + int bPrefix /* True if there is a trailing "*" */ +){ + Fts5Config *pConfig = pParse->pConfig; + TokenCtx sCtx; /* Context object passed to callback */ + int rc; /* Tokenize return code */ + char *z = 0; + + memset(&sCtx, 0, sizeof(TokenCtx)); + sCtx.pPhrase = pAppend; + + rc = fts5ParseStringFromToken(pToken, &z); + if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int n; + sqlite3Fts5Dequote(z); + n = (int)strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); + } + sqlite3_free(z); + if( rc || (rc = sCtx.rc) ){ + pParse->rc = rc; + fts5ExprPhraseFree(sCtx.pPhrase); + sCtx.pPhrase = 0; + }else{ + + if( pAppend==0 ){ + if( (pParse->nPhrase % 8)==0 ){ + int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); + Fts5ExprPhrase **apNew; + apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte); + if( apNew==0 ){ + pParse->rc = SQLITE_NOMEM; + fts5ExprPhraseFree(sCtx.pPhrase); + return 0; + } + pParse->apPhrase = apNew; + } + pParse->nPhrase++; + } + + if( sCtx.pPhrase==0 ){ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + }else if( sCtx.pPhrase->nTerm ){ + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; + } + pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; + } + + return sCtx.pPhrase; +} + +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +static int sqlite3Fts5ExprClonePhrase( + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + int i; /* Used to iterate through phrase terms */ + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ + + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + + for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), + 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + pNew->pRoot->eType = FTS5_TERM; + pNew->pRoot->xNext = fts5ExprNodeNext_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + pNew->pRoot->xNext = fts5ExprNodeNext_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + +/* +** Token pTok has appeared in a MATCH expression where the NEAR operator +** is expected. If token pTok does not contain "NEAR", store an error +** in the pParse object. +*/ +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){ + if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p + ); + } +} + +static void sqlite3Fts5ParseSetDistance( + Fts5Parse *pParse, + Fts5ExprNearset *pNear, + Fts5Token *p +){ + if( pNear ){ + int nNear = 0; + int i; + if( p->n ){ + for(i=0; i<p->n; i++){ + char c = (char)p->p[i]; + if( c<'0' || c>'9' ){ + sqlite3Fts5ParseError( + pParse, "expected integer, got \"%.*s\"", p->n, p->p + ); + return; + } + nNear = nNear * 10 + (p->p[i] - '0'); + } + }else{ + nNear = FTS5_DEFAULT_NEARDIST; + } + pNear->nNear = nNear; + } +} + +/* +** The second argument passed to this function may be NULL, or it may be +** an existing Fts5Colset object. This function returns a pointer to +** a new colset object containing the contents of (p) with new value column +** number iCol appended. +** +** If an OOM error occurs, store an error code in pParse and return NULL. +** The old colset object (if any) is not freed in this case. +*/ +static Fts5Colset *fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *p, /* Existing colset object */ + int iCol /* New column to add to colset object */ +){ + int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */ + Fts5Colset *pNew; /* New colset object to return */ + + assert( pParse->rc==SQLITE_OK ); + assert( iCol>=0 && iCol<pParse->pConfig->nCol ); + + pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + if( pNew==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + int *aiCol = pNew->aiCol; + int i, j; + for(i=0; i<nCol; i++){ + if( aiCol[i]==iCol ) return pNew; + if( aiCol[i]>iCol ) break; + } + for(j=nCol; j>i; j--){ + aiCol[j] = aiCol[j-1]; + } + aiCol[i] = iCol; + pNew->nCol = nCol+1; + +#ifndef NDEBUG + /* Check that the array is in order and contains no duplicate entries. */ + for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] ); +#endif + } + + return pNew; +} + +static Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *pColset, /* Existing colset object */ + Fts5Token *p +){ + Fts5Colset *pRet = 0; + int iCol; + char *z; /* Dequoted copy of token p */ + + z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n); + if( pParse->rc==SQLITE_OK ){ + Fts5Config *pConfig = pParse->pConfig; + sqlite3Fts5Dequote(z); + for(iCol=0; iCol<pConfig->nCol; iCol++){ + if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break; + } + if( iCol==pConfig->nCol ){ + sqlite3Fts5ParseError(pParse, "no such column: %s", z); + }else{ + pRet = fts5ParseColset(pParse, pColset, iCol); + } + sqlite3_free(z); + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3_free(pColset); + } + + return pRet; +} + +static void sqlite3Fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNearset *pNear, + Fts5Colset *pColset +){ + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pParse->rc = SQLITE_ERROR; + pParse->zErr = sqlite3_mprintf( + "fts5: column queries are not supported (detail=none)" + ); + sqlite3_free(pColset); + return; + } + + if( pNear ){ + pNear->pColset = pColset; + }else{ + sqlite3_free(pColset); + } +} + +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ + switch( pNode->eType ){ + case FTS5_STRING: { + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + ){ + pNode->eType = FTS5_TERM; + pNode->xNext = fts5ExprNodeNext_TERM; + }else{ + pNode->xNext = fts5ExprNodeNext_STRING; + } + break; + }; + + case FTS5_OR: { + pNode->xNext = fts5ExprNodeNext_OR; + break; + }; + + case FTS5_AND: { + pNode->xNext = fts5ExprNodeNext_AND; + break; + }; + + default: assert( pNode->eType==FTS5_NOT ); { + pNode->xNext = fts5ExprNodeNext_NOT; + break; + }; + } +} + +static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ + int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; + memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); + p->nChild += pSub->nChild; + sqlite3_free(pSub); + }else{ + p->apChild[p->nChild++] = pSub; + } +} + +/* +** Allocate and return a new expression object. If anything goes wrong (i.e. +** OOM error), leave an error code in pParse and return NULL. +*/ +static Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, /* Parse context */ + int eType, /* FTS5_STRING, AND, OR or NOT */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight, /* Right hand child expression */ + Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */ +){ + Fts5ExprNode *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + int nChild = 0; /* Number of children of returned node */ + int nByte; /* Bytes of space to allocate for this node */ + + assert( (eType!=FTS5_STRING && !pNear) + || (eType==FTS5_STRING && !pLeft && !pRight) + ); + if( eType==FTS5_STRING && pNear==0 ) return 0; + if( eType!=FTS5_STRING && pLeft==0 ) return pRight; + if( eType!=FTS5_STRING && pRight==0 ) return pLeft; + + if( eType==FTS5_NOT ){ + nChild = 2; + }else if( eType==FTS5_AND || eType==FTS5_OR ){ + nChild = 2; + if( pLeft->eType==eType ) nChild += pLeft->nChild-1; + if( pRight->eType==eType ) nChild += pRight->nChild-1; + } + + nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + + if( pRet ){ + pRet->eType = eType; + pRet->pNear = pNear; + fts5ExprAssignXNext(pRet); + if( eType==FTS5_STRING ){ + int iPhrase; + for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ + pNear->apPhrase[iPhrase]->pNode = pRet; + if( pNear->apPhrase[iPhrase]->nTerm==0 ){ + pRet->xNext = 0; + pRet->eType = FTS5_EOF; + } + } + + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL + && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) + ){ + assert( pParse->rc==SQLITE_OK ); + pParse->rc = SQLITE_ERROR; + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_mprintf( + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3_free(pRet); + pRet = 0; + } + + }else{ + fts5ExprAddChildren(pRet, pLeft); + fts5ExprAddChildren(pRet, pRight); + } + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + sqlite3Fts5ParseNearsetFree(pNear); + } + return pRet; +} + +static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight /* Right hand child expression */ +){ + Fts5ExprNode *pRet = 0; + Fts5ExprNode *pPrev; + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + }else{ + + assert( pLeft->eType==FTS5_STRING + || pLeft->eType==FTS5_TERM + || pLeft->eType==FTS5_EOF + || pLeft->eType==FTS5_AND + ); + assert( pRight->eType==FTS5_STRING + || pRight->eType==FTS5_TERM + || pRight->eType==FTS5_EOF + ); + + if( pLeft->eType==FTS5_AND ){ + pPrev = pLeft->apChild[pLeft->nChild-1]; + }else{ + pPrev = pLeft; + } + assert( pPrev->eType==FTS5_STRING + || pPrev->eType==FTS5_TERM + || pPrev->eType==FTS5_EOF + ); + + if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); + sqlite3Fts5ParseNodeFree(pRight); + pRet = pLeft; + pParse->nPhrase--; + } + else if( pPrev->eType==FTS5_EOF ){ + Fts5ExprPhrase **ap; + + if( pPrev==pLeft ){ + pRet = pRight; + }else{ + pLeft->apChild[pLeft->nChild-1] = pRight; + pRet = pLeft; + } + + ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; + assert( ap[0]==pPrev->pNear->apPhrase[0] ); + memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); + pParse->nPhrase--; + + sqlite3Fts5ParseNodeFree(pPrev); + } + else{ + pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); + } + } + + return pRet; +} + +static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ + int nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc(nByte); + + if( zQuoted ){ + int i = 0; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->zTerm; + zQuoted[i++] = '"'; + while( *zIn ){ + if( *zIn=='"' ) zQuoted[i++] = '"'; + zQuoted[i++] = *zIn++; + } + zQuoted[i++] = '"'; + if( p->pSynonym ) zQuoted[i++] = '|'; + } + if( pTerm->bPrefix ){ + zQuoted[i++] = ' '; + zQuoted[i++] = '*'; + } + zQuoted[i++] = '\0'; + } + return zQuoted; +} + +static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){ + char *zNew; + va_list ap; + va_start(ap, zFmt); + zNew = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zApp && zNew ){ + char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); + sqlite3_free(zNew); + zNew = zNew2; + } + sqlite3_free(zApp); + return zNew; +} + +/* +** Compose a tcl-readable representation of expression pExpr. Return a +** pointer to a buffer containing that representation. It is the +** responsibility of the caller to at some point free the buffer using +** sqlite3_free(). +*/ +static char *fts5ExprPrintTcl( + Fts5Config *pConfig, + const char *zNearsetCmd, + Fts5ExprNode *pExpr +){ + char *zRet = 0; + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd); + if( zRet==0 ) return 0; + if( pNear->pColset ){ + int *aiCol = pNear->pColset->aiCol; + int nCol = pNear->pColset->nCol; + if( nCol==1 ){ + zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]); + }else{ + zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]); + for(i=1; i<pNear->pColset->nCol; i++){ + zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]); + } + zRet = fts5PrintfAppend(zRet, "} "); + } + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear); + if( zRet==0 ) return 0; + } + + zRet = fts5PrintfAppend(zRet, "--"); + if( zRet==0 ) return 0; + + for(i=0; i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + + zRet = fts5PrintfAppend(zRet, " {"); + for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ + char *zTerm = pPhrase->aTerm[iTerm].zTerm; + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } + } + + if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); + if( zRet==0 ) return 0; + } + + }else{ + char const *zOp = 0; + int i; + switch( pExpr->eType ){ + case FTS5_AND: zOp = "AND"; break; + case FTS5_NOT: zOp = "NOT"; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = "OR"; + break; + } + + zRet = sqlite3_mprintf("%s", zOp); + for(i=0; zRet && i<pExpr->nChild; i++){ + char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]); + if( !z ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + zRet = fts5PrintfAppend(zRet, " [%z]", z); + } + } + } + + return zRet; +} + +static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ + char *zRet = 0; + if( pExpr->eType==0 ){ + return sqlite3_mprintf("\"\""); + }else + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + if( pNear->pColset ){ + int iCol = pNear->pColset->aiCol[0]; + zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[iCol]); + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "NEAR("); + if( zRet==0 ) return 0; + } + + for(i=0; i<pNear->nPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( i!=0 ){ + zRet = fts5PrintfAppend(zRet, " "); + if( zRet==0 ) return 0; + } + for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){ + char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]); + if( zTerm ){ + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm); + sqlite3_free(zTerm); + } + if( zTerm==0 || zRet==0 ){ + sqlite3_free(zRet); + return 0; + } + } + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear); + if( zRet==0 ) return 0; + } + + }else{ + char const *zOp = 0; + int i; + + switch( pExpr->eType ){ + case FTS5_AND: zOp = " AND "; break; + case FTS5_NOT: zOp = " NOT "; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = " OR "; + break; + } + + for(i=0; i<pExpr->nChild; i++){ + char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); + if( z==0 ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + int e = pExpr->apChild[i]->eType; + int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); + zRet = fts5PrintfAppend(zRet, "%s%s%z%s", + (i==0 ? "" : zOp), + (b?"(":""), z, (b?")":"") + ); + } + if( zRet==0 ) break; + } + } + + return zRet; +} + +/* +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0) +** and fts5_expr_tcl() (bTcl!=0). +*/ +static void fts5ExprFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal, /* Function arguments */ + int bTcl +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zExpr = 0; + char *zErr = 0; + Fts5Expr *pExpr = 0; + int rc; + int i; + + const char **azConfig; /* Array of arguments for Fts5Config */ + const char *zNearsetCmd = "nearset"; + int nConfig; /* Size of azConfig[] */ + Fts5Config *pConfig = 0; + int iArg = 1; + + if( nArg<1 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function %s", + bTcl ? "fts5_expr_tcl" : "fts5_expr" + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + + if( bTcl && nArg>1 ){ + zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); + iArg = 2; + } + + nConfig = 3 + (nArg-iArg); + azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig); + if( azConfig==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + azConfig[0] = 0; + azConfig[1] = "main"; + azConfig[2] = "tbl"; + for(i=3; iArg<nArg; iArg++){ + azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]); + } + + zExpr = (const char*)sqlite3_value_text(apVal[0]); + + rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); + } + if( rc==SQLITE_OK ){ + char *zText; + if( pExpr->pRoot->xNext==0 ){ + zText = sqlite3_mprintf(""); + }else if( bTcl ){ + zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); + }else{ + zText = fts5ExprPrint(pConfig, pExpr->pRoot); + } + if( zText==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + } + + if( rc!=SQLITE_OK ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free((void *)azConfig); + sqlite3Fts5ConfigFree(pConfig); + sqlite3Fts5ExprFree(pExpr); +} + +static void fts5ExprFunctionHr( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 0); +} +static void fts5ExprFunctionTcl( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 1); +} + +/* +** The implementation of an SQLite user-defined-function that accepts a +** single integer as an argument. If the integer is an alpha-numeric +** unicode code point, 1 is returned. Otherwise 0. +*/ +static void fts5ExprIsAlnum( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + int iCode; + if( nArg!=1 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_isalnum", -1 + ); + return; + } + iCode = sqlite3_value_int(apVal[0]); + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode)); +} + +static void fts5ExprFold( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_fold", -1 + ); + }else{ + int iCode; + int bRemoveDiacritics = 0; + iCode = sqlite3_value_int(apVal[0]); + if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]); + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); + } +} + +/* +** This is called during initialization to register the fts5_expr() scalar +** UDF with the SQLite handle passed as the only argument. +*/ +static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ + struct Fts5ExprFunc { + const char *z; + void (*x)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "fts5_expr", fts5ExprFunctionHr }, + { "fts5_expr_tcl", fts5ExprFunctionTcl }, + { "fts5_isalnum", fts5ExprIsAlnum }, + { "fts5_fold", fts5ExprFold }, + }; + int i; + int rc = SQLITE_OK; + void *pCtx = (void*)pGlobal; + + for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){ + struct Fts5ExprFunc *p = &aFunc[i]; + rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); + } + + /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ +#ifndef NDEBUG + (void)sqlite3Fts5ParserTrace; +#endif + + return rc; +} + +/* +** Return the number of phrases in expression pExpr. +*/ +static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){ + return (pExpr ? pExpr->nPhrase : 0); +} + +/* +** Return the number of terms in the iPhrase'th phrase in pExpr. +*/ +static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){ + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0; + return pExpr->apExprPhrase[iPhrase]->nTerm; +} + +/* +** This function is used to access the current position list for phrase +** iPhrase. +*/ +static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ + int nRet; + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ + *pa = pPhrase->poslist.p; + nRet = pPhrase->poslist.n; + }else{ + *pa = 0; + nRet = 0; + } + return nRet; +} + +struct Fts5PoslistPopulator { + Fts5PoslistWriter writer; + int bOk; /* True if ok to populate */ + int bMiss; +}; + +static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ + Fts5PoslistPopulator *pRet; + pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + if( pRet ){ + int i; + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + for(i=0; i<pExpr->nPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + if( bLive && + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) + ){ + pRet[i].bMiss = 1; + }else{ + pBuf->n = 0; + } + } + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistPopulator *aPopulator; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +/* +** TODO: Make this more efficient! +*/ +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; i<pColset->nCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static int fts5ExprPopulatePoslistsCb( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Byte offset of token within input text */ + int iUnused2 /* Byte offset of end of token within input text */ +){ + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; + Fts5Expr *pExpr = p->pExpr; + int i; + + UNUSED_PARAM2(iUnused1, iUnused2); + + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; i<pExpr->nPhrase; i++){ + Fts5ExprTerm *pTerm; + if( p->aPopulator[i].bOk==0 ) continue; + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + int nTerm = (int)strlen(pTerm->zTerm); + if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) + && memcmp(pTerm->zTerm, pToken, nTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff + ); + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +static int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistPopulator *aPopulator, + int iCol, + const char *z, int n +){ + int i; + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aPopulator = aPopulator; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + for(i=0; i<pExpr->nPhrase; i++){ + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + Fts5Colset *pColset = pNode->pNear->pColset; + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + || aPopulator[i].bMiss + ){ + aPopulator[i].bOk = 0; + }else{ + aPopulator[i].bOk = 1; + } + } + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; i<pNode->nChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; i<pNode->nChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + break; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; i<pNode->nChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + break; + } + } + return 1; +} + +static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + +static void fts5ExprClearEof(Fts5ExprNode *pNode){ + int i; + for(i=0; i<pNode->nChild; i++){ + fts5ExprClearEof(pNode->apChild[i]); + } + pNode->bEof = 0; +} +static void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ + fts5ExprClearEof(pExpr->pRoot); +} + +/* +** This function is only called for detail=columns tables. +*/ +static int sqlite3Fts5ExprPhraseCollist( + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, + int *pnCollist +){ + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + int rc = SQLITE_OK; + + assert( iPhrase>=0 && iPhrase<pExpr->nPhrase ); + assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid + && pPhrase->poslist.n>0 + ){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; + rc = fts5ExprSynonymList( + pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist + ); + }else{ + *ppCollist = pPhrase->aTerm[0].pIter->pData; + *pnCollist = pPhrase->aTerm[0].pIter->nData; + } + }else{ + *ppCollist = 0; + *pnCollist = 0; + } + + return rc; +} + + +/* +** 2014 August 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +/* #include "fts5Int.h" */ + +typedef struct Fts5HashEntry Fts5HashEntry; + +/* +** This file contains the implementation of an in-memory hash table used +** to accumuluate "term -> doclist" content before it is flused to a level-0 +** segment. +*/ + + +struct Fts5Hash { + int eDetail; /* Copy of Fts5Config.eDetail */ + int *pnByte; /* Pointer to bytes counter */ + int nEntry; /* Number of entries currently in hash */ + int nSlot; /* Size of aSlot[] array */ + Fts5HashEntry *pScan; /* Current ordered scan item */ + Fts5HashEntry **aSlot; /* Array of hash slots */ +}; + +/* +** Each entry in the hash table is represented by an object of the +** following type. Each object, its key (zKey[]) and its current data +** are stored in a single memory allocation. The position list data +** immediately follows the key data in memory. +** +** The data that follows the key is in a similar, but not identical format +** to the doclist data stored in the database. It is: +** +** * Rowid, as a varint +** * Position list, without 0x00 terminator. +** * Size of previous position list and rowid, as a 4 byte +** big-endian integer. +** +** iRowidOff: +** Offset of last rowid written to data area. Relative to first byte of +** structure. +** +** nData: +** Bytes of data written since iRowidOff. +*/ +struct Fts5HashEntry { + Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ + Fts5HashEntry *pScanNext; /* Next entry in sorted order */ + + int nAlloc; /* Total size of allocation */ + int iSzPoslist; /* Offset of space for 4-byte poslist size */ + int nData; /* Total bytes of data (incl. structure) */ + int nKey; /* Length of zKey[] in bytes */ + u8 bDel; /* Set delete-flag @ iSzPoslist */ + u8 bContent; /* Set content-flag (detail=none mode) */ + i16 iCol; /* Column of last value written */ + int iPos; /* Position of last value written */ + i64 iRowid; /* Rowid of last value written */ + char zKey[8]; /* Nul-terminated entry key */ +}; + +/* +** Size of Fts5HashEntry without the zKey[] array. +*/ +#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) + + + +/* +** Allocate a new hash table. +*/ +static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ + int rc = SQLITE_OK; + Fts5Hash *pNew; + + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nByte; + memset(pNew, 0, sizeof(Fts5Hash)); + pNew->pnByte = pnByte; + pNew->eDetail = pConfig->eDetail; + + pNew->nSlot = 1024; + nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; + pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); + if( pNew->aSlot==0 ){ + sqlite3_free(pNew); + *ppNew = 0; + rc = SQLITE_NOMEM; + }else{ + memset(pNew->aSlot, 0, nByte); + } + } + return rc; +} + +/* +** Free a hash table object. +*/ +static void sqlite3Fts5HashFree(Fts5Hash *pHash){ + if( pHash ){ + sqlite3Fts5HashClear(pHash); + sqlite3_free(pHash->aSlot); + sqlite3_free(pHash); + } +} + +/* +** Empty (but do not delete) a hash table. +*/ +static void sqlite3Fts5HashClear(Fts5Hash *pHash){ + int i; + for(i=0; i<pHash->nSlot; i++){ + Fts5HashEntry *pNext; + Fts5HashEntry *pSlot; + for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){ + pNext = pSlot->pHashNext; + sqlite3_free(pSlot); + } + } + memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); + pHash->nEntry = 0; +} + +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + return (h % nSlot); +} + +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + h = (h << 3) ^ h ^ b; + return (h % nSlot); +} + +/* +** Resize the hash table by doubling the number of slots. +*/ +static int fts5HashResize(Fts5Hash *pHash){ + int nNew = pHash->nSlot*2; + int i; + Fts5HashEntry **apNew; + Fts5HashEntry **apOld = pHash->aSlot; + + apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*)); + if( !apNew ) return SQLITE_NOMEM; + memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); + + for(i=0; i<pHash->nSlot; i++){ + while( apOld[i] ){ + int iHash; + Fts5HashEntry *p = apOld[i]; + apOld[i] = p->pHashNext; + iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey)); + p->pHashNext = apNew[iHash]; + apNew[iHash] = p; + } + } + + sqlite3_free(apOld); + pHash->nSlot = nNew; + pHash->aSlot = apNew; + return SQLITE_OK; +} + +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ + if( p->iSzPoslist ){ + u8 *pPtr = (u8*)p; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + assert( p->nData==p->iSzPoslist ); + if( p->bDel ){ + pPtr[p->nData++] = 0x00; + if( p->bContent ){ + pPtr[p->nData++] = 0x00; + } + } + }else{ + int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ + + assert( p->bDel==0 || p->bDel==1 ); + if( nPos<=127 ){ + pPtr[p->iSzPoslist] = (u8)nPos; + }else{ + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); + p->nData += (nByte-1); + } + } + + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; + } +} + +/* +** Add an entry to the in-memory hash table. The key is the concatenation +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). +** +** (bByte || pToken) -> (iRowid,iCol,iPos) +** +** Or, if iCol is negative, then the value is a delete marker. +*/ +static int sqlite3Fts5HashWrite( + Fts5Hash *pHash, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, /* First byte of token */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + unsigned int iHash; + Fts5HashEntry *p; + u8 *pPtr; + int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); + + /* Attempt to locate an existing hash entry */ + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + if( p->zKey[0]==bByte + && p->nKey==nToken + && memcmp(&p->zKey[1], pToken, nToken)==0 + ){ + break; + } + } + + /* If an existing hash entry cannot be found, create a new one. */ + if( p==0 ){ + /* Figure out how much space to allocate */ + int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; + if( nByte<128 ) nByte = 128; + + /* Grow the Fts5Hash.aSlot[] array if necessary. */ + if( (pHash->nEntry*2)>=pHash->nSlot ){ + int rc = fts5HashResize(pHash); + if( rc!=SQLITE_OK ) return rc; + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + } + + /* Allocate new Fts5HashEntry and add it to the hash table. */ + p = (Fts5HashEntry*)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, FTS5_HASHENTRYSIZE); + p->nAlloc = nByte; + p->zKey[0] = bByte; + memcpy(&p->zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); + p->nKey = nToken; + p->zKey[nToken+1] = '\0'; + p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; + p->pHashNext = pHash->aSlot[iHash]; + pHash->aSlot[iHash] = p; + pHash->nEntry++; + + /* Add the first rowid field to the hash-entry */ + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); + p->iRowid = iRowid; + + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + } + + nIncr += p->nData; + }else{ + + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario + ** is: + ** + ** + 9 bytes for a new rowid, + ** + 4 byte reserved for the "poslist size" varint. + ** + 1 byte for a "new column" byte, + ** + 3 bytes for a new column number (16-bit max) as a varint, + ** + 5 bytes for the new position offset (32-bit max). + */ + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ + int nNew = p->nAlloc * 2; + Fts5HashEntry *pNew; + Fts5HashEntry **pp; + pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->nAlloc = nNew; + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); + *pp = pNew; + p = pNew; + } + nIncr -= p->nData; + } + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); + + pPtr = (u8*)p; + + /* If this is a new rowid, append the 4-byte size field for the previous + ** entry, and the new rowid for this entry. */ + if( iRowid!=p->iRowid ){ + fts5HashAddPoslistSize(pHash, p); + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); + p->iRowid = iRowid; + bNew = 1; + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + p->iPos = 0; + } + } + + if( iCol>=0 ){ + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + p->bContent = 1; + }else{ + /* Append a new column value, if necessary */ + assert( iCol>=p->iCol ); + if( iCol!=p->iCol ){ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = iCol; + p->iPos = 0; + }else{ + bNew = 1; + p->iCol = iPos = iCol; + } + } + + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } + } + }else{ + /* This is a delete. Set the delete flag. */ + p->bDel = 1; + } + + nIncr += p->nData; + *pHash->pnByte += nIncr; + return SQLITE_OK; +} + + +/* +** Arguments pLeft and pRight point to linked-lists of hash-entry objects, +** each sorted in key order. This function merges the two lists into a +** single list and returns a pointer to its first element. +*/ +static Fts5HashEntry *fts5HashEntryMerge( + Fts5HashEntry *pLeft, + Fts5HashEntry *pRight +){ + Fts5HashEntry *p1 = pLeft; + Fts5HashEntry *p2 = pRight; + Fts5HashEntry *pRet = 0; + Fts5HashEntry **ppOut = &pRet; + + while( p1 || p2 ){ + if( p1==0 ){ + *ppOut = p2; + p2 = 0; + }else if( p2==0 ){ + *ppOut = p1; + p1 = 0; + }else{ + int i = 0; + while( p1->zKey[i]==p2->zKey[i] ) i++; + + if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){ + /* p2 is smaller */ + *ppOut = p2; + ppOut = &p2->pScanNext; + p2 = p2->pScanNext; + }else{ + /* p1 is smaller */ + *ppOut = p1; + ppOut = &p1->pScanNext; + p1 = p1->pScanNext; + } + *ppOut = 0; + } + } + + return pRet; +} + +/* +** Extract all tokens from hash table iHash and link them into a list +** in sorted order. The hash table is cleared before returning. It is +** the responsibility of the caller to free the elements of the returned +** list. +*/ +static int fts5HashEntrySort( + Fts5Hash *pHash, + const char *pTerm, int nTerm, /* Query prefix, if any */ + Fts5HashEntry **ppSorted +){ + const int nMergeSlot = 32; + Fts5HashEntry **ap; + Fts5HashEntry *pList; + int iSlot; + int i; + + *ppSorted = 0; + ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot); + if( !ap ) return SQLITE_NOMEM; + memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); + + for(iSlot=0; iSlot<pHash->nSlot; iSlot++){ + Fts5HashEntry *pIter; + for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ + if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){ + Fts5HashEntry *pEntry = pIter; + pEntry->pScanNext = 0; + for(i=0; ap[i]; i++){ + pEntry = fts5HashEntryMerge(pEntry, ap[i]); + ap[i] = 0; + } + ap[i] = pEntry; + } + } + } + + pList = 0; + for(i=0; i<nMergeSlot; i++){ + pList = fts5HashEntryMerge(pList, ap[i]); + } + + pHash->nEntry = 0; + sqlite3_free(ap); + *ppSorted = pList; + return SQLITE_OK; +} + +/* +** Query the hash table for a doclist associated with term pTerm/nTerm. +*/ +static int sqlite3Fts5HashQuery( + Fts5Hash *pHash, /* Hash table to query */ + const char *pTerm, int nTerm, /* Query term */ + const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +){ + unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); + Fts5HashEntry *p; + + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; + } + + if( p ){ + fts5HashAddPoslistSize(pHash, p); + *ppDoclist = (const u8*)&p->zKey[nTerm+1]; + *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + }else{ + *ppDoclist = 0; + *pnDoclist = 0; + } + + return SQLITE_OK; +} + +static int sqlite3Fts5HashScanInit( + Fts5Hash *p, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +){ + return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); +} + +static void sqlite3Fts5HashScanNext(Fts5Hash *p){ + assert( !sqlite3Fts5HashScanEof(p) ); + p->pScan = p->pScan->pScanNext; +} + +static int sqlite3Fts5HashScanEof(Fts5Hash *p){ + return (p->pScan==0); +} + +static void sqlite3Fts5HashScanEntry( + Fts5Hash *pHash, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +){ + Fts5HashEntry *p; + if( (p = pHash->pScan) ){ + int nTerm = (int)strlen(p->zKey); + fts5HashAddPoslistSize(pHash, p); + *pzTerm = p->zKey; + *ppDoclist = (const u8*)&p->zKey[nTerm+1]; + *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + }else{ + *pzTerm = 0; + *ppDoclist = 0; + *pnDoclist = 0; + } +} + + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Low level access to the FTS index stored in the database file. The +** routines in this file file implement all read and write access to the +** %_data table. Other parts of the system access this functionality via +** the interface defined in fts5Int.h. +*/ + + +/* #include "fts5Int.h" */ + +/* +** Overview: +** +** The %_data table contains all the FTS indexes for an FTS5 virtual table. +** As well as the main term index, there may be up to 31 prefix indexes. +** The format is similar to FTS3/4, except that: +** +** * all segment b-tree leaf data is stored in fixed size page records +** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is +** taken to ensure it is possible to iterate in either direction through +** the entries in a doclist, or to seek to a specific entry within a +** doclist, without loading it into memory. +** +** * large doclists that span many pages have associated "doclist index" +** records that contain a copy of the first rowid on each page spanned by +** the doclist. This is used to speed up seek operations, and merges of +** large doclists with very small doclists. +** +** * extra fields in the "structure record" record the state of ongoing +** incremental merge operations. +** +*/ + + +#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ +#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ + +#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ + +#define FTS5_MAIN_PREFIX '0' + +#if FTS5_MAX_PREFIX_INDEXES > 31 +# error "FTS5_MAX_PREFIX_INDEXES is too large" +#endif + +/* +** Details: +** +** The %_data table managed by this module, +** +** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); +** +** , contains the following 5 types of records. See the comments surrounding +** the FTS5_*_ROWID macros below for a description of how %_data rowids are +** assigned to each fo them. +** +** 1. Structure Records: +** +** The set of segments that make up an index - the index structure - are +** recorded in a single record within the %_data table. The record consists +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. If the FTS table features more than one index (because +** there are one or more prefix indexes), it is guaranteed that all share +** the same cookie value. +** +** Immediately following the configuration cookie, the record begins with +** three varints: +** +** + number of levels, +** + total number of segments on all levels, +** + value of write counter. +** +** Then, for each level from 0 to nMax: +** +** + number of input segments in ongoing merge. +** + total number of segments in level. +** + for each segment from oldest to newest: +** + segment id (always > 0) +** + first leaf page number (often 1, always greater than 0) +** + final leaf page number +** +** 2. The Averages Record: +** +** A single record within the %_data table. The data is a list of varints. +** The first value is the number of rows in the index. Then, for each column +** from left to right, the total number of tokens in the column for all +** rows of the table. +** +** 3. Segment leaves: +** +** TERM/DOCLIST FORMAT: +** +** Most of each segment leaf is taken up by term/doclist data. The +** general format of term/doclist, starting with the first term +** on the leaf page, is: +** +** varint : size of first term +** blob: first term data +** doclist: first doclist +** zero-or-more { +** varint: number of bytes in common with previous term +** varint: number of bytes of new term data (nNew) +** blob: nNew bytes of new term data +** doclist: next doclist +** } +** +** doclist format: +** +** varint: first rowid +** poslist: first poslist +** zero-or-more { +** varint: rowid delta (always > 0) +** poslist: next poslist +** } +** +** poslist format: +** +** varint: size of poslist in bytes multiplied by 2, not including +** this field. Plus 1 if this entry carries the "delete" flag. +** collist: collist for column 0 +** zero-or-more { +** 0x01 byte +** varint: column number (I) +** collist: collist for column I +** } +** +** collist format: +** +** varint: first offset + 2 +** zero-or-more { +** varint: offset delta + 2 +** } +** +** PAGE FORMAT +** +** Each leaf page begins with a 4-byte header containing 2 16-bit +** unsigned integer fields in big-endian format. They are: +** +** * The byte offset of the first rowid on the page, if it exists +** and occurs before the first term (otherwise 0). +** +** * The byte offset of the start of the page footer. If the page +** footer is 0 bytes in size, then this field is the same as the +** size of the leaf page in bytes. +** +** The page footer consists of a single varint for each term located +** on the page. Each varint is the byte offset of the current term +** within the page, delta-compressed against the previous value. In +** other words, the first varint in the footer is the byte offset of +** the first term, the second is the byte offset of the second less that +** of the first, and so on. +** +** The term/doclist format described above is accurate if the entire +** term/doclist data fits on a single leaf page. If this is not the case, +** the format is changed in two ways: +** +** + if the first rowid on a page occurs before the first term, it +** is stored as a literal value: +** +** varint: first rowid +** +** + the first term on each page is stored in the same way as the +** very first term of the segment: +** +** varint : size of first term +** blob: first term data +** +** 5. Segment doclist indexes: +** +** Doclist indexes are themselves b-trees, however they usually consist of +** a single leaf record only. The format of each doclist index leaf page +** is: +** +** * Flags byte. Bits are: +** 0x01: Clear if leaf is also the root page, otherwise set. +** +** * Page number of fts index leaf page. As a varint. +** +** * First rowid on page indicated by previous field. As a varint. +** +** * A list of varints, one for each subsequent termless page. A +** positive delta if the termless page contains at least one rowid, +** or an 0x00 byte otherwise. +** +** Internal doclist index nodes are: +** +** * Flags byte. Bits are: +** 0x01: Clear for root page, otherwise set. +** +** * Page number of first child page. As a varint. +** +** * Copy of first rowid on page indicated by previous field. As a varint. +** +** * A list of delta-encoded varints - the first rowid on each subsequent +** child page. +** +*/ + +/* +** Rowids for the averages and structure records in the %_data table. +*/ +#define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ + +/* +** Macros determining the rowids used by segment leaves and dlidx leaves +** and nodes. All nodes and leaves are stored in the %_data table with large +** positive rowids. +** +** Each segment has a unique non-zero 16-bit id. +** +** The rowid for each segment leaf is found by passing the segment id and +** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered +** sequentially starting from 1. +*/ +#define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ +#define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */ +#define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */ +#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ + +#define fts5_dri(segid, dlidx, height, pgno) ( \ + ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \ + ((i64)(dlidx) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ + ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ + ((i64)(pgno)) \ +) + +#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) +#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) + +/* +** Maximum segments permitted in a single index +*/ +#define FTS5_MAX_SEGMENT 2000 + +#ifdef SQLITE_DEBUG +static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } +#endif + + +/* +** Each time a blob is read from the %_data table, it is padded with this +** many zero bytes. This makes it easier to decode the various record formats +** without overreading if the records are corrupt. +*/ +#define FTS5_DATA_ZERO_PADDING 8 +#define FTS5_DATA_PADDING 20 + +typedef struct Fts5Data Fts5Data; +typedef struct Fts5DlidxIter Fts5DlidxIter; +typedef struct Fts5DlidxLvl Fts5DlidxLvl; +typedef struct Fts5DlidxWriter Fts5DlidxWriter; +typedef struct Fts5Iter Fts5Iter; +typedef struct Fts5PageWriter Fts5PageWriter; +typedef struct Fts5SegIter Fts5SegIter; +typedef struct Fts5DoclistIter Fts5DoclistIter; +typedef struct Fts5SegWriter Fts5SegWriter; +typedef struct Fts5Structure Fts5Structure; +typedef struct Fts5StructureLevel Fts5StructureLevel; +typedef struct Fts5StructureSegment Fts5StructureSegment; + +struct Fts5Data { + u8 *p; /* Pointer to buffer containing record */ + int nn; /* Size of record in bytes */ + int szLeaf; /* Size of leaf without page-index */ +}; + +/* +** One object per %_data table. +*/ +struct Fts5Index { + Fts5Config *pConfig; /* Virtual table configuration */ + char *zDataTbl; /* Name of %_data table */ + int nWorkUnit; /* Leaf pages in a "unit" of work */ + + /* + ** Variables related to the accumulation of tokens and doclists within the + ** in-memory hash tables before they are flushed to disk. + */ + Fts5Hash *pHash; /* Hash table for in-memory data */ + int nPendingData; /* Current bytes of pending data */ + i64 iWriteRowid; /* Rowid for current doc being written */ + int bDelete; /* Current write is a delete */ + + /* Error state. */ + int rc; /* Current error code */ + + /* State used by the fts5DataXXX() functions. */ + sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ + sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ + sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxSelect; + int nRead; /* Total number of blocks read */ + + sqlite3_stmt *pDataVersion; + i64 iStructVersion; /* data_version when pStruct read */ + Fts5Structure *pStruct; /* Current db structure (or NULL) */ +}; + +struct Fts5DoclistIter { + u8 *aEof; /* Pointer to 1 byte past end of doclist */ + + /* Output variables. aPoslist==0 at EOF */ + i64 iRowid; + u8 *aPoslist; + int nPoslist; + int nSize; +}; + +/* +** The contents of the "structure" record for each index are represented +** using an Fts5Structure record in memory. Which uses instances of the +** other Fts5StructureXXX types as components. +*/ +struct Fts5StructureSegment { + int iSegid; /* Segment id */ + int pgnoFirst; /* First leaf page number in segment */ + int pgnoLast; /* Last leaf page number in segment */ +}; +struct Fts5StructureLevel { + int nMerge; /* Number of segments in incr-merge */ + int nSeg; /* Total number of segments on level */ + Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ +}; +struct Fts5Structure { + int nRef; /* Object reference count */ + u64 nWriteCounter; /* Total leaves written to level 0 */ + int nSegment; /* Total segments in this structure */ + int nLevel; /* Number of levels in this index */ + Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ +}; + +/* +** An object of type Fts5SegWriter is used to write to segments. +*/ +struct Fts5PageWriter { + int pgno; /* Page number for this page */ + int iPrevPgidx; /* Previous value written into pgidx */ + Fts5Buffer buf; /* Buffer containing leaf data */ + Fts5Buffer pgidx; /* Buffer containing page-index */ + Fts5Buffer term; /* Buffer containing previous term on page */ +}; +struct Fts5DlidxWriter { + int pgno; /* Page number for this page */ + int bPrevValid; /* True if iPrev is valid */ + i64 iPrev; /* Previous rowid value written to page */ + Fts5Buffer buf; /* Buffer containing page data */ +}; +struct Fts5SegWriter { + int iSegid; /* Segid to write to */ + Fts5PageWriter writer; /* PageWriter object */ + i64 iPrevRowid; /* Previous rowid written to current leaf */ + u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ + u8 bFirstRowidInPage; /* True if next rowid is first in page */ + /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */ + u8 bFirstTermInPage; /* True if next term will be first in leaf */ + int nLeafWritten; /* Number of leaf pages written */ + int nEmpty; /* Number of contiguous term-less nodes */ + + int nDlidx; /* Allocated size of aDlidx[] array */ + Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ + + /* Values to insert into the %_idx table */ + Fts5Buffer btterm; /* Next term to insert into %_idx table */ + int iBtPage; /* Page number corresponding to btterm */ +}; + +typedef struct Fts5CResult Fts5CResult; +struct Fts5CResult { + u16 iFirst; /* aSeg[] index of firstest iterator */ + u8 bTermEq; /* True if the terms are equal */ +}; + +/* +** Object for iterating through a single segment, visiting each term/rowid +** pair in the segment. +** +** pSeg: +** The segment to iterate through. +** +** iLeafPgno: +** Current leaf page number within segment. +** +** iLeafOffset: +** Byte offset within the current leaf that is the first byte of the +** position list data (one byte passed the position-list size field). +** rowid field of the current entry. Usually this is the size field of the +** position list data. The exception is if the rowid for the current entry +** is the last thing on the leaf page. +** +** pLeaf: +** Buffer containing current leaf page data. Set to NULL at EOF. +** +** iTermLeafPgno, iTermLeafOffset: +** Leaf page number containing the last term read from the segment. And +** the offset immediately following the term data. +** +** flags: +** Mask of FTS5_SEGITER_XXX values. Interpreted as follows: +** +** FTS5_SEGITER_ONETERM: +** If set, set the iterator to point to EOF after the current doclist +** has been exhausted. Do not proceed to the next term in the segment. +** +** FTS5_SEGITER_REVERSE: +** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If +** it is set, iterate through rowid in descending order instead of the +** default ascending order. +** +** iRowidOffset/nRowidOffset/aRowidOffset: +** These are used if the FTS5_SEGITER_REVERSE flag is set. +** +** For each rowid on the page corresponding to the current term, the +** corresponding aRowidOffset[] entry is set to the byte offset of the +** start of the "position-list-size" field within the page. +** +** iTermIdx: +** Index of current term on iTermLeafPgno. +*/ +struct Fts5SegIter { + Fts5StructureSegment *pSeg; /* Segment to iterate through */ + int flags; /* Mask of configuration flags */ + int iLeafPgno; /* Current leaf page number */ + Fts5Data *pLeaf; /* Current leaf data */ + Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ + int iLeafOffset; /* Byte offset within current leaf */ + + /* Next method */ + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); + + /* The page and offset from which the current term was read. The offset + ** is the offset of the first rowid in the current doclist. */ + int iTermLeafPgno; + int iTermLeafOffset; + + int iPgidxOff; /* Next offset in pgidx */ + int iEndofDoclist; + + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ + int iRowidOffset; /* Current entry in aRowidOffset[] */ + int nRowidOffset; /* Allocated size of aRowidOffset[] array */ + int *aRowidOffset; /* Array of offset to rowid fields */ + + Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ + + /* Variables populated based on current entry. */ + Fts5Buffer term; /* Current term */ + i64 iRowid; /* Current rowid */ + int nPos; /* Number of bytes in current position list */ + u8 bDel; /* True if the delete flag is set */ +}; + +/* +** Argument is a pointer to an Fts5Data structure that contains a +** leaf page. +*/ +#define ASSERT_SZLEAF_OK(x) assert( \ + (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ +) + +#define FTS5_SEGITER_ONETERM 0x01 +#define FTS5_SEGITER_REVERSE 0x02 + +/* +** Argument is a pointer to an Fts5Data structure that contains a leaf +** page. This macro evaluates to true if the leaf contains no terms, or +** false if it contains at least one term. +*/ +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) + +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) + +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) + +/* +** Object for iterating through the merged results of one or more segments, +** visiting each term/rowid pair in the merged data. +** +** nSeg is always a power of two greater than or equal to the number of +** segments that this object is merging data from. Both the aSeg[] and +** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded +** with zeroed objects - these are handled as if they were iterators opened +** on empty segments. +** +** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the +** comparison in this context is the index of the iterator that currently +** points to the smaller term/rowid combination. Iterators at EOF are +** considered to be greater than all other iterators. +** +** aFirst[1] contains the index in aSeg[] of the iterator that points to +** the smallest key overall. aFirst[0] is unused. +** +** poslist: +** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. +** There is no way to tell if this is populated or not. +*/ +struct Fts5Iter { + Fts5IndexIter base; /* Base class containing output vars */ + + Fts5Index *pIndex; /* Index that owns this iterator */ + Fts5Structure *pStruct; /* Database structure for this iterator */ + Fts5Buffer poslist; /* Buffer containing current poslist */ + Fts5Colset *pColset; /* Restrict matches to these columns */ + + /* Invoked to set output variables. */ + void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); + + int nSeg; /* Size of aSeg[] array */ + int bRev; /* True to iterate in reverse order */ + u8 bSkipEmpty; /* True to skip deleted entries */ + + i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ + Fts5CResult *aFirst; /* Current merge state (see above) */ + Fts5SegIter aSeg[1]; /* Array of segment iterators */ +}; + + +/* +** An instance of the following type is used to iterate through the contents +** of a doclist-index record. +** +** pData: +** Record containing the doclist-index data. +** +** bEof: +** Set to true once iterator has reached EOF. +** +** iOff: +** Set to the current offset within record pData. +*/ +struct Fts5DlidxLvl { + Fts5Data *pData; /* Data for current page of this level */ + int iOff; /* Current offset into pData */ + int bEof; /* At EOF already */ + int iFirstOff; /* Used by reverse iterators */ + + /* Output variables */ + int iLeafPgno; /* Page number of current leaf page */ + i64 iRowid; /* First rowid on leaf iLeafPgno */ +}; +struct Fts5DlidxIter { + int nLvl; + int iSegid; + Fts5DlidxLvl aLvl[1]; +}; + +static void fts5PutU16(u8 *aOut, u16 iVal){ + aOut[0] = (iVal>>8); + aOut[1] = (iVal&0xFF); +} + +static u16 fts5GetU16(const u8 *aIn){ + return ((u16)aIn[0] << 8) + aIn[1]; +} + +/* +** Allocate and return a buffer at least nByte bytes in size. +** +** If an OOM error is encountered, return NULL and set the error code in +** the Fts5Index handle passed as the first argument. +*/ +static void *fts5IdxMalloc(Fts5Index *p, int nByte){ + return sqlite3Fts5MallocZero(&p->rc, nByte); +} + +/* +** Compare the contents of the pLeft buffer with the pRight/nRight blob. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +#ifdef SQLITE_DEBUG +static int fts5BufferCompareBlob( + Fts5Buffer *pLeft, /* Left hand side of comparison */ + const u8 *pRight, int nRight /* Right hand side of comparison */ +){ + int nCmp = MIN(pLeft->n, nRight); + int res = memcmp(pLeft->p, pRight, nCmp); + return (res==0 ? (pLeft->n - nRight) : res); +} +#endif + +/* +** Compare the contents of the two buffers using memcmp(). If one buffer +** is a prefix of the other, it is considered the lesser. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ + int nCmp = MIN(pLeft->n, pRight->n); + int res = memcmp(pLeft->p, pRight->p, nCmp); + return (res==0 ? (pLeft->n - pRight->n) : res); +} + +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ + int ret; + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); + return ret; +} + +/* +** Close the read-only blob handle, if it is open. +*/ +static void fts5CloseReader(Fts5Index *p){ + if( p->pReader ){ + sqlite3_blob *pReader = p->pReader; + p->pReader = 0; + sqlite3_blob_close(pReader); + } +} + + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = 0; + if( p->rc==SQLITE_OK ){ + int rc = SQLITE_OK; + + if( p->pReader ){ + /* This call may return SQLITE_ABORT if there has been a savepoint + ** rollback since it was last used. In this case a new blob handle + ** is required. */ + sqlite3_blob *pBlob = p->pReader; + p->pReader = 0; + rc = sqlite3_blob_reopen(pBlob, iRowid); + assert( p->pReader==0 ); + p->pReader = pBlob; + if( rc!=SQLITE_OK ){ + fts5CloseReader(p); + } + if( rc==SQLITE_ABORT ) rc = SQLITE_OK; + } + + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ + if( p->pReader==0 && rc==SQLITE_OK ){ + Fts5Config *pConfig = p->pConfig; + rc = sqlite3_blob_open(pConfig->db, + pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader + ); + } + + /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls + ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. + ** All the reasons those functions might return SQLITE_ERROR - missing + ** table, missing row, non-blob/text in block column - indicate + ** backing store corruption. */ + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + + if( rc==SQLITE_OK ){ + u8 *aOut = 0; /* Read blob data into this buffer */ + int nByte = sqlite3_blob_bytes(p->pReader); + int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc(nAlloc); + if( pRet ){ + pRet->nn = nByte; + aOut = pRet->p = (u8*)&pRet[1]; + }else{ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pRet); + pRet = 0; + }else{ + /* TODO1: Fix this */ + pRet->szLeaf = fts5GetU16(&pRet->p[2]); + } + } + p->rc = rc; + p->nRead++; + } + + assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + return pRet; +} + + +/* +** Release a reference to data record returned by an earlier call to +** fts5DataRead(). +*/ +static void fts5DataRelease(Fts5Data *pData){ + sqlite3_free(pData); +} + +static int fts5IndexPrepareStmt( + Fts5Index *p, + sqlite3_stmt **ppStmt, + char *zSql +){ + if( p->rc==SQLITE_OK ){ + if( zSql ){ + p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); + }else{ + p->rc = SQLITE_NOMEM; + } + } + sqlite3_free(zSql); + return p->rc; +} + + +/* +** INSERT OR REPLACE a record into the %_data table. +*/ +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + pConfig->zDb, pConfig->zName + )); + if( p->rc ) return; + } + + sqlite3_bind_int64(p->pWriter, 1, iRowid); + sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); + sqlite3_step(p->pWriter); + p->rc = sqlite3_reset(p->pWriter); +} + +/* +** Execute the following SQL: +** +** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast +*/ +static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pDeleter==0 ){ + int rc; + Fts5Config *pConfig = p->pConfig; + char *zSql = sqlite3_mprintf( + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + pConfig->zDb, pConfig->zName + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); + sqlite3_free(zSql); + } + if( rc!=SQLITE_OK ){ + p->rc = rc; + return; + } + } + + sqlite3_bind_int64(p->pDeleter, 1, iFirst); + sqlite3_bind_int64(p->pDeleter, 2, iLast); + sqlite3_step(p->pDeleter); + p->rc = sqlite3_reset(p->pDeleter); +} + +/* +** Remove all records associated with segment iSegid. +*/ +static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; + fts5DataDelete(p, iFirst, iLast); + if( p->pIdxDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE segid=?", + pConfig->zDb, pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); + sqlite3_step(p->pIdxDeleter); + p->rc = sqlite3_reset(p->pIdxDeleter); + } +} + +/* +** Release a reference to an Fts5Structure object returned by an earlier +** call to fts5StructureRead() or fts5StructureDecode(). +*/ +static void fts5StructureRelease(Fts5Structure *pStruct){ + if( pStruct && 0>=(--pStruct->nRef) ){ + int i; + assert( pStruct->nRef==0 ); + for(i=0; i<pStruct->nLevel; i++){ + sqlite3_free(pStruct->aLevel[i].aSeg); + } + sqlite3_free(pStruct); + } +} + +static void fts5StructureRef(Fts5Structure *pStruct){ + pStruct->nRef++; +} + +/* +** Deserialize and return the structure record currently stored in serialized +** form within buffer pData/nData. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated by one slot. This allows the structure contents +** to be more easily edited. +** +** If an error occurs, *ppOut is set to NULL and an SQLite error code +** returned. Otherwise, *ppOut is set to point to the new object and +** SQLITE_OK returned. +*/ +static int fts5StructureDecode( + const u8 *pData, /* Buffer containing serialized structure */ + int nData, /* Size of buffer pData in bytes */ + int *piCookie, /* Configuration cookie value */ + Fts5Structure **ppOut /* OUT: Deserialized object */ +){ + int rc = SQLITE_OK; + int i = 0; + int iLvl; + int nLevel = 0; + int nSegment = 0; + int nByte; /* Bytes of space to allocate at pRet */ + Fts5Structure *pRet = 0; /* Structure object to return */ + + /* Grab the cookie value */ + if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); + i = 4; + + /* Read the total number of levels and segments from the start of the + ** structure record. */ + i += fts5GetVarint32(&pData[i], nLevel); + i += fts5GetVarint32(&pData[i], nSegment); + nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ + ); + pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); + + if( pRet ){ + pRet->nRef = 1; + pRet->nLevel = nLevel; + pRet->nSegment = nSegment; + i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); + + for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ + Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; + int nTotal = 0; + int iSeg; + + if( i>=nData ){ + rc = FTS5_CORRUPT; + }else{ + i += fts5GetVarint32(&pData[i], pLvl->nMerge); + i += fts5GetVarint32(&pData[i], nTotal); + assert( nTotal>=pLvl->nMerge ); + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, + nTotal * sizeof(Fts5StructureSegment) + ); + } + + if( rc==SQLITE_OK ){ + pLvl->nSeg = nTotal; + for(iSeg=0; iSeg<nTotal; iSeg++){ + if( i>=nData ){ + rc = FTS5_CORRUPT; + break; + } + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid); + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); + } + } + } + if( rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + *ppOut = pRet; + return rc; +} + +/* +** +*/ +static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ + if( *pRc==SQLITE_OK ){ + Fts5Structure *pStruct = *ppStruct; + int nLevel = pStruct->nLevel; + int nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ + ); + + pStruct = sqlite3_realloc(pStruct, nByte); + if( pStruct ){ + memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); + pStruct->nLevel++; + *ppStruct = pStruct; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Extend level iLvl so that there is room for at least nExtra more +** segments. +*/ +static void fts5StructureExtendLevel( + int *pRc, + Fts5Structure *pStruct, + int iLvl, + int nExtra, + int bInsert +){ + if( *pRc==SQLITE_OK ){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureSegment *aNew; + int nByte; + + nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); + aNew = sqlite3_realloc(pLvl->aSeg, nByte); + if( aNew ){ + if( bInsert==0 ){ + memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); + }else{ + int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment); + memmove(&aNew[nExtra], aNew, nMove); + memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra); + } + pLvl->aSeg = aNew; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ + Fts5Structure *pRet = 0; + Fts5Config *pConfig = p->pConfig; + int iCookie; /* Configuration cookie */ + Fts5Data *pData; + + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc==SQLITE_OK ){ + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + fts5DataRelease(pData); + if( p->rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + return pRet; +} + +static i64 fts5IndexDataVersion(Fts5Index *p){ + i64 iVersion = 0; + + if( p->rc==SQLITE_OK ){ + if( p->pDataVersion==0 ){ + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) + ); + if( p->rc ) return 0; + } + + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ + iVersion = sqlite3_column_int64(p->pDataVersion, 0); + } + p->rc = sqlite3_reset(p->pDataVersion); + } + + return iVersion; +} + +/* +** Read, deserialize and return the structure record. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated as described for function fts5StructureDecode() +** above. +** +** If an error occurs, NULL is returned and an error code left in the +** Fts5Index handle. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static Fts5Structure *fts5StructureRead(Fts5Index *p){ + + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } + } + +#if 0 + else{ + Fts5Structure *pTest = fts5StructureReadUncached(p); + if( pTest ){ + int i, j; + assert_nc( p->pStruct->nSegment==pTest->nSegment ); + assert_nc( p->pStruct->nLevel==pTest->nLevel ); + for(i=0; i<pTest->nLevel; i++){ + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); + for(j=0; j<pTest->aLevel[i].nSeg; j++){ + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; + assert_nc( p1->iSegid==p2->iSegid ); + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); + assert_nc( p1->pgnoLast==p2->pgnoLast ); + } + } + fts5StructureRelease(pTest); + } + } +#endif + + if( p->rc!=SQLITE_OK ) return 0; + assert( p->iStructVersion!=0 ); + assert( p->pStruct!=0 ); + fts5StructureRef(p->pStruct); + return p->pStruct; +} + +static void fts5StructureInvalidate(Fts5Index *p){ + if( p->pStruct ){ + fts5StructureRelease(p->pStruct); + p->pStruct = 0; + } +} + +/* +** Return the total number of segments in index structure pStruct. This +** function is only ever used as part of assert() conditions. +*/ +#ifdef SQLITE_DEBUG +static int fts5StructureCountSegments(Fts5Structure *pStruct){ + int nSegment = 0; /* Total number of segments */ + if( pStruct ){ + int iLvl; /* Used to iterate through levels */ + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + nSegment += pStruct->aLevel[iLvl].nSeg; + } + } + + return nSegment; +} +#endif + +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ +} + + +/* +** Serialize and store the "structure" record. +** +** If an error occurs, leave an error code in the Fts5Index object. If an +** error has already occurred, this function is a no-op. +*/ +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ + if( p->rc==SQLITE_OK ){ + Fts5Buffer buf; /* Buffer to serialize record into */ + int iLvl; /* Used to iterate through levels */ + int iCookie; /* Cookie value to store */ + + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + memset(&buf, 0, sizeof(Fts5Buffer)); + + /* Append the current configuration cookie */ + iCookie = p->pConfig->iCookie; + if( iCookie<0 ) iCookie = 0; + + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + sqlite3Fts5Put32(buf.p, iCookie); + buf.n = 4; + fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); + fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); + fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); + } + + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg); + assert( pLvl->nMerge<=pLvl->nSeg ); + + for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + } + } + + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); + fts5BufferFree(&buf); + } +} + +#if 0 +static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*); +static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ + int rc = SQLITE_OK; + Fts5Buffer buf; + memset(&buf, 0, sizeof(buf)); + fts5DebugStructure(&rc, &buf, pStruct); + fprintf(stdout, "%s: %s\n", zCaption, buf.p); + fflush(stdout); + fts5BufferFree(&buf); +} +#else +# define fts5PrintStructure(x,y) +#endif + +static int fts5SegmentSize(Fts5StructureSegment *pSeg){ + return 1 + pSeg->pgnoLast - pSeg->pgnoFirst; +} + +/* +** Return a copy of index structure pStruct. Except, promote as many +** segments as possible to level iPromote. If an OOM occurs, NULL is +** returned. +*/ +static void fts5StructurePromoteTo( + Fts5Index *p, + int iPromote, + int szPromote, + Fts5Structure *pStruct +){ + int il, is; + Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote]; + + if( pOut->nMerge==0 ){ + for(il=iPromote+1; il<pStruct->nLevel; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + if( pLvl->nMerge ) return; + for(is=pLvl->nSeg-1; is>=0; is--){ + int sz = fts5SegmentSize(&pLvl->aSeg[is]); + if( sz>szPromote ) return; + fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1); + if( p->rc ) return; + memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment)); + pOut->nSeg++; + pLvl->nSeg--; + } + } + } +} + +/* +** A new segment has just been written to level iLvl of index structure +** pStruct. This function determines if any segments should be promoted +** as a result. Segments are promoted in two scenarios: +** +** a) If the segment just written is smaller than one or more segments +** within the previous populated level, it is promoted to the previous +** populated level. +** +** b) If the segment just written is larger than the newest segment on +** the next populated level, then that segment, and any other adjacent +** segments that are also smaller than the one just written, are +** promoted. +** +** If one or more segments are promoted, the structure object is updated +** to reflect this. +*/ +static void fts5StructurePromote( + Fts5Index *p, /* FTS5 backend object */ + int iLvl, /* Index level just updated */ + Fts5Structure *pStruct /* Index structure */ +){ + if( p->rc==SQLITE_OK ){ + int iTst; + int iPromote = -1; + int szPromote = 0; /* Promote anything this size or smaller */ + Fts5StructureSegment *pSeg; /* Segment just written */ + int szSeg; /* Size of segment just written */ + int nSeg = pStruct->aLevel[iLvl].nSeg; + + if( nSeg==0 ) return; + pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; + szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); + + /* Check for condition (a) */ + for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); + if( iTst>=0 ){ + int i; + int szMax = 0; + Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; + assert( pTst->nMerge==0 ); + for(i=0; i<pTst->nSeg; i++){ + int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; + if( sz>szMax ) szMax = sz; + } + if( szMax>=szSeg ){ + /* Condition (a) is true. Promote the newest segment on level + ** iLvl to level iTst. */ + iPromote = iTst; + szPromote = szMax; + } + } + + /* If condition (a) is not met, assume (b) is true. StructurePromoteTo() + ** is a no-op if it is not. */ + if( iPromote<0 ){ + iPromote = iLvl; + szPromote = szSeg; + } + fts5StructurePromoteTo(p, iPromote, szPromote, pStruct); + } +} + + +/* +** Advance the iterator passed as the only argument. If the end of the +** doclist-index page is reached, return non-zero. +*/ +static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ + Fts5Data *pData = pLvl->pData; + + if( pLvl->iOff==0 ){ + assert( pLvl->bEof==0 ); + pLvl->iOff = 1; + pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno); + pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid); + pLvl->iFirstOff = pLvl->iOff; + }else{ + int iOff; + for(iOff=pLvl->iOff; iOff<pData->nn; iOff++){ + if( pData->p[iOff] ) break; + } + + if( iOff<pData->nn ){ + i64 iVal; + pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; + iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + pLvl->iRowid += iVal; + pLvl->iOff = iOff; + }else{ + pLvl->bEof = 1; + } + } + + return pLvl->bEof; +} + +/* +** Advance the iterator passed as the only argument. +*/ +static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvl<pIter->nLvl ); + if( fts5DlidxLvlNext(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterNextR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ) fts5DlidxLvlNext(pLvl); + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterNextR(p, pIter, 0); +} + +/* +** The iterator passed as the first argument has the following fields set +** as follows. This function sets up the rest of the iterator so that it +** points to the first rowid in the doclist-index. +** +** pData: +** pointer to doclist-index record, +** +** When this function is called pIter->iLeafPgno is the page number the +** doclist is associated with (the one featuring the term). +*/ +static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){ + int i; + for(i=0; i<pIter->nLvl; i++){ + fts5DlidxLvlNext(&pIter->aLvl[i]); + } + return pIter->aLvl[0].bEof; +} + + +static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ + return p->rc!=SQLITE_OK || pIter->aLvl[0].bEof; +} + +static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){ + int i; + + /* Advance each level to the last entry on the last page */ + for(i=pIter->nLvl-1; p->rc==SQLITE_OK && i>=0; i--){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[i]; + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + + if( i>0 ){ + Fts5DlidxLvl *pChild = &pLvl[-1]; + fts5DataRelease(pChild->pData); + memset(pChild, 0, sizeof(Fts5DlidxLvl)); + pChild->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno) + ); + } + } +} + +/* +** Move the iterator passed as the only argument to the previous entry. +*/ +static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ + int iOff = pLvl->iOff; + + assert( pLvl->bEof==0 ); + if( iOff<=pLvl->iFirstOff ){ + pLvl->bEof = 1; + }else{ + u8 *a = pLvl->pData->p; + i64 iVal; + int iLimit; + int ii; + int nZero = 0; + + /* Currently iOff points to the first byte of a varint. This block + ** decrements iOff until it points to the first byte of the previous + ** varint. Taking care not to read any memory locations that occur + ** before the buffer in memory. */ + iLimit = (iOff>9 ? iOff-9 : 0); + for(iOff--; iOff>iLimit; iOff--){ + if( (a[iOff-1] & 0x80)==0 ) break; + } + + fts5GetVarint(&a[iOff], (u64*)&iVal); + pLvl->iRowid -= iVal; + pLvl->iLeafPgno--; + + /* Skip backwards past any 0x00 varints. */ + for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ + nZero++; + } + if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ + /* The byte immediately before the last 0x00 byte has the 0x80 bit + ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 + ** bytes before a[ii]. */ + int bZero = 0; /* True if last 0x00 counts */ + if( (ii-8)>=pLvl->iFirstOff ){ + int j; + for(j=1; j<=8 && (a[ii-j] & 0x80); j++); + bZero = (j>8); + } + if( bZero==0 ) nZero--; + } + pLvl->iLeafPgno -= nZero; + pLvl->iOff = iOff - nZero; + } + + return pLvl->bEof; +} + +static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvl<pIter->nLvl ); + if( fts5DlidxLvlPrev(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterPrevR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ){ + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + } + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterPrevR(p, pIter, 0); +} + +/* +** Free a doclist-index iterator object allocated by fts5DlidxIterInit(). +*/ +static void fts5DlidxIterFree(Fts5DlidxIter *pIter){ + if( pIter ){ + int i; + for(i=0; i<pIter->nLvl; i++){ + fts5DataRelease(pIter->aLvl[i].pData); + } + sqlite3_free(pIter); + } +} + +static Fts5DlidxIter *fts5DlidxIterInit( + Fts5Index *p, /* Fts5 Backend to iterate within */ + int bRev, /* True for ORDER BY ASC */ + int iSegid, /* Segment id */ + int iLeafPg /* Leaf page number to load dlidx for */ +){ + Fts5DlidxIter *pIter = 0; + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + int nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + Fts5DlidxIter *pNew; + + pNew = (Fts5DlidxIter*)sqlite3_realloc(pIter, nByte); + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg); + Fts5DlidxLvl *pLvl = &pNew->aLvl[i]; + pIter = pNew; + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, iRowid); + if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){ + bDone = 1; + } + pIter->nLvl = i+1; + } + } + + if( p->rc==SQLITE_OK ){ + pIter->iSegid = iSegid; + if( bRev==0 ){ + fts5DlidxIterFirst(pIter); + }else{ + fts5DlidxIterLast(p, pIter); + } + } + + if( p->rc!=SQLITE_OK ){ + fts5DlidxIterFree(pIter); + pIter = 0; + } + + return pIter; +} + +static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iRowid; +} +static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iLeafPgno; +} + +/* +** Load the next leaf page into the segment iterator. +*/ +static void fts5SegIterNextPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter /* Iterator to advance to next page */ +){ + Fts5Data *pLeaf; + Fts5StructureSegment *pSeg = pIter->pSeg; + fts5DataRelease(pIter->pLeaf); + pIter->iLeafPgno++; + if( pIter->pNextLeaf ){ + pIter->pLeaf = pIter->pNextLeaf; + pIter->pNextLeaf = 0; + }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ + pIter->pLeaf = fts5DataRead(p, + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) + ); + }else{ + pIter->pLeaf = 0; + } + pLeaf = pIter->pLeaf; + + if( pLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf; + if( fts5LeafIsTermless(pLeaf) ){ + pIter->iEndofDoclist = pLeaf->nn+1; + }else{ + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], + pIter->iEndofDoclist + ); + } + } +} + +/* +** Argument p points to a buffer containing a varint to be interpreted as a +** position list size field. Read the varint and return the number of bytes +** read. Before returning, set *pnSz to the number of bytes in the position +** list, and *pbDel to true if the delete flag is set, or false otherwise. +*/ +static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ + int nSz; + int n = 0; + fts5FastGetVarint32(p, n, nSz); + assert_nc( nSz>=0 ); + *pnSz = nSz/2; + *pbDel = nSz & 0x0001; + return n; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of a +** position-list size field. Read the value of the field and store it +** in the following variables: +** +** Fts5SegIter.nPos +** Fts5SegIter.bDel +** +** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the +** position list content (if any). +*/ +static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ + if( p->rc==SQLITE_OK ){ + int iOff = pIter->iLeafOffset; /* Offset to read at */ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); + pIter->bDel = 0; + pIter->nPos = 1; + if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ + pIter->bDel = 1; + iOff++; + if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ + pIter->nPos = 1; + iOff++; + }else{ + pIter->nPos = 0; + } + } + }else{ + int nSz; + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + pIter->iLeafOffset = iOff; + } +} + +static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + int iOff = pIter->iLeafOffset; + + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + return; + } + iOff = 4; + a = pIter->pLeaf->p; + } + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of the +** "nSuffix" field of a term. Function parameter nKeep contains the value +** of the "nPrefix" field (if there was one - it is passed 0 if this is +** the first term in the segment). +** +** This function populates: +** +** Fts5SegIter.term +** Fts5SegIter.rowid +** +** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of +** the first position list. The position list belonging to document +** (Fts5SegIter.iRowid). +*/ +static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + int iOff = pIter->iLeafOffset; /* Offset to read at */ + int nNew; /* Bytes of new data */ + + iOff += fts5GetVarint32(&a[iOff], nNew); + if( iOff+nNew>pIter->pLeaf->nn ){ + p->rc = FTS5_CORRUPT; + return; + } + pIter->term.n = nKeep; + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + iOff += nNew; + pIter->iTermLeafOffset = iOff; + pIter->iTermLeafPgno = pIter->iLeafPgno; + pIter->iLeafOffset = iOff; + + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); + pIter->iEndofDoclist += nExtra; + } + + fts5SegIterLoadRowid(p, pIter); +} + +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); + +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + pIter->xNext = fts5SegIterNext_Reverse; + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xNext = fts5SegIterNext_None; + }else{ + pIter->xNext = fts5SegIterNext; + } +} + +/* +** Initialize the iterator object pIter to iterate through the entries in +** segment pSeg. The iterator is left pointing to the first entry when +** this function returns. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterInit( + Fts5Index *p, /* FTS index object */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + if( pSeg->pgnoFirst==0 ){ + /* This happens if the segment is being used as an input to an incremental + ** merge and all data has already been "trimmed". See function + ** fts5TrimSegments() for details. In this case leave the iterator empty. + ** The caller will see the (pIter->pLeaf==0) and assume the iterator is + ** at EOF already. */ + assert( pIter->pLeaf==0 ); + return; + } + + if( p->rc==SQLITE_OK ){ + memset(pIter, 0, sizeof(*pIter)); + fts5SegIterSetNext(p, pIter); + pIter->pSeg = pSeg; + pIter->iLeafPgno = pSeg->pgnoFirst-1; + fts5SegIterNextPage(p, pIter); + } + + if( p->rc==SQLITE_OK ){ + pIter->iLeafOffset = 4; + assert_nc( pIter->pLeaf->nn>4 ); + assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + } +} + +/* +** This function is only ever called on iterators created by calls to +** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set. +** +** The iterator is in an unusual state when this function is called: the +** Fts5SegIter.iLeafOffset variable is set to the offset of the start of +** the position-list size field for the first relevant rowid on the page. +** Fts5SegIter.rowid is set, but nPos and bDel are not. +** +** This function advances the iterator so that it points to the last +** relevant rowid on the page and, if necessary, initializes the +** aRowidOffset[] and iRowidOffset variables. At this point the iterator +** is in its regular state - Fts5SegIter.iLeafOffset points to the first +** byte of the position list content associated with said rowid. +*/ +static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; + int n = pIter->pLeaf->szLeaf; + int i = pIter->iLeafOffset; + u8 *a = pIter->pLeaf->p; + int iRowidOffset = 0; + + if( n>pIter->iEndofDoclist ){ + n = pIter->iEndofDoclist; + } + + ASSERT_SZLEAF_OK(pIter->pLeaf); + while( 1 ){ + i64 iDelta = 0; + + if( eDetail==FTS5_DETAIL_NONE ){ + /* todo */ + if( i<n && a[i]==0 ){ + i++; + if( i<n && a[i]==0 ) i++; + } + }else{ + int nPos; + int bDummy; + i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); + i += nPos; + } + if( i>=n ) break; + i += fts5GetVarint(&a[i], (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* If necessary, grow the pIter->aRowidOffset[] array. */ + if( iRowidOffset>=pIter->nRowidOffset ){ + int nNew = pIter->nRowidOffset + 8; + int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + pIter->aRowidOffset = aNew; + pIter->nRowidOffset = nNew; + } + + pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset; + pIter->iLeafOffset = i; + } + pIter->iRowidOffset = iRowidOffset; + fts5SegIterLoadNPos(p, pIter); +} + +/* +** +*/ +static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ + Fts5Data *pNew; + pIter->iLeafPgno--; + pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( + pIter->pSeg->iSegid, pIter->iLeafPgno + )); + if( pNew ){ + /* iTermLeafOffset may be equal to szLeaf if the term is the last + ** thing on the page - i.e. the first rowid is on the following page. + ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ + assert( pIter->pLeaf==0 ); + if( pIter->iTermLeafOffset<pNew->szLeaf ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; + } + }else{ + int iRowidOff; + iRowidOff = fts5LeafFirstRowidOff(pNew); + if( iRowidOff ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = iRowidOff; + } + } + + if( pIter->pLeaf ){ + u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; + pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid); + break; + }else{ + fts5DataRelease(pNew); + } + } + } + + if( pIter->pLeaf ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + fts5SegIterReverseInitPage(p, pIter); + } +} + +/* +** Return true if the iterator passed as the second argument currently +** points to a delete marker. A delete marker is an entry with a 0 byte +** position-list. +*/ +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used by reverse iterators. +*/ +static void fts5SegIterNext_Reverse( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbUnused /* Unused */ +){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->pNextLeaf==0 ); + UNUSED_PARAM(pbUnused); + + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + i64 iDelta; + + pIter->iRowidOffset--; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += pIter->nPos; + } + fts5GetVarint(&a[iOff], (u64*)&iDelta); + pIter->iRowid -= iDelta; + }else{ + fts5SegIterReverseNewPage(p, pIter); + } +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used if detail=none and the +** iterator is not a reverse direction iterator. +*/ +static void fts5SegIterNext_None( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + int iOff; + + assert( p->rc==SQLITE_OK ); + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); + + ASSERT_SZLEAF_OK(pIter->pLeaf); + iOff = pIter->iLeafOffset; + + /* Next entry is on the next page */ + if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( p->rc || pIter->pLeaf==0 ) return; + pIter->iRowid = 0; + iOff = 4; + } + + if( iOff<pIter->iEndofDoclist ){ + /* Next entry is on the current page */ + i64 iDelta; + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); + pIter->iLeafOffset = iOff; + pIter->iRowid += iDelta; + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ + if( pIter->pSeg ){ + int nKeep = 0; + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); + } + pIter->iLeafOffset = iOff; + fts5SegIterLoadTerm(p, pIter, nKeep); + }else{ + const u8 *pList = 0; + const char *zTerm = 0; + int nList; + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + if( pList==0 ) goto next_none_eof; + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList; + sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + } + + if( pbNewTerm ) *pbNewTerm = 1; + }else{ + goto next_none_eof; + } + + fts5SegIterLoadNPos(p, pIter); + + return; + next_none_eof: + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + + +/* +** Advance iterator pIter to the next entry. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. It +** is not considered an error if the iterator reaches EOF. If an error has +** already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterNext( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; + u8 *a; + int n; + + assert( pbNewTerm==0 || *pbNewTerm==0 ); + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + /* Search for the end of the position list within the current page. */ + a = pLeaf->p; + n = pLeaf->szLeaf; + + ASSERT_SZLEAF_OK(pLeaf); + iOff = pIter->iLeafOffset + pIter->nPos; + + if( iOff<n ){ + /* The next entry is on the current page. */ + assert_nc( iOff<=pIter->iEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ + bNewTerm = 1; + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); + } + }else{ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); + } + pIter->iLeafOffset = iOff; + + }else if( pIter->pSeg==0 ){ + const u8 *pList = 0; + const char *zTerm = 0; + int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), + (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; + } + }else{ + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; + ASSERT_SZLEAF_OK(pLeaf); + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + ); + } + + } + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff + ); + pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; + bNewTerm = 1; + } + assert_nc( iOff<pLeaf->szLeaf ); + if( iOff>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + } + } + + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf ){ + if( bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); + fts5SegIterLoadNPos(p, pIter); + if( pbNewTerm ) *pbNewTerm = 1; + } + }else{ + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. + ** + ** Later: Switched back to fts5SegIterLoadNPos() because it supports + ** detail=none mode. Not ideal. + */ + int nSz; + assert( p->rc==SQLITE_OK ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + } +} + +#define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } + +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOff<iEnd ); \ +} + +/* +** Iterator pIter currently points to the first rowid in a doclist. This +** function sets the iterator up so that iterates in reverse order through +** the doclist. +*/ +static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ + Fts5DlidxIter *pDlidx = pIter->pDlidx; + Fts5Data *pLast = 0; + int pgnoLast = 0; + + if( pDlidx ){ + int iSegid = pIter->pSeg->iSegid; + pgnoLast = fts5DlidxIterPgno(pDlidx); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); + }else{ + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of + ** position-list content for the current rowid. Back it up so that it + ** points to the start of the position-list size field. */ + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + pIter->iLeafOffset = iPoslist; + + /* If this condition is true then the largest rowid for the current + ** term may not be stored on the current page. So search forward to + ** see where said rowid really is. */ + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ + int pgno; + Fts5StructureSegment *pSeg = pIter->pSeg; + + /* The last rowid in the doclist may not be on the current page. Search + ** forward to find the page containing the last rowid. */ + for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + Fts5Data *pNew = fts5DataRead(p, iAbs); + if( pNew ){ + int iRowid, bTermless; + iRowid = fts5LeafFirstRowidOff(pNew); + bTermless = fts5LeafIsTermless(pNew); + if( iRowid ){ + SWAPVAL(Fts5Data*, pNew, pLast); + pgnoLast = pgno; + } + fts5DataRelease(pNew); + if( bTermless==0 ) break; + } + } + } + } + + /* If pLast is NULL at this point, then the last rowid for this doclist + ** lies on the page currently indicated by the iterator. In this case + ** pIter->iLeafOffset is already set to point to the position-list size + ** field associated with the first relevant rowid on the page. + ** + ** Or, if pLast is non-NULL, then it is the page that contains the last + ** rowid. In this case configure the iterator so that it points to the + ** first rowid on this page. + */ + if( pLast ){ + int iOff; + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = pLast; + pIter->iLeafPgno = pgnoLast; + iOff = fts5LeafFirstRowidOff(pLast); + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } + + } + + fts5SegIterReverseInitPage(p, pIter); +} + +/* +** Iterator pIter currently points to the first rowid of a doclist. +** There is a doclist-index associated with the final term on the current +** page. If the current term is the last term on the page, load the +** doclist-index from disk and initialize an iterator at (pIter->pDlidx). +*/ +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ + int iSeg = pIter->pSeg->iSegid; + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx==0 ); + + /* Check if the current doclist ends on this page. If it does, return + ** early without loading the doclist-index (as it belongs to a different + ** term. */ + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclist<pLeaf->szLeaf + ){ + return; + } + + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); +} + +/* +** The iterator object passed as the second argument currently contains +** no valid values except for the Fts5SegIter.pLeaf member variable. This +** function searches the leaf page for a term matching (pTerm/nTerm). +** +** If the specified term is found on the page, then the iterator is left +** pointing to it. If argument bGe is zero and the term is not found, +** the iterator is left pointing at EOF. +** +** If bGe is non-zero and the specified term is not found, then the +** iterator is left pointing to the smallest term in the segment that +** is larger than the specified term, even if this term is not on the +** current page. +*/ +static void fts5LeafSeek( + Fts5Index *p, /* Leave any error code here */ + int bGe, /* True for a >= search */ + Fts5SegIter *pIter, /* Iterator to seek */ + const u8 *pTerm, int nTerm /* Term to search for */ +){ + int iOff; + const u8 *a = pIter->pLeaf->p; + int szLeaf = pIter->pLeaf->szLeaf; + int n = pIter->pLeaf->nn; + + int nMatch = 0; + int nKeep = 0; + int nNew = 0; + int iTermOff; + int iPgidx; /* Current offset in pgidx */ + int bEndOfPage = 0; + + assert( p->rc==SQLITE_OK ); + + iPgidx = szLeaf; + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); + iOff = iTermOff; + if( iOff>n ){ + p->rc = FTS5_CORRUPT; + return; + } + + while( 1 ){ + + /* Figure out how many new bytes are in this term */ + fts5FastGetVarint32(a, iOff, nNew); + if( nKeep<nMatch ){ + goto search_failed; + } + + assert( nKeep>=nMatch ); + if( nKeep==nMatch ){ + int nCmp; + int i; + nCmp = MIN(nNew, nTerm-nMatch); + for(i=0; i<nCmp; i++){ + if( a[iOff+i]!=pTerm[nMatch+i] ) break; + } + nMatch += i; + + if( nTerm==nMatch ){ + if( i==nNew ){ + goto search_success; + }else{ + goto search_failed; + } + }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){ + goto search_failed; + } + } + + if( iPgidx>=n ){ + bEndOfPage = 1; + break; + } + + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); + iTermOff += nKeep; + iOff = iTermOff; + + /* Read the nKeep field of the next term. */ + fts5FastGetVarint32(a, iOff, nKeep); + } + + search_failed: + if( bGe==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + return; + }else if( bEndOfPage ){ + do { + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) return; + a = pIter->pLeaf->p; + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ + iPgidx = pIter->pLeaf->szLeaf; + iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); + if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + nKeep = 0; + iTermOff = iOff; + n = pIter->pLeaf->nn; + iOff += fts5GetVarint32(&a[iOff], nNew); + break; + } + } + }while( 1 ); + } + + search_success: + + pIter->iLeafOffset = iOff + nNew; + pIter->iTermLeafOffset = pIter->iLeafOffset; + pIter->iTermLeafPgno = pIter->iLeafPgno; + + fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + + if( iPgidx>=n ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); + pIter->iEndofDoclist = iTermOff + nExtra; + } + pIter->iPgidxOff = iPgidx; + + fts5SegIterLoadRowid(p, pIter); + fts5SegIterLoadNPos(p, pIter); +} + +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + } + return p->pIdxSelect; +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within segment +** pSeg. If there is no such term in the index, the iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterSeekInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = 1; + int bGe = (flags & FTS5INDEX_QUERY_SCAN); + int bDlidx = 0; /* True if there is a doclist-index */ + sqlite3_stmt *pIdxSelect = 0; + + assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); + assert( pTerm && nTerm ); + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + + /* This block sets stack variable iPg to the leaf page number that may + ** contain term (pTerm/nTerm), if it is present in the segment. */ + pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc ) return; + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ + i64 val = sqlite3_column_int(pIdxSelect, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pIdxSelect); + + if( iPg<pSeg->pgnoFirst ){ + iPg = pSeg->pgnoFirst; + bDlidx = 0; + } + + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + + if( pIter->pLeaf ){ + fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); + } + + if( p->rc==SQLITE_OK && bGe==0 ){ + pIter->flags |= FTS5_SEGITER_ONETERM; + if( pIter->pLeaf ){ + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + } + if( bDlidx ){ + fts5SegIterLoadDlidx(p, pIter); + } + if( flags & FTS5INDEX_QUERY_DESC ){ + fts5SegIterReverse(p, pIter); + } + } + } + + fts5SegIterSetNext(p, pIter); + + /* Either: + ** + ** 1) an error has occurred, or + ** 2) the iterator points to EOF, or + ** 3) the iterator points to an entry with term (pTerm/nTerm), or + ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points + ** to an entry with a term greater than or equal to (pTerm/nTerm). + */ + assert( p->rc!=SQLITE_OK /* 1 */ + || pIter->pLeaf==0 /* 2 */ + || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */ + || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */ + ); +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within the +** in-memory hash table. If there is no such term in the hash-table, the +** iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterHashInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5SegIter *pIter /* Object to populate */ +){ + const u8 *pList = 0; + int nList = 0; + const u8 *z = 0; + int n = 0; + + assert( p->pHash ); + assert( p->rc==SQLITE_OK ); + + if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); + n = (z ? (int)strlen((const char*)z) : 0); + }else{ + pIter->flags |= FTS5_SEGITER_ONETERM; + sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); + z = pTerm; + n = nTerm; + } + + if( pList ){ + Fts5Data *pLeaf; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf==0 ) return; + pLeaf->p = (u8*)pList; + pLeaf->nn = pLeaf->szLeaf = nList; + pIter->pLeaf = pLeaf; + pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pLeaf->nn; + + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + } + + fts5SegIterSetNext(p, pIter); +} + +/* +** Zero the iterator passed as the only argument. +*/ +static void fts5SegIterClear(Fts5SegIter *pIter){ + fts5BufferFree(&pIter->term); + fts5DataRelease(pIter->pLeaf); + fts5DataRelease(pIter->pNextLeaf); + fts5DlidxIterFree(pIter->pDlidx); + sqlite3_free(pIter->aRowidOffset); + memset(pIter, 0, sizeof(Fts5SegIter)); +} + +#ifdef SQLITE_DEBUG + +/* +** This function is used as part of the big assert() procedure implemented by +** fts5AssertMultiIterSetup(). It ensures that the result currently stored +** in *pRes is the correct result of comparing the current positions of the +** two iterators. +*/ +static void fts5AssertComparisonResult( + Fts5Iter *pIter, + Fts5SegIter *p1, + Fts5SegIter *p2, + Fts5CResult *pRes +){ + int i1 = p1 - pIter->aSeg; + int i2 = p2 - pIter->aSeg; + + if( p1->pLeaf || p2->pLeaf ){ + if( p1->pLeaf==0 ){ + assert( pRes->iFirst==i2 ); + }else if( p2->pLeaf==0 ){ + assert( pRes->iFirst==i1 ); + }else{ + int nMin = MIN(p1->term.n, p2->term.n); + int res = memcmp(p1->term.p, p2->term.p, nMin); + if( res==0 ) res = p1->term.n - p2->term.n; + + if( res==0 ){ + assert( pRes->bTermEq==1 ); + assert( p1->iRowid!=p2->iRowid ); + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1; + }else{ + assert( pRes->bTermEq==0 ); + } + + if( res<0 ){ + assert( pRes->iFirst==i1 ); + }else{ + assert( pRes->iFirst==i2 ); + } + } + } +} + +/* +** This function is a no-op unless SQLITE_DEBUG is defined when this module +** is compiled. In that case, this function is essentially an assert() +** statement used to verify that the contents of the pIter->aFirst[] array +** are correct. +*/ +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int i; + + assert( (pFirst->pLeaf==0)==pIter->base.bEof ); + + /* Check that pIter->iSwitchRowid is set correctly. */ + for(i=0; i<pIter->nSeg; i++){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + assert( p1==pFirst + || p1->pLeaf==0 + || fts5BufferCompare(&pFirst->term, &p1->term) + || p1->iRowid==pIter->iSwitchRowid + || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev + ); + } + + for(i=0; i<pIter->nSeg; i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + Fts5SegIter *p2 = &pIter->aSeg[i+1]; + Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + + for(i=1; i<(pIter->nSeg / 2); i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ]; + Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ]; + Fts5CResult *pRes = &pIter->aFirst[i]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + } +} +#else +# define fts5AssertMultiIterSetup(x,y) +#endif + +/* +** Do the comparison necessary to populate pIter->aFirst[iOut]. +** +** If the returned value is non-zero, then it is the index of an entry +** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing +** to a key that is a duplicate of another, higher priority, +** segment-iterator in the pSeg->aSeg[] array. +*/ +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ + int i1; /* Index of left-hand Fts5SegIter */ + int i2; /* Index of right-hand Fts5SegIter */ + int iRes; + Fts5SegIter *p1; /* Left-hand Fts5SegIter */ + Fts5SegIter *p2; /* Right-hand Fts5SegIter */ + Fts5CResult *pRes = &pIter->aFirst[iOut]; + + assert( iOut<pIter->nSeg && iOut>0 ); + assert( pIter->bRev==0 || pIter->bRev==1 ); + + if( iOut>=(pIter->nSeg/2) ){ + i1 = (iOut - pIter->nSeg/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pIter->aFirst[iOut*2].iFirst; + i2 = pIter->aFirst[iOut*2+1].iFirst; + } + p1 = &pIter->aSeg[i1]; + p2 = &pIter->aSeg[i2]; + + pRes->bTermEq = 0; + if( p1->pLeaf==0 ){ /* If p1 is at EOF */ + iRes = i2; + }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */ + iRes = i1; + }else{ + int res = fts5BufferCompare(&p1->term, &p2->term); + if( res==0 ){ + assert( i2>i1 ); + assert( i2!=0 ); + pRes->bTermEq = 1; + if( p1->iRowid==p2->iRowid ){ + p1->bDel = p2->bDel; + return i2; + } + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; + } + assert( res!=0 ); + if( res<0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pRes->iFirst = (u16)iRes; + return 0; +} + +/* +** Move the seg-iter so that it points to the first rowid on page iLeafPgno. +** It is an error if leaf iLeafPgno does not exist or contains no rowids. +*/ +static void fts5SegIterGotoPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int iLeafPgno +){ + assert( iLeafPgno>pIter->iLeafPgno ); + + if( iLeafPgno>pIter->pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5DataRelease(pIter->pNextLeaf); + pIter->pNextLeaf = 0; + pIter->iLeafPgno = iLeafPgno-1; + fts5SegIterNextPage(p, pIter); + assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); + + if( p->rc==SQLITE_OK ){ + int iOff; + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + + iOff = fts5LeafFirstRowidOff(pIter->pLeaf); + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + } + } +} + +/* +** Advance the iterator passed as the second argument until it is at or +** past rowid iFrom. Regardless of the value of iFrom, the iterator is +** always advanced at least once. +*/ +static void fts5SegIterNextFrom( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + i64 iMatch /* Advance iterator at least this far */ +){ + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5DlidxIter *pDlidx = pIter->pDlidx; + int iLeafPgno = pIter->iLeafPgno; + int bMove = 1; + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx ); + assert( pIter->pLeaf ); + + if( bRev==0 ){ + while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){ + iLeafPgno = fts5DlidxIterPgno(pDlidx); + fts5DlidxIterNext(p, pDlidx); + } + assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc ); + if( iLeafPgno>pIter->iLeafPgno ){ + fts5SegIterGotoPage(p, pIter, iLeafPgno); + bMove = 0; + } + }else{ + assert( pIter->pNextLeaf==0 ); + assert( iMatch<pIter->iRowid ); + while( !fts5DlidxIterEof(p, pDlidx) && iMatch<fts5DlidxIterRowid(pDlidx) ){ + fts5DlidxIterPrev(p, pDlidx); + } + iLeafPgno = fts5DlidxIterPgno(pDlidx); + + assert( fts5DlidxIterEof(p, pDlidx) || iLeafPgno<=pIter->iLeafPgno ); + + if( iLeafPgno<pIter->iLeafPgno ){ + pIter->iLeafPgno = iLeafPgno+1; + fts5SegIterReverseNewPage(p, pIter); + bMove = 0; + } + } + + do{ + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); + if( pIter->pLeaf==0 ) break; + if( bRev==0 && pIter->iRowid>=iMatch ) break; + if( bRev!=0 && pIter->iRowid<=iMatch ) break; + bMove = 1; + }while( p->rc==SQLITE_OK ); +} + + +/* +** Free the iterator object passed as the second argument. +*/ +static void fts5MultiIterFree(Fts5Iter *pIter){ + if( pIter ){ + int i; + for(i=0; i<pIter->nSeg; i++){ + fts5SegIterClear(&pIter->aSeg[i]); + } + fts5StructureRelease(pIter->pStruct); + fts5BufferFree(&pIter->poslist); + sqlite3_free(pIter); + } +} + +static void fts5MultiIterAdvanced( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + int iMinset /* Minimum entry in aFirst[] to set */ +){ + int i; + for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, 0); + i = pIter->nSeg + iEq; + } + } +} + +/* +** Sub-iterator iChanged of iterator pIter has just been advanced. It still +** points to the same term though - just a different rowid. This function +** attempts to update the contents of the pIter->aFirst[] accordingly. +** If it does so successfully, 0 is returned. Otherwise 1. +** +** If non-zero is returned, the caller should call fts5MultiIterAdvanced() +** on the iterator instead. That function does the same as this one, except +** that it deals with more complicated cases as well. +*/ +static int fts5MultiIterAdvanceRowid( + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + Fts5SegIter **ppFirst +){ + Fts5SegIter *pNew = &pIter->aSeg[iChanged]; + + if( pNew->iRowid==pIter->iSwitchRowid + || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev + ){ + int i; + Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001]; + pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64; + for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){ + Fts5CResult *pRes = &pIter->aFirst[i]; + + assert( pNew->pLeaf ); + assert( pRes->bTermEq==0 || pOther->pLeaf ); + + if( pRes->bTermEq ){ + if( pNew->iRowid==pOther->iRowid ){ + return 1; + }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + pNew = pOther; + }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + } + } + pRes->iFirst = (u16)(pNew - pIter->aSeg); + if( i==1 ) break; + + pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; + } + } + + *ppFirst = pNew; + return 0; +} + +/* +** Set the pIter->bEof variable based on the state of the sub-iterators. +*/ +static void fts5MultiIterSetEof(Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + pIter->base.bEof = pSeg->pLeaf==0; + pIter->iSwitchRowid = pSeg->iRowid; +} + +/* +** Move the iterator to the next entry. +** +** If an error occurs, an error code is left in Fts5Index.rc. It is not +** considered an error if the iterator reaches EOF, or if it is already at +** EOF when this function is called. +*/ +static void fts5MultiIterNext( + Fts5Index *p, + Fts5Iter *pIter, + int bFrom, /* True if argument iFrom is valid */ + i64 iFrom /* Advance at least as far as this */ +){ + int bUseFrom = bFrom; + while( p->rc==SQLITE_OK ){ + int iFirst = pIter->aFirst[1].iFirst; + int bNewTerm = 0; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + assert( p->rc==SQLITE_OK ); + if( bUseFrom && pSeg->pDlidx ){ + fts5SegIterNextFrom(p, pSeg, iFrom); + }else{ + pSeg->xNext(p, pSeg, &bNewTerm); + } + + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + if( pSeg->pLeaf==0 ) return; + } + + fts5AssertMultiIterSetup(p, pIter); + assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); + if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + pIter->xSetOutputs(pIter, pSeg); + return; + } + bUseFrom = 0; + } +} + +static void fts5MultiIterNext2( + Fts5Index *p, + Fts5Iter *pIter, + int *pbNewTerm /* OUT: True if *might* be new term */ +){ + assert( pIter->bSkipEmpty ); + if( p->rc==SQLITE_OK ){ + do { + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + int bNewTerm = 0; + + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, &bNewTerm); + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + *pbNewTerm = 1; + }else{ + *pbNewTerm = 0; + } + fts5AssertMultiIterSetup(p, pIter); + + }while( fts5MultiIterIsEmpty(p, pIter) ); + } +} + +static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ + UNUSED_PARAM2(pUnused1, pUnused2); +} + +static Fts5Iter *fts5MultiIterAlloc( + Fts5Index *p, /* FTS5 backend to iterate within */ + int nSeg +){ + Fts5Iter *pNew; + int nSlot; /* Power of two >= nSeg */ + + for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); + pNew = fts5IdxMalloc(p, + sizeof(Fts5Iter) + /* pNew */ + sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */ + sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ + ); + if( pNew ){ + pNew->nSeg = nSlot; + pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; + pNew->pIndex = p; + pNew->xSetOutputs = fts5IterSetOutputs_Noop; + } + return pNew; +} + +static void fts5PoslistCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} + +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; + +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; i<pColset->nCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static void fts5PoslistOffsetsCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( i<nChunk ){ + int iVal; + i += fts5GetVarint32(&pChunk[i], iVal); + iVal += pCtx->iRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } + } + } +} + +static void fts5PoslistFilterCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; + + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); + }else{ + pCtx->eState = 0; + } + } + + do { + while( i<nChunk && pChunk[i]!=0x01 ){ + while( pChunk[i] & 0x80 ) i++; + i++; + } + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i<nChunk ){ + int iCol; + iStart = i; + i++; + if( i>=nChunk ){ + pCtx->eState = 2; + }else{ + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } + } + } + }while( i<nChunk ); + } +} + +static void fts5ChunkIterate( + Fts5Index *p, /* Index object */ + Fts5SegIter *pSeg, /* Poslist of this iterator */ + void *pCtx, /* Context pointer for xChunk callback */ + void (*xChunk)(Fts5Index*, void*, const u8*, int) +){ + int nRem = pSeg->nPos; /* Number of bytes still to come */ + Fts5Data *pData = 0; + u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); + int pgno = pSeg->iLeafPgno; + int pgnoSave = 0; + + /* This function does notmwork with detail=none databases. */ + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ + pgnoSave = pgno+1; + } + + while( 1 ){ + xChunk(p, pCtx, pChunk, nChunk); + nRem -= nChunk; + fts5DataRelease(pData); + if( nRem<=0 ){ + break; + }else{ + pgno++; + pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); + if( pData==0 ) break; + pChunk = &pData->p[4]; + nChunk = MIN(nRem, pData->szLeaf - 4); + if( pgno==pgnoSave ){ + assert( pSeg->pNextLeaf==0 ); + pSeg->pNextLeaf = pData; + pData = 0; + } + } + } +} + +/* +** Iterator pIter currently points to a valid entry (not EOF). This +** function appends the position list data for the current entry to +** buffer pBuf. It does not make a copy of the position-list size +** field. +*/ +static void fts5SegiterPoslist( + Fts5Index *p, + Fts5SegIter *pSeg, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } + } + } +} + +/* +** IN/OUT parameter (*pa) points to a position list n bytes in size. If +** the position list contains entries for column iCol, then (*pa) is set +** to point to the sub-position-list for that column and the number of +** bytes in it returned. Or, if the argument position list does not +** contain any entries for column iCol, return 0. +*/ +static int fts5IndexExtractCol( + const u8 **pa, /* IN/OUT: Pointer to poslist */ + int n, /* IN: Size of poslist in bytes */ + int iCol /* Column to extract from poslist */ +){ + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ + const u8 *p = *pa; + const u8 *pEnd = &p[n]; /* One byte past end of position list */ + + while( iCol>iCurrent ){ + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint. Note that it is not possible for a negative + ** or extremely large varint to occur within an uncorrupted position + ** list. So the last byte of each varint may be assumed to have a clear + ** 0x80 bit. */ + while( *p!=0x01 ){ + while( *p++ & 0x80 ); + if( p>=pEnd ) return 0; + } + *pa = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } + } + if( iCol!=iCurrent ) return 0; + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( p<pEnd && *p!=0x01 ){ + while( *p++ & 0x80 ); + } + + return p - (*pa); +} + +static int fts5IndexExtractColset ( + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Buffer *pBuf /* Output buffer */ +){ + int rc = SQLITE_OK; + int i; + + fts5BufferZero(pBuf); + for(i=0; i<pColset->nCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. +*/ +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pIter->pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; + + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( a<pEnd ){ + iPrev += (int)a++[0] - 2; + while( *aiCol<iPrev ){ + aiCol++; + if( aiCol==aiColEnd ) goto setoutputs_col_out; + } + if( *aiCol==iPrev ){ + *aOut++ = (iPrev - iPrevOut) + 2; + iPrevOut = iPrev; + } + } + +setoutputs_col_out: + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset->nCol==1 ){ + pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); + pIter->base.pData = a; + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + if( *pRc==SQLITE_OK ){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } + } + } +} + + +/* +** Allocate a new Fts5Iter object. +** +** The new object will be used to iterate through data in structure pStruct. +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel +** is zero or greater, data from the first nSegment segments on level iLevel +** is merged. +** +** The iterator initially points to the first term/rowid entry in the +** iterated data. +*/ +static void fts5MultiIterNew( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Structure *pStruct, /* Structure of specific index */ + int flags, /* FTS5INDEX_QUERY_XXX flags */ + Fts5Colset *pColset, /* Colset to filter on (or NULL) */ + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ + int iLevel, /* Level to iterate (-1 for all) */ + int nSegment, /* Number of segments to merge (iLevel>=0) */ + Fts5Iter **ppOut /* New object */ +){ + int nSeg = 0; /* Number of segment-iters in use */ + int iIter = 0; /* */ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl; + Fts5Iter *pNew; + + assert( (pTerm==0 && nTerm==0) || iLevel<0 ); + + /* Allocate space for the new multi-seg-iterator. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->pHash ? 1 : 0); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + } + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); + if( pNew==0 ) return; + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pStruct = pStruct; + pNew->pColset = pColset; + fts5StructureRef(pStruct); + if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ + fts5IterSetOutputCb(&p->rc, pNew); + } + + /* Initialize each of the component segment iterators. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->pHash ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); + } + for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ + for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + if( pTerm==0 ){ + fts5SegIterInit(p, pSeg, pIter); + }else{ + fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); + } + } + } + }else{ + pLvl = &pStruct->aLevel[iLevel]; + for(iSeg=nSeg-1; iSeg>=0; iSeg--){ + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + } + } + assert( iIter==nSeg ); + } + + /* If the above was successful, each component iterators now points + ** to the first entry in its segment. In this case initialize the + ** aFirst[] array. Or, if an error has occurred, free the iterator + ** object and set the output variable to NULL. */ + if( p->rc==SQLITE_OK ){ + for(iIter=pNew->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pNew, iEq, iIter); + } + } + fts5MultiIterSetEof(pNew); + fts5AssertMultiIterSetup(p, pNew); + + if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + fts5MultiIterNext(p, pNew, 0, 0); + }else if( pNew->base.bEof==0 ){ + Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; + pNew->xSetOutputs(pNew, pSeg); + } + + }else{ + fts5MultiIterFree(pNew); + *ppOut = 0; + } +} + +/* +** Create an Fts5Iter that iterates through the doclist provided +** as the second argument. +*/ +static void fts5MultiIterNew2( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Data *pData, /* Doclist to iterate through */ + int bDesc, /* True for descending rowid order */ + Fts5Iter **ppOut /* New object */ +){ + Fts5Iter *pNew; + pNew = fts5MultiIterAlloc(p, 2); + if( pNew ){ + Fts5SegIter *pIter = &pNew->aSeg[1]; + + pIter->flags = FTS5_SEGITER_ONETERM; + if( pData->szLeaf>0 ){ + pIter->pLeaf = pData; + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; + pNew->aFirst[1].iFirst = 1; + if( bDesc ){ + pNew->bRev = 1; + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + pData = 0; + }else{ + pNew->base.bEof = 1; + } + fts5SegIterSetNext(p, pIter); + + *ppOut = pNew; + } + + fts5DataRelease(pData); +} + +/* +** Return true if the iterator is at EOF or if an error has occurred. +** False otherwise. +*/ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ + assert( p->rc + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + ); + return (p->rc || pIter->base.bEof); +} + +/* +** Return the rowid of the entry that the iterator currently points +** to. If the iterator points to EOF when this function is called the +** results are undefined. +*/ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; +} + +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5Iter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter, 1, iMatch); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid>=iMatch ) break; + if( pIter->bRev!=0 && iRowid<=iMatch ) break; + } +} + +/* +** Return a pointer to a buffer containing the term associated with the +** entry that the iterator currently points to. +*/ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + *pn = p->term.n; + return p->term.p; +} + +/* +** Allocate a new segment-id for the structure pStruct. The new segment +** id must be between 1 and 65335 inclusive, and must not be used by +** any currently existing segment. If a free segment id cannot be found, +** SQLITE_FULL is returned. +** +** If an error has already occurred, this function is a no-op. 0 is +** returned in this case. +*/ +static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ + int iSegid = 0; + + if( p->rc==SQLITE_OK ){ + if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ + p->rc = SQLITE_FULL; + }else{ + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT ){ + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); + } + } + } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & (1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + + { + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc==SQLITE_OK ){ + u8 aBlob[2] = {0xff, 0xff}; + sqlite3_bind_int(pIdxSelect, 1, iSegid); + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); + assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + p->rc = sqlite3_reset(pIdxSelect); + } + } +#endif + } + } + + return iSegid; +} + +/* +** Discard all data currently cached in the hash-tables. +*/ +static void fts5IndexDiscardData(Fts5Index *p){ + assert( p->pHash || p->nPendingData==0 ); + if( p->pHash ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + } +} + +/* +** Return the size of the prefix, in bytes, that buffer +** (pNew/<length-unknown>) shares with buffer (pOld/nOld). +** +** Buffer (pNew/<length-unknown>) is guaranteed to be greater +** than buffer (pOld/nOld). +*/ +static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ + int i; + for(i=0; i<nOld; i++){ + if( pOld[i]!=pNew[i] ) break; + } + return i; +} + +static void fts5WriteDlidxClear( + Fts5Index *p, + Fts5SegWriter *pWriter, + int bFlush /* If true, write dlidx to disk */ +){ + int i; + assert( bFlush==0 || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n>0) ); + for(i=0; i<pWriter->nDlidx; i++){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + if( pDlidx->buf.n==0 ) break; + if( bFlush ){ + assert( pDlidx->pgno!=0 ); + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + } + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + } +} + +/* +** Grow the pWriter->aDlidx[] array to at least nLvl elements in size. +** Any new array elements are zeroed before returning. +*/ +static int fts5WriteDlidxGrow( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nLvl +){ + if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){ + Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc( + pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl + ); + if( aDlidx==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + memset(&aDlidx[pWriter->nDlidx], 0, nByte); + pWriter->aDlidx = aDlidx; + pWriter->nDlidx = nLvl; + } + } + return p->rc; +} + +/* +** If the current doclist-index accumulating in pWriter->aDlidx[] is large +** enough, flush it to disk and return 1. Otherwise discard it and return +** zero. +*/ +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag = 0; + + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written + ** to the database, also write the doclist-index to disk. */ + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ + bFlag = 1; + } + fts5WriteDlidxClear(p, pWriter, bFlag); + pWriter->nEmpty = 0; + return bFlag; +} + +/* +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. +** +** The doclist-index for that term is currently stored in-memory within the +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function +** writes it out to disk. Or, if it is too small to bother with, discards +** it. +** +** Fts5SegWriter.btterm currently contains the first term on page iBtPage. +*/ +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag; + + assert( pWriter->iBtPage || pWriter->nEmpty==0 ); + if( pWriter->iBtPage==0 ) return; + bFlag = fts5WriteFlushDlidx(p, pWriter); + + if( p->rc==SQLITE_OK ){ + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); + /* The following was already done in fts5WriteInit(): */ + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); + sqlite3_step(p->pIdxWriter); + p->rc = sqlite3_reset(p->pIdxWriter); + } + pWriter->iBtPage = 0; +} + +/* +** This is called once for each leaf page except the first that contains +** at least one term. Argument (nTerm/pTerm) is the split-key - a term that +** is larger than all terms written to earlier leaves, and equal to or +** smaller than the first term on the new leaf. +** +** If an error occurs, an error code is left in Fts5Index.rc. If an error +** has already occurred when this function is called, it is a no-op. +*/ +static void fts5WriteBtreeTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter, /* Writer object */ + int nTerm, const u8 *pTerm /* First term on new page */ +){ + fts5WriteFlushBtree(p, pWriter); + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; +} + +/* +** This function is called when flushing a leaf page that contains no +** terms at all to disk. +*/ +static void fts5WriteBtreeNoTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter /* Writer object */ +){ + /* If there were no rowids on the leaf page either and the doclist-index + ** has already been started, append an 0x00 byte to it. */ + if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0]; + assert( pDlidx->bPrevValid ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0); + } + + /* Increment the "number of sequential leaves without a term" counter. */ + pWriter->nEmpty++; +} + +static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){ + i64 iRowid; + int iOff; + + iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid); + fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid); + return iRowid; +} + +/* +** Rowid iRowid has just been appended to the current leaf page. It is the +** first on the page. This function appends an appropriate entry to the current +** doclist-index. +*/ +static void fts5WriteDlidxAppend( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + i64 iVal; + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + + if( pDlidx->buf.n>=p->pConfig->pgsz ){ + /* The current doclist-index page is full. Write it to disk and push + ** a copy of iRowid (which will become the first rowid on the next + ** doclist-index leaf page) up into the next level of the b-tree + ** hierarchy. If the node being flushed is currently the root node, + ** also push its first rowid upwards. */ + pDlidx->buf.p[0] = 0x01; /* Not the root node */ + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + fts5WriteDlidxGrow(p, pWriter, i+2); + pDlidx = &pWriter->aDlidx[i]; + if( p->rc==SQLITE_OK && pDlidx[1].buf.n==0 ){ + i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf); + + /* This was the root node. Push its first rowid up to the new root. */ + pDlidx[1].pgno = pDlidx->pgno; + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst); + pDlidx[1].bPrevValid = 1; + pDlidx[1].iPrev = iFirst; + } + + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + pDlidx->pgno++; + }else{ + bDone = 1; + } + + if( pDlidx->bPrevValid ){ + iVal = iRowid - pDlidx->iPrev; + }else{ + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); + assert( pDlidx->buf.n==0 ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); + iVal = iRowid; + } + + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal); + pDlidx->bPrevValid = 1; + pDlidx->iPrev = iRowid; + } +} + +static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ + static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; + Fts5PageWriter *pPage = &pWriter->writer; + i64 iRowid; + +static int nCall = 0; +nCall++; + + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); + + /* Set the szLeaf header field. */ + assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); + + if( pWriter->bFirstTermInPage ){ + /* No term was written to this page. */ + assert( pPage->pgidx.n==0 ); + fts5WriteBtreeNoTerm(p, pWriter); + }else{ + /* Append the pgidx to the page buffer. Set the szLeaf header field. */ + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); + } + + /* Write the page out to disk */ + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); + fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); + + /* Initialize the next page. */ + fts5BufferZero(&pPage->buf); + fts5BufferZero(&pPage->pgidx); + fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); + pPage->iPrevPgidx = 0; + pPage->pgno++; + + /* Increase the leaves written counter */ + pWriter->nLeafWritten++; + + /* The new leaf holds no terms or rowids */ + pWriter->bFirstTermInPage = 1; + pWriter->bFirstRowidInPage = 1; +} + +/* +** Append term pTerm/nTerm to the segment being written by the writer passed +** as the second argument. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5WriteAppendTerm( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nTerm, const u8 *pTerm +){ + int nPrefix; /* Bytes of prefix compression for term */ + Fts5PageWriter *pPage = &pWriter->writer; + Fts5Buffer *pPgidx = &pWriter->writer.pgidx; + + assert( p->rc==SQLITE_OK ); + assert( pPage->buf.n>=4 ); + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); + + /* If the current leaf page is full, flush it to disk. */ + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ + if( pPage->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); + } + + /* TODO1: Updating pgidx here. */ + pPgidx->n += sqlite3Fts5PutVarint( + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx + ); + pPage->iPrevPgidx = pPage->buf.n; +#if 0 + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); + pPgidx->n += 2; +#endif + + if( pWriter->bFirstTermInPage ){ + nPrefix = 0; + if( pPage->pgno!=1 ){ + /* This is the first term on a leaf that is not the leftmost leaf in + ** the segment b-tree. In this case it is necessary to add a term to + ** the b-tree hierarchy that is (a) larger than the largest term + ** already written to the segment and (b) smaller than or equal to + ** this term. In other words, a prefix of (pTerm/nTerm) that is one + ** byte longer than the longest prefix (pTerm/nTerm) shares with the + ** previous term. + ** + ** Usually, the previous term is available in pPage->term. The exception + ** is if this is the first term written in an incremental-merge step. + ** In this case the previous term is not available, so just write a + ** copy of (pTerm/nTerm) into the parent node. This is slightly + ** inefficient, but still correct. */ + int n = nTerm; + if( pPage->term.n ){ + n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); + } + fts5WriteBtreeTerm(p, pWriter, n, pTerm); + pPage = &pWriter->writer; + } + }else{ + nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); + fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); + } + + /* Append the number of bytes of new data, then the term data itself + ** to the page. */ + fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); + fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); + + /* Update the Fts5PageWriter.term field. */ + fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); + pWriter->bFirstTermInPage = 0; + + pWriter->bFirstRowidInPage = 0; + pWriter->bFirstRowidInDoclist = 1; + + assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); + pWriter->aDlidx[0].pgno = pPage->pgno; +} + +/* +** Append a rowid and position-list size field to the writers output. +*/ +static void fts5WriteAppendRowid( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + if( p->rc==SQLITE_OK ){ + Fts5PageWriter *pPage = &pWriter->writer; + + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ + fts5WriteFlushLeaf(p, pWriter); + } + + /* If this is to be the first rowid written to the page, set the + ** rowid-pointer in the page-header. Also append a value to the dlidx + ** buffer, in case a doclist-index is required. */ + if( pWriter->bFirstRowidInPage ){ + fts5PutU16(pPage->buf.p, (u16)pPage->buf.n); + fts5WriteDlidxAppend(p, pWriter, iRowid); + } + + /* Write the rowid. */ + if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); + }else{ + assert( p->rc || iRowid>pWriter->iPrevRowid ); + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + } + pWriter->iPrevRowid = iRowid; + pWriter->bFirstRowidInDoclist = 0; + pWriter->bFirstRowidInPage = 0; + } +} + +static void fts5WriteAppendPoslistData( + Fts5Index *p, + Fts5SegWriter *pWriter, + const u8 *aData, + int nData +){ + Fts5PageWriter *pPage = &pWriter->writer; + const u8 *a = aData; + int n = nData; + + assert( p->pConfig->pgsz>0 ); + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + ){ + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; + int nCopy = 0; + while( nCopy<nReq ){ + i64 dummy; + nCopy += fts5GetVarint(&a[nCopy], (u64*)&dummy); + } + fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a); + a += nCopy; + n -= nCopy; + fts5WriteFlushLeaf(p, pWriter); + } + if( n>0 ){ + fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); + } +} + +/* +** Flush any data cached by the writer object to the database. Free any +** allocations associated with the writer. +*/ +static void fts5WriteFinish( + Fts5Index *p, + Fts5SegWriter *pWriter, /* Writer object */ + int *pnLeaf /* OUT: Number of leaf pages in b-tree */ +){ + int i; + Fts5PageWriter *pLeaf = &pWriter->writer; + if( p->rc==SQLITE_OK ){ + assert( pLeaf->pgno>=1 ); + if( pLeaf->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + *pnLeaf = pLeaf->pgno-1; + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } + } + fts5BufferFree(&pLeaf->term); + fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pLeaf->pgidx); + fts5BufferFree(&pWriter->btterm); + + for(i=0; i<pWriter->nDlidx; i++){ + sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); + } + sqlite3_free(pWriter->aDlidx); +} + +static void fts5WriteInit( + Fts5Index *p, + Fts5SegWriter *pWriter, + int iSegid +){ + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; + + memset(pWriter, 0, sizeof(Fts5SegWriter)); + pWriter->iSegid = iSegid; + + fts5WriteDlidxGrow(p, pWriter, 1); + pWriter->writer.pgno = 1; + pWriter->bFirstTermInPage = 1; + pWriter->iBtPage = 1; + + assert( pWriter->writer.buf.n==0 ); + assert( pWriter->writer.pgidx.n==0 ); + + /* Grow the two buffers to pgsz + padding bytes in size. */ + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer); + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer); + + if( p->pIdxWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + pConfig->zDb, pConfig->zName + )); + } + + if( p->rc==SQLITE_OK ){ + /* Initialize the 4-byte leaf-page header to 0x00. */ + memset(pWriter->writer.buf.p, 0, 4); + pWriter->writer.buf.n = 4; + + /* Bind the current output segment id to the index-writer. This is an + ** optimization over binding the same value over and over as rows are + ** inserted into %_idx by the current writer. */ + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); + } +} + +/* +** Iterator pIter was used to iterate through the input segments of on an +** incremental merge operation. This function is called if the incremental +** merge step has finished but the input has not been completely exhausted. +*/ +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ + int i; + Fts5Buffer buf; + memset(&buf, 0, sizeof(Fts5Buffer)); + for(i=0; i<pIter->nSeg; i++){ + Fts5SegIter *pSeg = &pIter->aSeg[i]; + if( pSeg->pSeg==0 ){ + /* no-op */ + }else if( pSeg->pLeaf==0 ){ + /* All keys from this input segment have been transfered to the output. + ** Set both the first and last page-numbers to 0 to indicate that the + ** segment is now empty. */ + pSeg->pSeg->pgnoLast = 0; + pSeg->pSeg->pgnoFirst = 0; + }else{ + int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */ + i64 iLeafRowid; + Fts5Data *pData; + int iId = pSeg->pSeg->iSegid; + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; + + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); + pData = fts5DataRead(p, iLeafRowid); + if( pData ){ + fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); + fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); + fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); + fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], (u16)buf.n); + } + + /* Set up the new page-index array */ + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclist<pData->szLeaf + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } + + fts5DataRelease(pData); + pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); + fts5DataWrite(p, iLeafRowid, buf.p, buf.n); + } + } + } + fts5BufferFree(&buf); +} + +static void fts5MergeChunkCallback( + Fts5Index *p, + void *pCtx, + const u8 *pChunk, int nChunk +){ + Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; + fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); +} + +/* +** +*/ +static void fts5IndexMergeLevel( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ + int iLvl, /* Level to read input from */ + int *pnRem /* Write up to this many output leaves */ +){ + Fts5Structure *pStruct = *ppStruct; + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureLevel *pLvlOut; + Fts5Iter *pIter = 0; /* Iterator to read input data */ + int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ + int nInput; /* Number of input segments */ + Fts5SegWriter writer; /* Writer object */ + Fts5StructureSegment *pSeg; /* Output segment */ + Fts5Buffer term; + int bOldest; /* True if the output segment is the oldest */ + int eDetail = p->pConfig->eDetail; + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + + assert( iLvl<pStruct->nLevel ); + assert( pLvl->nMerge<=pLvl->nSeg ); + + memset(&writer, 0, sizeof(Fts5SegWriter)); + memset(&term, 0, sizeof(Fts5Buffer)); + if( pLvl->nMerge ){ + pLvlOut = &pStruct->aLevel[iLvl+1]; + assert( pLvlOut->nSeg>0 ); + nInput = pLvl->nMerge; + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; + + fts5WriteInit(p, &writer, pSeg->iSegid); + writer.writer.pgno = pSeg->pgnoLast+1; + writer.iBtPage = 0; + }else{ + int iSegid = fts5AllocateSegid(p, pStruct); + + /* Extend the Fts5Structure object as required to ensure the output + ** segment exists. */ + if( iLvl==pStruct->nLevel-1 ){ + fts5StructureAddLevel(&p->rc, ppStruct); + pStruct = *ppStruct; + } + fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); + if( p->rc ) return; + pLvl = &pStruct->aLevel[iLvl]; + pLvlOut = &pStruct->aLevel[iLvl+1]; + + fts5WriteInit(p, &writer, iSegid); + + /* Add the new segment to the output level */ + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; + pLvlOut->nSeg++; + pSeg->pgnoFirst = 1; + pSeg->iSegid = iSegid; + pStruct->nSegment++; + + /* Read input from all segments in the input level */ + nInput = pLvl->nSeg; + } + bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); + + assert( iLvl>=0 ); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int nPos; /* position-list size field value */ + int nTerm; + const u8 *pTerm; + + /* Check for key annihilation. */ + if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; + + pTerm = fts5MultiIterTerm(pIter, &nTerm); + if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){ + if( pnRem && writer.nLeafWritten>nRem ){ + break; + } + + /* This is a new term. Append a term to the output segment. */ + fts5WriteAppendTerm(p, &writer, nTerm, pTerm); + fts5BufferSet(&p->rc, &term, nTerm, pTerm); + } + + /* Append the rowid to the output */ + /* WRITEPOSLISTSIZE */ + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( pSegIter->bDel ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + if( pSegIter->nPos>0 ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + } + } + }else{ + /* Append the position-list data to the output */ + nPos = pSegIter->nPos*2 + pSegIter->bDel; + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + } + } + + /* Flush the last leaf page to disk. Set the output segment b-tree height + ** and last leaf page number at the same time. */ + fts5WriteFinish(p, &writer, &pSeg->pgnoLast); + + if( fts5MultiIterEof(p, pIter) ){ + int i; + + /* Remove the redundant segments from the %_data table */ + for(i=0; i<nInput; i++){ + fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid); + } + + /* Remove the redundant segments from the input level */ + if( pLvl->nSeg!=nInput ){ + int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); + memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); + } + pStruct->nSegment -= nInput; + pLvl->nSeg -= nInput; + pLvl->nMerge = 0; + if( pSeg->pgnoLast==0 ){ + pLvlOut->nSeg--; + pStruct->nSegment--; + } + }else{ + assert( pSeg->pgnoLast>0 ); + fts5TrimSegments(p, pIter); + pLvl->nMerge = nInput; + } + + fts5MultiIterFree(pIter); + fts5BufferFree(&term); + if( pnRem ) *pnRem -= writer.nLeafWritten; +} + +/* +** Do up to nPg pages of automerge work on the index. +** +** Return true if any changes were actually made, or false otherwise. +*/ +static int fts5IndexMerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nPg, /* Pages of work to do */ + int nMin /* Minimum number of segments to merge */ +){ + int nRem = nPg; + int bRet = 0; + Fts5Structure *pStruct = *ppStruct; + while( nRem>0 && p->rc==SQLITE_OK ){ + int iLvl; /* To iterate through levels */ + int iBestLvl = 0; /* Level offering the most input segments */ + int nBest = 0; /* Number of input segments on best level */ + + /* Set iBestLvl to the level to read input segments from. */ + assert( pStruct->nLevel>0 ); + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + if( pLvl->nMerge ){ + if( pLvl->nMerge>nBest ){ + iBestLvl = iLvl; + nBest = pLvl->nMerge; + } + break; + } + if( pLvl->nSeg>nBest ){ + nBest = pLvl->nSeg; + iBestLvl = iLvl; + } + } + + /* If nBest is still 0, then the index must be empty. */ +#ifdef SQLITE_DEBUG + for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ + assert( pStruct->aLevel[iLvl].nSeg==0 ); + } +#endif + + if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){ + break; + } + bRet = 1; + fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); + if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ + fts5StructurePromote(p, iBestLvl+1, pStruct); + } + } + *ppStruct = pStruct; + return bRet; +} + +/* +** A total of nLeaf leaf pages of data has just been flushed to a level-0 +** segment. This function updates the write-counter accordingly and, if +** necessary, performs incremental merge work. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5IndexAutomerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nLeaf /* Number of output leaves just written */ +){ + if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ + Fts5Structure *pStruct = *ppStruct; + u64 nWrite; /* Initial value of write-counter */ + int nWork; /* Number of work-quanta to perform */ + int nRem; /* Number of leaf pages left to write */ + + /* Update the write-counter. While doing so, set nWork. */ + nWrite = pStruct->nWriteCounter; + nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); + pStruct->nWriteCounter += nLeaf; + nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); + + fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); + } +} + +static void fts5IndexCrisismerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ +){ + const int nCrisis = p->pConfig->nCrisisMerge; + Fts5Structure *pStruct = *ppStruct; + int iLvl = 0; + + assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; +} + +static int fts5IndexReturn(Fts5Index *p){ + int rc = p->rc; + p->rc = SQLITE_OK; + return rc; +} + +typedef struct Fts5FlushCtx Fts5FlushCtx; +struct Fts5FlushCtx { + Fts5Index *pIdx; + Fts5SegWriter writer; +}; + +/* +** Buffer aBuf[] contains a list of varints, all small enough to fit +** in a 32-bit integer. Return the size of the largest prefix of this +** list nMax bytes or less in size. +*/ +static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ + int ret; + u32 dummy; + ret = fts5GetVarint32(aBuf, dummy); + if( ret<nMax ){ + while( 1 ){ + int i = fts5GetVarint32(&aBuf[ret], dummy); + if( (ret + i) > nMax ) break; + ret += i; + } + } + return ret; +} + +/* +** Flush the contents of in-memory hash table iHash to a new level-0 +** segment on disk. Also update the corresponding structure record. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5FlushOneHash(Fts5Index *p){ + Fts5Hash *pHash = p->pHash; + Fts5Structure *pStruct; + int iSegid; + int pgnoLast = 0; /* Last leaf page number in segment */ + + /* Obtain a reference to the index structure and allocate a new segment-id + ** for the new level-0 segment. */ + pStruct = fts5StructureRead(p); + iSegid = fts5AllocateSegid(p, pStruct); + fts5StructureInvalidate(p); + + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Write the term for this entry to disk. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); + fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); + + assert( writer.bFirstRowidInPage==0 ); + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + i64 iRowid = 0; + i64 iDelta = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOff<nDoclist ){ + iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); + iRowid += iDelta; + + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + } + assert( pBuf->n<=pBuf->nSpace ); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOff<nDoclist && pDoclist[iOff]==0 ){ + pBuf->p[pBuf->n++] = 0; + iOff++; + if( iOff<nDoclist && pDoclist[iOff]==0 ){ + pBuf->p[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; + } + } + } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + sqlite3Fts5HashScanNext(pHash); + } + sqlite3Fts5HashClear(pHash); + fts5WriteFinish(p, &writer, &pgnoLast); + + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); + } + + fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexCrisismerge(p, &pStruct); + fts5StructureWrite(p, pStruct); + fts5StructureRelease(pStruct); +} + +/* +** Flush any data stored in the in-memory hash tables to the database. +*/ +static void fts5IndexFlush(Fts5Index *p){ + /* Unless it is empty, flush the hash table to disk */ + if( p->nPendingData ){ + assert( p->pHash ); + p->nPendingData = 0; + fts5FlushOneHash(p); + } +} + +static Fts5Structure *fts5IndexOptimizeStruct( + Fts5Index *p, + Fts5Structure *pStruct +){ + Fts5Structure *pNew = 0; + int nByte = sizeof(Fts5Structure); + int nSeg = pStruct->nSegment; + int i; + + /* Figure out if this structure requires optimization. A structure does + ** not require optimization if either: + ** + ** + it consists of fewer than two segments, or + ** + all segments are on the same level, or + ** + all segments except one are currently inputs to a merge operation. + ** + ** In the first case, return NULL. In the second, increment the ref-count + ** on *pStruct and return a copy of the pointer to it. + */ + if( nSeg<2 ) return 0; + for(i=0; i<pStruct->nLevel; i++){ + int nThis = pStruct->aLevel[i].nSeg; + if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + fts5StructureRef(pStruct); + return pStruct; + } + assert( pStruct->aLevel[i].nMerge<=nThis ); + } + + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + + if( pNew ){ + Fts5StructureLevel *pLvl; + nByte = nSeg * sizeof(Fts5StructureSegment); + pNew->nLevel = pStruct->nLevel+1; + pNew->nRef = 1; + pNew->nWriteCounter = pStruct->nWriteCounter; + pLvl = &pNew->aLevel[pStruct->nLevel]; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pLvl->aSeg ){ + int iLvl, iSeg; + int iSegOut = 0; + /* Iterate through all segments, from oldest to newest. Add them to + ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest + ** segment in the data structure. */ + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; + iSegOut++; + } + } + pNew->nSegment = pLvl->nSeg = nSeg; + }else{ + sqlite3_free(pNew); + pNew = 0; + } + } + + return pNew; +} + +static int sqlite3Fts5IndexOptimize(Fts5Index *p){ + Fts5Structure *pStruct; + Fts5Structure *pNew = 0; + + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + fts5StructureInvalidate(p); + + if( pStruct ){ + pNew = fts5IndexOptimizeStruct(p, pStruct); + } + fts5StructureRelease(pStruct); + + assert( pNew==0 || pNew->nSegment>0 ); + if( pNew ){ + int iLvl; + for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ + int nRem = FTS5_OPT_WORK_UNIT; + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); + } + + fts5StructureWrite(p, pNew); + fts5StructureRelease(pNew); + } + + return fts5IndexReturn(p); +} + +/* +** This is called to implement the special "VALUES('merge', $nMerge)" +** INSERT command. +*/ +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ + Fts5Structure *pStruct = fts5StructureRead(p); + if( pStruct ){ + int nMin = p->pConfig->nUsermerge; + fts5StructureInvalidate(p); + if( nMerge<0 ){ + Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); + fts5StructureRelease(pStruct); + pStruct = pNew; + nMin = 2; + nMerge = nMerge*-1; + } + if( pStruct && pStruct->nLevel ){ + if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ + fts5StructureWrite(p, pStruct); + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +static void fts5AppendRowid( + Fts5Index *p, + i64 iDelta, + Fts5Iter *pUnused, + Fts5Buffer *pBuf +){ + UNUSED_PARAM(pUnused); + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); +} + +static void fts5AppendPoslist( + Fts5Index *p, + i64 iDelta, + Fts5Iter *pMulti, + Fts5Buffer *pBuf +){ + int nData = pMulti->base.nData; + assert( nData>0 ); + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nData*2); + fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); + } +} + + +static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ + u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; + + assert( pIter->aPoslist ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + pIter->nSize = fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); + }else{ + pIter->nPoslist = ((int)(p[0])) >> 1; + pIter->nSize = 1; + } + + pIter->aPoslist = p; + } +} + +static void fts5DoclistIterInit( + Fts5Buffer *pBuf, + Fts5DoclistIter *pIter +){ + memset(pIter, 0, sizeof(*pIter)); + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; + fts5DoclistIterNext(pIter); +} + +#if 0 +/* +** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. +*/ +static void fts5MergeAppendDocid( + Fts5Buffer *pBuf, /* Buffer to write to */ + i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ + i64 iRowid /* Rowid to append */ +){ + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); + *piLastRowid = iRowid; +} +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} + +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ + Fts5Buffer tmp = *p1; + *p1 = *p2; + *p2 = tmp; +} + +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ + int i = *piOff; + if( i>=pBuf->n ){ + *piOff = -1; + }else{ + u64 iVal; + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); + *piRowid += iVal; + } +} + +/* +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. +** In this case the buffers consist of a delta-encoded list of rowids only. +*/ +static void fts5MergeRowidLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + Fts5Buffer *p2 /* Second list to merge */ +){ + int i1 = 0; + int i2 = 0; + i64 iRowid1 = 0; + i64 iRowid2 = 0; + i64 iOut = 0; + + Fts5Buffer out; + memset(&out, 0, sizeof(out)); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); + if( p->rc ) return; + + fts5NextRowid(p1, &i1, &iRowid1); + fts5NextRowid(p2, &i2, &iRowid2); + while( i1>=0 || i2>=0 ){ + if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){ + assert( iOut==0 || iRowid1>iOut ); + fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); + iOut = iRowid1; + fts5NextRowid(p1, &i1, &iRowid1); + }else{ + assert( iOut==0 || iRowid2>iOut ); + fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); + iOut = iRowid2; + if( i1>=0 && iRowid1==iRowid2 ){ + fts5NextRowid(p1, &i1, &iRowid1); + } + fts5NextRowid(p2, &i2, &iRowid2); + } + } + + fts5BufferSwap(&out, p1); + fts5BufferFree(&out); +} + +/* +** Buffers p1 and p2 contain doclists. This function merges the content +** of the two doclists together and sets buffer p1 to the result before +** returning. +** +** If an error occurs, an error code is left in p->rc. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5MergePrefixLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + Fts5Buffer *p2 /* Second list to merge */ +){ + if( p2->n ){ + i64 iLastRowid = 0; + Fts5DoclistIter i1; + Fts5DoclistIter i2; + Fts5Buffer out = {0, 0, 0}; + Fts5Buffer tmp = {0, 0, 0}; + + if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; + fts5DoclistIterInit(p1, &i1); + fts5DoclistIterInit(p2, &i2); + + while( 1 ){ + if( i1.iRowid<i2.iRowid ){ + /* Copy entry from i1 */ + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); + fts5DoclistIterNext(&i1); + if( i1.aPoslist==0 ) break; + } + else if( i2.iRowid!=i1.iRowid ){ + /* Copy entry from i2 */ + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); + fts5DoclistIterNext(&i2); + if( i2.aPoslist==0 ) break; + } + else{ + /* Merge the two position lists. */ + i64 iPos1 = 0; + i64 iPos2 = 0; + int iOff1 = 0; + int iOff2 = 0; + u8 *a1 = &i1.aPoslist[i1.nSize]; + u8 *a2 = &i2.aPoslist[i2.nSize]; + + i64 iPrev = 0; + Fts5PoslistWriter writer; + memset(&writer, 0, sizeof(writer)); + + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferZero(&tmp); + sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist); + if( p->rc ) break; + + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + assert( iPos1>=0 && iPos2>=0 ); + + if( iPos1<iPos2 ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + }else{ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + } + + if( iPos1>=0 && iPos2>=0 ){ + while( 1 ){ + if( iPos1<iPos2 ){ + if( iPos1!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); + } + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + if( iPos1<0 ) break; + }else{ + assert( iPos2!=iPrev ); + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + if( iPos2<0 ) break; + } + } + } + + if( iPos1>=0 ){ + if( iPos1!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); + } + fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1); + }else{ + assert( iPos2>=0 && iPos2!=iPrev ); + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); + fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2); + } + + /* WRITEPOSLISTSIZE */ + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + fts5DoclistIterNext(&i1); + fts5DoclistIterNext(&i2); + if( i1.aPoslist==0 || i2.aPoslist==0 ) break; + } + } + + if( i1.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); + } + else if( i2.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); + } + + fts5BufferSet(&p->rc, p1, out.n, out.p); + fts5BufferFree(&tmp); + fts5BufferFree(&out); + } +} + +static void fts5SetupPrefixIter( + Fts5Index *p, /* Index to read from */ + int bDesc, /* True for "ORDER BY rowid DESC" */ + const u8 *pToken, /* Buffer containing prefix to match */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset, /* Restrict matches to these columns */ + Fts5Iter **ppIter /* OUT: New iterator */ +){ + Fts5Structure *pStruct; + Fts5Buffer *aBuf; + const int nBuf = 32; + + void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + xMerge = fts5MergeRowidLists; + xAppend = fts5AppendRowid; + }else{ + xMerge = fts5MergePrefixLists; + xAppend = fts5AppendPoslist; + } + + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); + pStruct = fts5StructureRead(p); + + if( aBuf && pStruct ){ + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + int i; + i64 iLastRowid = 0; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + Fts5Data *pData; + Fts5Buffer doclist; + int bNewTerm = 1; + + memset(&doclist, 0, sizeof(doclist)); + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); + + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); + if( bNewTerm ){ + if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; + } + + if( p1->base.nData==0 ) continue; + + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ + for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ + assert( i<nBuf ); + if( aBuf[i].n==0 ){ + fts5BufferSwap(&doclist, &aBuf[i]); + fts5BufferZero(&doclist); + }else{ + xMerge(p, &doclist, &aBuf[i]); + fts5BufferZero(&aBuf[i]); + } + } + iLastRowid = 0; + } + + xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; + } + + for(i=0; i<nBuf; i++){ + if( p->rc==SQLITE_OK ){ + xMerge(p, &doclist, &aBuf[i]); + } + fts5BufferFree(&aBuf[i]); + } + fts5MultiIterFree(p1); + + pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); + if( pData ){ + pData->p = (u8*)&pData[1]; + pData->nn = pData->szLeaf = doclist.n; + memcpy(pData->p, doclist.p, doclist.n); + fts5MultiIterNew2(p, pData, bDesc, ppIter); + } + fts5BufferFree(&doclist); + } + + fts5StructureRelease(pStruct); + sqlite3_free(aBuf); +} + + +/* +** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain +** to the document with rowid iRowid. +*/ +static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ + assert( p->rc==SQLITE_OK ); + + /* Allocate the hash table if it has not already been allocated */ + if( p->pHash==0 ){ + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); + } + + /* Flush the hash table to disk if required */ + if( iRowid<p->iWriteRowid + || (iRowid==p->iWriteRowid && p->bDelete==0) + || (p->nPendingData > p->pConfig->nHashSize) + ){ + fts5IndexFlush(p); + } + + p->iWriteRowid = iRowid; + p->bDelete = bDelete; + return fts5IndexReturn(p); +} + +/* +** Commit data to disk. +*/ +static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + if( bCommit ) fts5CloseReader(p); + return fts5IndexReturn(p); +} + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +static int sqlite3Fts5IndexRollback(Fts5Index *p){ + fts5CloseReader(p); + fts5IndexDiscardData(p); + fts5StructureInvalidate(p); + /* assert( p->rc==SQLITE_OK ); */ + return SQLITE_OK; +} + +/* +** The %_data table is completely empty when this function is called. This +** function populates it with the initial structure objects for each index, +** and the initial version of the "averages" record (a zero-byte blob). +*/ +static int sqlite3Fts5IndexReinit(Fts5Index *p){ + Fts5Structure s; + fts5StructureInvalidate(p); + memset(&s, 0, sizeof(Fts5Structure)); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); + fts5StructureWrite(p, &s); + return fts5IndexReturn(p); +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying %_data table. +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +static int sqlite3Fts5IndexOpen( + Fts5Config *pConfig, + int bCreate, + Fts5Index **pp, + char **pzErr +){ + int rc = SQLITE_OK; + Fts5Index *p; /* New object */ + + *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); + if( rc==SQLITE_OK ){ + p->pConfig = pConfig; + p->nWorkUnit = FTS5_WORK_UNIT; + p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); + if( p->zDataTbl && bCreate ){ + rc = sqlite3Fts5CreateTable( + pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", + 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p); + } + } + } + + assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); + if( rc ){ + sqlite3Fts5IndexClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). +*/ +static int sqlite3Fts5IndexClose(Fts5Index *p){ + int rc = SQLITE_OK; + if( p ){ + assert( p->pReader==0 ); + fts5StructureInvalidate(p); + sqlite3_finalize(p->pWriter); + sqlite3_finalize(p->pDeleter); + sqlite3_finalize(p->pIdxWriter); + sqlite3_finalize(p->pIdxDeleter); + sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pDataVersion); + sqlite3Fts5HashFree(p->pHash); + sqlite3_free(p->zDataTbl); + sqlite3_free(p); + } + return rc; +} + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +static int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +){ + int n = 0; + int i; + for(i=0; i<nChar; i++){ + if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ + if( (unsigned char)p[n++]>=0xc0 ){ + while( (p[n] & 0xc0)==0x80 ) n++; + } + } + return n; +} + +/* +** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of +** unicode characters in the string. +*/ +static int fts5IndexCharlen(const char *pIn, int nIn){ + int nChar = 0; + int i = 0; + while( i<nIn ){ + if( (unsigned char)pIn[i++]>=0xc0 ){ + while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++; + } + nChar++; + } + return nChar; +} + +/* +** Insert or remove data to or from the index. Each time a document is +** added to or removed from the index, this function is called one or more +** times. +** +** For an insert, it must be called once for each token in the new document. +** If the operation is a delete, it must be called (at least) once for each +** unique token in the document with an iCol value less than zero. The iPos +** argument is ignored for a delete. +*/ +static int sqlite3Fts5IndexWrite( + Fts5Index *p, /* Index to write to */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + int i; /* Used to iterate through indexes */ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = p->pConfig; + + assert( p->rc==SQLITE_OK ); + assert( (iCol<0)==p->bDelete ); + + /* Add the entry to the main terms index. */ + rc = sqlite3Fts5HashWrite( + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken + ); + + for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ + const int nChar = pConfig->aPrefix[i]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5HashWrite(p->pHash, + p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, + nByte + ); + } + } + + return rc; +} + +/* +** Open a new iterator to iterate though all rowid that match the +** specified token or token prefix. +*/ +static int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +){ + Fts5Config *pConfig = p->pConfig; + Fts5Iter *pRet = 0; + Fts5Buffer buf = {0, 0, 0}; + + /* If the QUERY_SCAN flag is set, all other flags must be clear. */ + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); + + if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ + int iIdx = 0; /* Index to search */ + memcpy(&buf.p[1], pToken, nToken); + + /* Figure out which index to search and set iIdx accordingly. If this + ** is a prefix query for which there is no prefix index, set iIdx to + ** greater than pConfig->nPrefix to indicate that the query will be + ** satisfied by scanning multiple terms in the main index. + ** + ** If the QUERY_TEST_NOIDX flag was specified, then this must be a + ** prefix-query. Instead of using a prefix-index (if one exists), + ** evaluate the prefix query using the main FTS index. This is used + ** for internal sanity checking by the integrity-check in debug + ** mode only. */ +#ifdef SQLITE_DEBUG + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ + assert( flags & FTS5INDEX_QUERY_PREFIX ); + iIdx = 1+pConfig->nPrefix; + }else +#endif + if( flags & FTS5INDEX_QUERY_PREFIX ){ + int nChar = fts5IndexCharlen(pToken, nToken); + for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ + if( pConfig->aPrefix[iIdx-1]==nChar ) break; + } + } + + if( iIdx<=pConfig->nPrefix ){ + /* Straight index lookup */ + Fts5Structure *pStruct = fts5StructureRead(p); + buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); + if( pStruct ){ + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + pColset, buf.p, nToken+1, -1, 0, &pRet + ); + fts5StructureRelease(pStruct); + } + }else{ + /* Scan multiple terms in the main index */ + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; + buf.p[0] = FTS5_MAIN_PREFIX; + fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); + assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } + } + + if( p->rc ){ + sqlite3Fts5IterClose(&pRet->base); + pRet = 0; + fts5CloseReader(p); + } + + *ppIter = &pRet->base; + sqlite3Fts5BufferFree(&buf); + } + return fts5IndexReturn(p); +} + +/* +** Return true if the iterator passed as the only argument is at EOF. +*/ +/* +** Move to the next matching rowid. +*/ +static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + assert( pIter->pIndex->rc==SQLITE_OK ); + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching term/rowid. Used by the fts5vocab module. +*/ +static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *p = pIter->pIndex; + + assert( pIter->pIndex->rc==SQLITE_OK ); + + fts5MultiIterNext(p, pIter, 0, 0); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; + pIter->base.bEof = 1; + } + } + + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching rowid that occurs at or after iMatch. The +** definition of "at or after" depends on whether this iterator iterates +** in ascending or descending rowid order. +*/ +static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Return the current term. +*/ +static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ + int n; + const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); + *pn = n-1; + return &z[1]; +} + +/* +** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). +*/ +static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *pIndex = pIter->pIndex; + fts5MultiIterFree(pIter); + fts5CloseReader(pIndex); + } +} + +/* +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. +*/ +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->nn ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; i<pData->nn && iCol<nCol; iCol++){ + i += fts5GetVarint(&pData->p[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); + return fts5IndexReturn(p); +} + +/* +** Replace the current "averages" record with the contents of the buffer +** supplied as the second argument. +*/ +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ + assert( p->rc==SQLITE_OK ); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); + return fts5IndexReturn(p); +} + +/* +** Return the total number of blocks this module has read from the %_data +** table since it was created. +*/ +static int sqlite3Fts5IndexReads(Fts5Index *p){ + return p->nRead; +} + +/* +** Set the 32-bit cookie value stored at the start of all structure +** records to the value passed as the second argument. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ + int rc; /* Return code */ + Fts5Config *pConfig = p->pConfig; /* Configuration object */ + u8 aCookie[4]; /* Binary representation of iNew */ + sqlite3_blob *pBlob = 0; + + assert( p->rc==SQLITE_OK ); + sqlite3Fts5Put32(aCookie, iNew); + + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob + ); + if( rc==SQLITE_OK ){ + sqlite3_blob_write(pBlob, aCookie, 4, 0); + rc = sqlite3_blob_close(pBlob); + } + + return rc; +} + +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + fts5StructureRelease(pStruct); + return fts5IndexReturn(p); +} + + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the integrity-check +** functionality. +*/ + +/* +** Return a simple checksum value based on the arguments. +*/ +static u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +){ + int i; + u64 ret = iRowid; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); + for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i]; + return ret; +} + +#ifdef SQLITE_DEBUG +/* +** This function is purely an internal test. It does not contribute to +** FTS functionality, or even the integrity-check, in any way. +** +** Instead, it tests that the same set of pgno/rowid combinations are +** visited regardless of whether the doclist-index identified by parameters +** iSegid/iLeaf is iterated in forwards or reverse order. +*/ +static void fts5TestDlidxReverse( + Fts5Index *p, + int iSegid, /* Segment id to load from */ + int iLeaf /* Load doclist-index for this leaf */ +){ + Fts5DlidxIter *pDlidx = 0; + u64 cksum1 = 13; + u64 cksum2 = 13; + + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterNext(p, pDlidx) + ){ + i64 iRowid = fts5DlidxIterRowid(pDlidx); + int pgno = fts5DlidxIterPgno(pDlidx); + assert( pgno>iLeaf ); + cksum1 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterPrev(p, pDlidx) + ){ + i64 iRowid = fts5DlidxIterRowid(pDlidx); + int pgno = fts5DlidxIterPgno(pDlidx); + assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); + cksum2 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; +} + +static int fts5QueryCksum( + Fts5Index *p, /* Fts5 index object */ + int iIdx, + const char *z, /* Index key to query for */ + int n, /* Size of index key in bytes */ + int flags, /* Flags for Fts5IndexQuery */ + u64 *pCksum /* IN/OUT: Checksum value */ +){ + int eDetail = p->pConfig->eDetail; + u64 cksum = *pCksum; + Fts5IndexIter *pIter = 0; + int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + + while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ + i64 rowid = pIter->iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); + }else{ + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNext(pIter); + } + } + sqlite3Fts5IterClose(pIter); + + *pCksum = cksum; + return rc; +} + + +/* +** This function is also purely an internal test. It does not contribute to +** FTS functionality, or even the integrity-check, in any way. +*/ +static void fts5TestTerm( + Fts5Index *p, + Fts5Buffer *pPrev, /* Previous term */ + const char *z, int n, /* Possibly new term to test */ + u64 expected, + u64 *pCksum +){ + int rc = p->rc; + if( pPrev->n==0 ){ + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + }else + if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + u64 cksum3 = *pCksum; + const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ + int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ + int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX); + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); + u64 ck1 = 0; + u64 ck2 = 0; + + /* Check that the results returned for ASC and DESC queries are + ** the same. If not, call this corruption. */ + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); + if( rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_DESC; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + } + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + + /* If this is a prefix query, check that the results returned if the + ** the index is disabled are the same. In both ASC and DESC order. + ** + ** This check may only be performed if the hash table is empty. This + ** is because the hash table only supports a single scan query at + ** a time, and the multi-iter loop from which this function is called + ** is already performing such a scan. */ + if( p->nPendingData==0 ){ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + } + + cksum3 ^= ck1; + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + + if( rc==SQLITE_OK && cksum3!=expected ){ + rc = FTS5_CORRUPT; + } + *pCksum = cksum3; + } + p->rc = rc; +} + +#else +# define fts5TestDlidxReverse(x,y,z) +# define fts5TestTerm(u,v,w,x,y,z) +#endif + +/* +** Check that: +** +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and +** contain zero terms. +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and +** contain zero rowids. +*/ +static void fts5IndexIntegrityCheckEmpty( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ + int iFirst, + int iNoRowid, + int iLast +){ + int i; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + if( pLeaf ){ + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + } +} + +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ + int iTermOff = 0; + int ii; + + Fts5Buffer buf1 = {0,0,0}; + Fts5Buffer buf2 = {0,0,0}; + + ii = pLeaf->szLeaf; + while( ii<pLeaf->nn && p->rc==SQLITE_OK ){ + int res; + int iOff; + int nIncr; + + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); + iTermOff += nIncr; + iOff = iTermOff; + + if( iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( iTermOff==nIncr ){ + int nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + }else{ + int nKeep, nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + buf1.n = nKeep; + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + + if( p->rc==SQLITE_OK ){ + res = fts5BufferCompare(&buf1, &buf2); + if( res<=0 ) p->rc = FTS5_CORRUPT; + } + } + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); + } + + fts5BufferFree(&buf1); + fts5BufferFree(&buf2); +} + +static void fts5IndexIntegrityCheckSegment( + Fts5Index *p, /* FTS5 backend object */ + Fts5StructureSegment *pSeg /* Segment to check internal consistency */ +){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + int rc2; + int iIdxPrevLeaf = pSeg->pgnoFirst-1; + int iDlidxPrevLeaf = pSeg->pgnoLast; + + if( pSeg->pgnoFirst==0 ) return; + + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d", + pConfig->zDb, pConfig->zName, pSeg->iSegid + )); + + /* Iterate through the b-tree hierarchy. */ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iRow; /* Rowid for this leaf */ + Fts5Data *pLeaf; /* Data for this leaf */ + + int nIdxTerm = sqlite3_column_bytes(pStmt, 1); + const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); + int iIdxLeaf = sqlite3_column_int(pStmt, 2); + int bIdxDlidx = sqlite3_column_int(pStmt, 3); + + /* If the leaf in question has already been trimmed from the segment, + ** ignore this b-tree entry. Otherwise, load it into memory. */ + if( iIdxLeaf<pSeg->pgnoFirst ) continue; + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); + pLeaf = fts5DataRead(p, iRow); + if( pLeaf==0 ) break; + + /* Check that the leaf contains at least one term, and that it is equal + ** to or larger than the split-key in zIdxTerm. Also check that if there + ** is also a rowid pointer within the leaf page header, it points to a + ** location before the term. */ + if( pLeaf->nn<=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + int iOff; /* Offset of first term on leaf */ + int iRowidOff; /* Offset of first rowid on leaf */ + int nTerm; /* Size of term on leaf in bytes */ + int res; /* Comparison of term and split-key */ + + iOff = fts5LeafFirstTermOff(pLeaf); + iRowidOff = fts5LeafFirstRowidOff(pLeaf); + if( iRowidOff>=iOff ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); + res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + if( res==0 ) res = nTerm - nIdxTerm; + if( res<0 ) p->rc = FTS5_CORRUPT; + } + + fts5IntegrityCheckPgidx(p, pLeaf); + } + fts5DataRelease(pLeaf); + if( p->rc ) break; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + fts5IndexIntegrityCheckEmpty( + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 + ); + if( p->rc ) break; + + /* If there is a doclist-index, check that it looks right. */ + if( bIdxDlidx ){ + Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ + int iPrevLeaf = iIdxLeaf; + int iSegid = pSeg->iSegid; + int iPg = 0; + i64 iKey; + + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterNext(p, pDlidx) + ){ + + /* Check any rowid-less pages that occur before the current leaf. */ + for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){ + iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); + pLeaf = fts5DataRead(p, iKey); + if( pLeaf ){ + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + fts5DataRelease(pLeaf); + } + } + iPrevLeaf = fts5DlidxIterPgno(pDlidx); + + /* Check that the leaf page indicated by the iterator really does + ** contain the rowid suggested by the same. */ + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); + pLeaf = fts5DataRead(p, iKey); + if( pLeaf ){ + i64 iRowid; + int iRowidOff = fts5LeafFirstRowidOff(pLeaf); + ASSERT_SZLEAF_OK(pLeaf); + if( iRowidOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); + if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + } + } + + iDlidxPrevLeaf = iPg; + fts5DlidxIterFree(pDlidx); + fts5TestDlidxReverse(p, iSegid, iIdxLeaf); + }else{ + iDlidxPrevLeaf = pSeg->pgnoLast; + /* TODO: Check there is no doclist index */ + } + + iIdxPrevLeaf = iIdxLeaf; + } + + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ +#if 0 + if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + } +#endif +} + + +/* +** Run internal checks to ensure that the FTS index (a) is internally +** consistent and (b) contains entries for which the XOR of the checksums +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. +** +** Return SQLITE_CORRUPT if any of the internal checks fail, or if the +** checksum does not match. Return SQLITE_OK if all checks pass without +** error, or some other SQLite error code if another error (e.g. OOM) +** occurs. +*/ +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ + int eDetail = p->pConfig->eDetail; + u64 cksum2 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ + Fts5Iter *pIter; /* Used to iterate through entire index */ + Fts5Structure *pStruct; /* Index structure */ + +#ifdef SQLITE_DEBUG + /* Used by extra internal tests only run if NDEBUG is not defined */ + u64 cksum3 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ +#endif + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + + /* Load the FTS index structure */ + pStruct = fts5StructureRead(p); + + /* Check that the internal nodes of each segment match the leaves */ + if( pStruct ){ + int iLvl, iSeg; + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5IndexIntegrityCheckSegment(p, pSeg); + } + } + } + + /* The cksum argument passed to this function is a checksum calculated + ** based on all expected entries in the FTS index (including prefix index + ** entries). This block checks that a checksum calculated based on the + ** actual contents of FTS index is identical. + ** + ** Two versions of the same checksum are calculated. The first (stack + ** variable cksum2) based on entries extracted from the full-text index + ** while doing a linear scan of each individual index in turn. + ** + ** As each term visited by the linear scans, a separate query for the + ** same term is performed. cksum3 is calculated based on the entries + ** extracted by these queries. + */ + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + int n; /* Size of term in bytes */ + i64 iPos = 0; /* Position read from poslist */ + int iOff = 0; /* Offset within poslist */ + i64 iRowid = fts5MultiIterRowid(pIter); + char *z = (char*)fts5MultiIterTerm(pIter, &n); + + /* If this is a new term, query for it. Update cksum3 with the results. */ + fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } + }else{ + poslist.n = 0; + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } + } + } + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + + fts5MultiIterFree(pIter); + if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; + + fts5StructureRelease(pStruct); +#ifdef SQLITE_DEBUG + fts5BufferFree(&term); +#endif + fts5BufferFree(&poslist); + return fts5IndexReturn(p); +} + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the fts5_decode() scalar +** function only. +*/ + +/* +** Decode a segment-data rowid from the %_data table. This function is +** the opposite of macro FTS5_SEGMENT_ROWID(). +*/ +static void fts5DecodeRowid( + i64 iRowid, /* Rowid from %_data table */ + int *piSegid, /* OUT: Segment id */ + int *pbDlidx, /* OUT: Dlidx flag */ + int *piHeight, /* OUT: Height */ + int *piPgno /* OUT: Page number */ +){ + *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); + iRowid >>= FTS5_DATA_PAGE_B; + + *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); + iRowid >>= FTS5_DATA_HEIGHT_B; + + *pbDlidx = (int)(iRowid & 0x0001); + iRowid >>= FTS5_DATA_DLI_B; + + *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); +} + +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ + int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ + fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + + if( iSegid==0 ){ + if( iKey==FTS5_AVERAGES_ROWID ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); + }else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); + } + } + else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + ); + } +} + +static void fts5DebugStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + Fts5Structure *p +){ + int iLvl, iSeg; /* Iterate through levels, segments */ + + for(iLvl=0; iLvl<p->nLevel; iLvl++){ + Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg + ); + for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); + } +} + +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This +** function appends a human-readable representation of the same object +** to the buffer passed as the second argument. +*/ +static void fts5DecodeStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int rc; /* Return code */ + Fts5Structure *p = 0; /* Decoded structure object */ + + rc = fts5StructureDecode(pBlob, nBlob, 0, &p); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + + fts5DebugStructure(pRc, pBuf, p); + fts5StructureRelease(p); +} + +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain an "averages" record. This function +** appends a human-readable representation of record to the buffer passed +** as the second argument. +*/ +static void fts5DecodeAverages( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int i = 0; + const char *zSpace = ""; + + while( i<nBlob ){ + u64 iVal; + i += sqlite3Fts5GetVarint(&pBlob[i], &iVal); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal); + zSpace = " "; + } +} + +/* +** Buffer (a/n) is assumed to contain a list of serialized varints. Read +** each varint and append its string representation to buffer pBuf. Return +** after either the input buffer is exhausted or a 0 value is read. +** +** The return value is the number of bytes read from the input buffer. +*/ +static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ + int iOff = 0; + while( iOff<n ){ + int iVal; + iOff += fts5GetVarint32(&a[iOff], iVal); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal); + } + return iOff; +} + +/* +** The start of buffer (a/n) contains the start of a doclist. The doclist +** may or may not finish within the buffer. This function appends a text +** representation of the part of the doclist that is present to buffer +** pBuf. +** +** The return value is the number of bytes read from the input buffer. +*/ +static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ + i64 iDocid = 0; + int iOff = 0; + + if( n>0 ){ + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } + while( iOff<n ){ + int nPos; + int bDel; + iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":""); + iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)); + if( iOff<n ){ + i64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta); + iDocid += iDelta; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } + } + + return iOff; +} + +/* +** This function is part of the fts5_decode() debugging function. It is +** only ever used with detail=none tables. +** +** Buffer (pData/nData) contains a doclist in the format used by detail=none +** tables. This function appends a human-readable version of that list to +** buffer pBuf. +** +** If *pRc is other than SQLITE_OK when this function is called, it is a +** no-op. If an OOM or other error occurs within this function, *pRc is +** set to an SQLite error code before returning. The final state of buffer +** pBuf is undefined in this case. +*/ +static void fts5DecodeRowidList( + int *pRc, /* IN/OUT: Error code */ + Fts5Buffer *pBuf, /* Buffer to append text to */ + const u8 *pData, int nData /* Data to decode list-of-rowids from */ +){ + int i = 0; + i64 iRowid = 0; + + while( i<nData ){ + const char *zApp = ""; + u64 iVal; + i += sqlite3Fts5GetVarint(&pData[i], &iVal); + iRowid += iVal; + + if( i<nData && pData[i]==0x00 ){ + i++; + if( i<nData && pData[i]==0x00 ){ + i++; + zApp = "+"; + }else{ + zApp = "*"; + } + } + + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); + } +} + +/* +** The implementation of user-defined scalar function fts5_decode(). +*/ +static void fts5DecodeFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args (always 2) */ + sqlite3_value **apVal /* Function arguments */ +){ + i64 iRowid; /* Rowid for record being decoded */ + int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + const u8 *aBlob; int n; /* Record to decode */ + u8 *a = 0; + Fts5Buffer s; /* Build up text to return here */ + int rc = SQLITE_OK; /* Return code */ + int nSpace = 0; + int eDetailNone = (sqlite3_user_data(pCtx)!=0); + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + memset(&s, 0, sizeof(Fts5Buffer)); + iRowid = sqlite3_value_int64(apVal[0]); + + /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[] + ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents + ** buffer overreads even if the record is corrupt. */ + n = sqlite3_value_bytes(apVal[1]); + aBlob = sqlite3_value_blob(apVal[1]); + nSpace = n + FTS5_DATA_ZERO_PADDING; + a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); + if( a==0 ) goto decode_out; + memcpy(a, aBlob, n); + + + fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + + fts5DebugRowid(&rc, &s, iRowid); + if( bDlidx ){ + Fts5Data dlidx; + Fts5DlidxLvl lvl; + + dlidx.p = a; + dlidx.nn = n; + + memset(&lvl, 0, sizeof(Fts5DlidxLvl)); + lvl.pData = &dlidx; + lvl.iLeafPgno = iPgno; + + for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, + " %d(%lld)", lvl.iLeafPgno, lvl.iRowid + ); + } + }else if( iSegid==0 ){ + if( iRowid==FTS5_AVERAGES_ROWID ){ + fts5DecodeAverages(&rc, &s, a, n); + }else{ + fts5DecodeStructure(&rc, &s, a, n); + } + }else if( eDetailNone ){ + Fts5Buffer term; /* Current term read from page */ + int szLeaf; + int iPgidxOff = szLeaf = fts5GetU16(&a[2]); + int iTermOff; + int nKeep = 0; + int iOff; + + memset(&term, 0, sizeof(Fts5Buffer)); + + /* Decode any entries that occur before the first term. */ + if( szLeaf<n ){ + iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff); + }else{ + iTermOff = szLeaf; + } + fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4); + + iOff = iTermOff; + while( iOff<szLeaf ){ + int nAppend; + + /* Read the term data for the next term*/ + iOff += fts5GetVarint32(&a[iOff], nAppend); + term.n = nKeep; + fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]); + sqlite3Fts5BufferAppendPrintf( + &rc, &s, " term=%.*s", term.n, (const char*)term.p + ); + iOff += nAppend; + + /* Figure out where the doclist for this term ends */ + if( iPgidxOff<n ){ + int nIncr; + iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr); + iTermOff += nIncr; + }else{ + iTermOff = szLeaf; + } + + fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff); + iOff = iTermOff; + if( iOff<szLeaf ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); + } + } + + fts5BufferFree(&term); + }else{ + Fts5Buffer term; /* Current term read from page */ + int szLeaf; /* Offset of pgidx in a[] */ + int iPgidxOff; + int iPgidxPrev = 0; /* Previous value read from pgidx */ + int iTermOff = 0; + int iRowidOff = 0; + int iOff; + int nDoclist; + + memset(&term, 0, sizeof(Fts5Buffer)); + + if( n<4 ){ + sqlite3Fts5BufferSet(&rc, &s, 7, (const u8*)"corrupt"); + goto decode_out; + }else{ + iRowidOff = fts5GetU16(&a[0]); + iPgidxOff = szLeaf = fts5GetU16(&a[2]); + if( iPgidxOff<n ){ + fts5GetVarint32(&a[iPgidxOff], iTermOff); + } + } + + /* Decode the position list tail at the start of the page */ + if( iRowidOff!=0 ){ + iOff = iRowidOff; + }else if( iTermOff!=0 ){ + iOff = iTermOff; + }else{ + iOff = szLeaf; + } + fts5DecodePoslist(&rc, &s, &a[4], iOff-4); + + /* Decode any more doclist data that appears on the page before the + ** first term. */ + nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff; + fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist); + + while( iPgidxOff<n ){ + int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */ + int nByte; /* Bytes of data */ + int iEnd; + + iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte); + iPgidxPrev += nByte; + iOff = iPgidxPrev; + + if( iPgidxOff<n ){ + fts5GetVarint32(&a[iPgidxOff], nByte); + iEnd = iPgidxPrev + nByte; + }else{ + iEnd = szLeaf; + } + + if( bFirst==0 ){ + iOff += fts5GetVarint32(&a[iOff], nByte); + term.n = nByte; + } + iOff += fts5GetVarint32(&a[iOff], nByte); + fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); + iOff += nByte; + + sqlite3Fts5BufferAppendPrintf( + &rc, &s, " term=%.*s", term.n, (const char*)term.p + ); + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff); + } + + fts5BufferFree(&term); + } + + decode_out: + sqlite3_free(a); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + fts5BufferFree(&s); +} + +/* +** The implementation of user-defined scalar function fts5_rowid(). +*/ +static void fts5RowidFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args (always 2) */ + sqlite3_value **apVal /* Function arguments */ +){ + const char *zArg; + if( nArg==0 ){ + sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); + }else{ + zArg = (const char*)sqlite3_value_text(apVal[0]); + if( 0==sqlite3_stricmp(zArg, "segment") ){ + i64 iRowid; + int segid, pgno; + if( nArg!=3 ){ + sqlite3_result_error(pCtx, + "should be: fts5_rowid('segment', segid, pgno))", -1 + ); + }else{ + segid = sqlite3_value_int(apVal[1]); + pgno = sqlite3_value_int(apVal[2]); + iRowid = FTS5_SEGMENT_ROWID(segid, pgno); + sqlite3_result_int64(pCtx, iRowid); + } + }else{ + sqlite3_result_error(pCtx, + "first arg to fts5_rowid() must be 'segment'" , -1 + ); + } + } +} + +/* +** This is called as part of registering the FTS5 module with database +** connection db. It registers several user-defined scalar functions useful +** with FTS5. +** +** If successful, SQLITE_OK is returned. If an error occurs, some other +** SQLite error code is returned instead. +*/ +static int sqlite3Fts5IndexInit(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 + ); + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_decode_none", 2, + SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 + ); + } + return rc; +} + + +static int sqlite3Fts5IndexReset(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} + +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + + +/* #include "fts5Int.h" */ + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts5 code are activated - conditions that are +** only true if it is guaranteed that the fts5 database is not corrupt. +*/ +SQLITE_API int sqlite3_fts5_may_be_corrupt = 1; + + +typedef struct Fts5Auxdata Fts5Auxdata; +typedef struct Fts5Auxiliary Fts5Auxiliary; +typedef struct Fts5Cursor Fts5Cursor; +typedef struct Fts5Sorter Fts5Sorter; +typedef struct Fts5Table Fts5Table; +typedef struct Fts5TokenizerModule Fts5TokenizerModule; + +/* +** NOTES ON TRANSACTIONS: +** +** SQLite invokes the following virtual table methods as transactions are +** opened and closed by the user: +** +** xBegin(): Start of a new transaction. +** xSync(): Initial part of two-phase commit. +** xCommit(): Final part of two-phase commit. +** xRollback(): Rollback the transaction. +** +** Anything that is required as part of a commit that may fail is performed +** in the xSync() callback. Current versions of SQLite ignore any errors +** returned by xCommit(). +** +** And as sub-transactions are opened/closed: +** +** xSavepoint(int S): Open savepoint S. +** xRelease(int S): Commit and close savepoint S. +** xRollbackTo(int S): Rollback to start of savepoint S. +** +** During a write-transaction the fts5_index.c module may cache some data +** in-memory. It is flushed to disk whenever xSync(), xRelease() or +** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() +** is called. +** +** Additionally, if SQLITE_DEBUG is defined, an instance of the following +** structure is used to record the current transaction state. This information +** is not required, but it is used in the assert() statements executed by +** function fts5CheckTransactionState() (see below). +*/ +struct Fts5TransactionState { + int eState; /* 0==closed, 1==open, 2==synced */ + int iSavepoint; /* Number of open savepoints (0 -> none) */ +}; + +/* +** A single object of this type is allocated when the FTS5 module is +** registered with a database handle. It is used to store pointers to +** all registered FTS5 extensions - tokenizers and auxiliary functions. +*/ +struct Fts5Global { + fts5_api api; /* User visible part of object (see fts5.h) */ + sqlite3 *db; /* Associated database connection */ + i64 iNextId; /* Used to allocate unique cursor ids */ + Fts5Auxiliary *pAux; /* First in list of all aux. functions */ + Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ + Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ + Fts5Cursor *pCsr; /* First in list of all open cursors */ +}; + +/* +** Each auxiliary function registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pAux list. +*/ +struct Fts5Auxiliary { + Fts5Global *pGlobal; /* Global context for this function */ + char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc; /* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5Auxiliary *pNext; /* Next registered auxiliary function */ +}; + +/* +** Each tokenizer module registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pTok list. +*/ +struct Fts5TokenizerModule { + char *zName; /* Name of tokenizer */ + void *pUserData; /* User pointer passed to xCreate() */ + fts5_tokenizer x; /* Tokenizer functions */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ +}; + +/* +** Virtual-table object. +*/ +struct Fts5Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts5Config *pConfig; /* Virtual table configuration */ + Fts5Index *pIndex; /* Full-text index */ + Fts5Storage *pStorage; /* Document store */ + Fts5Global *pGlobal; /* Global (connection wide) data */ + Fts5Cursor *pSortCsr; /* Sort data from this cursor */ +#ifdef SQLITE_DEBUG + struct Fts5TransactionState ts; +#endif +}; + +struct Fts5MatchPhrase { + Fts5Buffer *pPoslist; /* Pointer to current poslist */ + int nTerm; /* Size of phrase in terms */ +}; + +/* +** pStmt: +** SELECT rowid, <fts> FROM <fts> ORDER BY +rank; +** +** aIdx[]: +** There is one entry in the aIdx[] array for each phrase in the query, +** the value of which is the offset within aPoslist[] following the last +** byte of the position list for the corresponding phrase. +*/ +struct Fts5Sorter { + sqlite3_stmt *pStmt; + i64 iRowid; /* Current rowid */ + const u8 *aPoslist; /* Position lists for current row */ + int nIdx; /* Number of entries in aIdx[] */ + int aIdx[1]; /* Offsets into aPoslist for current row */ +}; + + +/* +** Virtual-table cursor object. +** +** iSpecial: +** If this is a 'special' query (refer to function fts5SpecialMatch()), +** then this variable contains the result of the query. +** +** iFirstRowid, iLastRowid: +** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the +** cursor iterates in ascending order of rowids, iFirstRowid is the lower +** limit of rowids to return, and iLastRowid the upper. In other words, the +** WHERE clause in the user's query might have been: +** +** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid +** +** If the cursor iterates in descending order of rowid, iFirstRowid +** is the upper limit (i.e. the "first" rowid visited) and iLastRowid +** the lower. +*/ +struct Fts5Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ + int *aColumnSize; /* Values for xColumnSize() */ + i64 iCsrId; /* Cursor id */ + + /* Zero from this point onwards on cursor reset */ + int ePlan; /* FTS5_PLAN_XXX value */ + int bDesc; /* True for "ORDER BY rowid DESC" queries */ + i64 iFirstRowid; /* Return no rowids earlier than this */ + i64 iLastRowid; /* Return no rowids later than this */ + sqlite3_stmt *pStmt; /* Statement used to read %_content */ + Fts5Expr *pExpr; /* Expression for MATCH queries */ + Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ + int csrflags; /* Mask of cursor flags (see below) */ + i64 iSpecial; /* Result of special query */ + + /* "rank" function. Populated on demand from vtab.xColumn(). */ + char *zRank; /* Custom rank function */ + char *zRankArgs; /* Custom rank function args */ + Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ + int nRankArg; /* Number of trailing arguments for rank() */ + sqlite3_value **apRankArg; /* Array of trailing arguments */ + sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ + + /* Auxiliary data storage */ + Fts5Auxiliary *pAux; /* Currently executing extension function */ + Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ + + /* Cache used by auxiliary functions xInst() and xInstCount() */ + Fts5PoslistReader *aInstIter; /* One for each phrase */ + int nInstAlloc; /* Size of aInst[] array (entries / 3) */ + int nInstCount; /* Number of phrase instances */ + int *aInst; /* 3 integers per phrase instance */ +}; + +/* +** Bits that make up the "idxNum" parameter passed indirectly by +** xBestIndex() to xFilter(). +*/ +#define FTS5_BI_MATCH 0x0001 /* <tbl> MATCH ? */ +#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */ +#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */ +#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */ +#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */ + +#define FTS5_BI_ORDER_RANK 0x0020 +#define FTS5_BI_ORDER_ROWID 0x0040 +#define FTS5_BI_ORDER_DESC 0x0080 + +/* +** Values for Fts5Cursor.csrflags +*/ +#define FTS5CSR_EOF 0x01 +#define FTS5CSR_REQUIRE_CONTENT 0x02 +#define FTS5CSR_REQUIRE_DOCSIZE 0x04 +#define FTS5CSR_REQUIRE_INST 0x08 +#define FTS5CSR_FREE_ZRANK 0x10 +#define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 + +#define BitFlagAllTest(x,y) (((x) & (y))==(y)) +#define BitFlagTest(x,y) (((x) & (y))!=0) + + +/* +** Macros to Set(), Clear() and Test() cursor flags. +*/ +#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag)) +#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag)) +#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag)) + +struct Fts5Auxdata { + Fts5Auxiliary *pAux; /* Extension to which this belongs */ + void *pPtr; /* Pointer value */ + void(*xDelete)(void*); /* Destructor */ + Fts5Auxdata *pNext; /* Next object in linked list */ +}; + +#ifdef SQLITE_DEBUG +#define FTS5_BEGIN 1 +#define FTS5_SYNC 2 +#define FTS5_COMMIT 3 +#define FTS5_ROLLBACK 4 +#define FTS5_SAVEPOINT 5 +#define FTS5_RELEASE 6 +#define FTS5_ROLLBACKTO 7 +static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ + switch( op ){ + case FTS5_BEGIN: + assert( p->ts.eState==0 ); + p->ts.eState = 1; + p->ts.iSavepoint = -1; + break; + + case FTS5_SYNC: + assert( p->ts.eState==1 ); + p->ts.eState = 2; + break; + + case FTS5_COMMIT: + assert( p->ts.eState==2 ); + p->ts.eState = 0; + break; + + case FTS5_ROLLBACK: + assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 ); + p->ts.eState = 0; + break; + + case FTS5_SAVEPOINT: + assert( p->ts.eState==1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint>p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint; + break; + + case FTS5_RELEASE: + assert( p->ts.eState==1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint<=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint-1; + break; + + case FTS5_ROLLBACKTO: + assert( p->ts.eState==1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint<=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint; + break; + } +} +#else +# define fts5CheckTransactionState(x,y,z) +#endif + +/* +** Return true if pTab is a contentless table. +*/ +static int fts5IsContentless(Fts5Table *pTab){ + return pTab->pConfig->eContent==FTS5_CONTENT_NONE; +} + +/* +** Delete a virtual table handle allocated by fts5InitVtab(). +*/ +static void fts5FreeVtab(Fts5Table *pTab){ + if( pTab ){ + sqlite3Fts5IndexClose(pTab->pIndex); + sqlite3Fts5StorageClose(pTab->pStorage); + sqlite3Fts5ConfigFree(pTab->pConfig); + sqlite3_free(pTab); + } +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5DisconnectMethod(sqlite3_vtab *pVtab){ + fts5FreeVtab((Fts5Table*)pVtab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5DestroyMethod(sqlite3_vtab *pVtab){ + Fts5Table *pTab = (Fts5Table*)pVtab; + int rc = sqlite3Fts5DropAll(pTab->pConfig); + if( rc==SQLITE_OK ){ + fts5FreeVtab((Fts5Table*)pVtab); + } + return rc; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts5InitVtab( + int bCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts5Global *pGlobal = (Fts5Global*)pAux; + const char **azConfig = (const char**)argv; + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = 0; /* Results of parsing argc/argv */ + Fts5Table *pTab = 0; /* New virtual table object */ + + /* Allocate the new vtab object and parse the configuration */ + pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table)); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); + assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); + } + if( rc==SQLITE_OK ){ + pTab->pConfig = pConfig; + pTab->pGlobal = pGlobal; + } + + /* Open the index sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr); + } + + /* Open the storage sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageOpen( + pConfig, pTab->pIndex, bCreate, &pTab->pStorage, pzErr + ); + } + + /* Call sqlite3_declare_vtab() */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigDeclareVtab(pConfig); + } + + /* Load the initial configuration */ + if( rc==SQLITE_OK ){ + assert( pConfig->pzErrmsg==0 ); + pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); + sqlite3Fts5IndexRollback(pTab->pIndex); + pConfig->pzErrmsg = 0; + } + + if( rc!=SQLITE_OK ){ + fts5FreeVtab(pTab); + pTab = 0; + }else if( bCreate ){ + fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); + } + *ppVTab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5InitVtab(). +*/ +static int fts5ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** The different query plans. +*/ +#define FTS5_PLAN_MATCH 1 /* (<tbl> MATCH ?) */ +#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */ +#define FTS5_PLAN_SPECIAL 3 /* An internal query */ +#define FTS5_PLAN_SORTED_MATCH 4 /* (<tbl> MATCH ? ORDER BY rank) */ +#define FTS5_PLAN_SCAN 5 /* No usable constraint */ +#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008012 ) +#endif + { + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS5 tables. Within the +** WHERE constraint, it searches for the following: +** +** 1. A MATCH constraint against the special column. +** 2. A MATCH constraint against the "rank" column. +** 3. An == constraint against the rowid column. +** 4. A < or <= constraint against the rowid column. +** 5. A > or >= constraint against the rowid column. +** +** Within the ORDER BY, either: +** +** 5. ORDER BY rank [ASC|DESC] +** 6. ORDER BY rowid [ASC|DESC] +** +** Costs are assigned as follows: +** +** a) If an unusable MATCH operator is present in the WHERE clause, the +** cost is unconditionally set to 1e50 (a really big number). +** +** a) If a MATCH operator is present, the cost depends on the other +** constraints also present. As follows: +** +** * No other constraints: cost=1000.0 +** * One rowid range constraint: cost=750.0 +** * Both rowid range constraints: cost=500.0 +** * An == rowid constraint: cost=100.0 +** +** b) Otherwise, if there is no MATCH: +** +** * No other constraints: cost=1000000.0 +** * One rowid range constraint: cost=750000.0 +** * Both rowid range constraints: cost=250000.0 +** * An == rowid constraint: cost=10.0 +** +** Costs are not modified by the ORDER BY clause. +*/ +static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts5Table *pTab = (Fts5Table*)pVTab; + Fts5Config *pConfig = pTab->pConfig; + int idxFlags = 0; /* Parameter passed through to xFilter() */ + int bHasMatch; + int iNext; + int i; + + struct Constraint { + int op; /* Mask against sqlite3_index_constraint.op */ + int fts5op; /* FTS5 mask for idxFlags */ + int iCol; /* 0==rowid, 1==tbl, 2==rank */ + int omit; /* True to omit this if found */ + int iConsIndex; /* Index in pInfo->aConstraint[] */ + } aConstraint[] = { + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, + FTS5_BI_MATCH, 1, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, + FTS5_BI_RANK, 2, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, + {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, + FTS5_BI_ROWID_LE, 0, 0, -1}, + {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, + FTS5_BI_ROWID_GE, 0, 0, -1}, + }; + + int aColMap[3]; + aColMap[0] = -1; + aColMap[1] = pConfig->nCol; + aColMap[2] = pConfig->nCol+1; + + /* Set idxFlags flags for all WHERE clause terms that will be used. */ + for(i=0; i<pInfo->nConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + int j; + for(j=0; j<ArraySize(aConstraint); j++){ + struct Constraint *pC = &aConstraint[j]; + if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){ + if( p->usable ){ + pC->iConsIndex = i; + idxFlags |= pC->fts5op; + }else if( j==0 ){ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Set a prohibitively high cost. */ + pInfo->estimatedCost = 1e50; + return SQLITE_OK; + } + } + } + } + + /* Set idxFlags flags for the ORDER BY clause */ + if( pInfo->nOrderBy==1 ){ + int iSort = pInfo->aOrderBy[0].iColumn; + if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ + idxFlags |= FTS5_BI_ORDER_RANK; + }else if( iSort==-1 ){ + idxFlags |= FTS5_BI_ORDER_ROWID; + } + if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ + pInfo->orderByConsumed = 1; + if( pInfo->aOrderBy[0].desc ){ + idxFlags |= FTS5_BI_ORDER_DESC; + } + } + } + + /* Calculate the estimated cost based on the flags set in idxFlags. */ + bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); + if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ + pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; + if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); + }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ + pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; + }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ + pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; + }else{ + pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; + } + + /* Assign argvIndex values to each constraint in use. */ + iNext = 1; + for(i=0; i<ArraySize(aConstraint); i++){ + struct Constraint *pC = &aConstraint[i]; + if( pC->iConsIndex>=0 ){ + pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; + pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; + } + } + + pInfo->idxNum = idxFlags; + return SQLITE_OK; +} + +static int fts5NewTransaction(Fts5Table *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; + } + return sqlite3Fts5StorageReset(pTab->pStorage); +} + +/* +** Implementation of xOpen method. +*/ +static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts5Table *pTab = (Fts5Table*)pVTab; + Fts5Config *pConfig = pTab->pConfig; + Fts5Cursor *pCsr = 0; /* New cursor object */ + int nByte; /* Bytes of space to allocate */ + int rc; /* Return code */ + + rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); + pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); + if( pCsr ){ + Fts5Global *pGlobal = pTab->pGlobal; + memset(pCsr, 0, nByte); + pCsr->aColumnSize = (int*)&pCsr[1]; + pCsr->pNext = pGlobal->pCsr; + pGlobal->pCsr = pCsr; + pCsr->iCsrId = ++pGlobal->iNextId; + }else{ + rc = SQLITE_NOMEM; + } + } + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static int fts5StmtType(Fts5Cursor *pCsr){ + if( pCsr->ePlan==FTS5_PLAN_SCAN ){ + return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; + } + return FTS5_STMT_LOOKUP; +} + +/* +** This function is called after the cursor passed as the only argument +** is moved to point at a different row. It clears all cached data +** specific to the previous row stored by the cursor object. +*/ +static void fts5CsrNewrow(Fts5Cursor *pCsr){ + CsrFlagSet(pCsr, + FTS5CSR_REQUIRE_CONTENT + | FTS5CSR_REQUIRE_DOCSIZE + | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST + ); +} + +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5Auxdata *pData; + Fts5Auxdata *pNext; + + sqlite3_free(pCsr->aInstIter); + sqlite3_free(pCsr->aInst); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } + + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); +} + + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ + if( pCursor ){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + Fts5Cursor **pp; + + fts5FreeCursorComponents(pCsr); + /* Remove the cursor from the Fts5Global.pCsr list */ + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); + *pp = pCsr->pNext; + + sqlite3_free(pCsr); + } + return SQLITE_OK; +} + +static int fts5SorterNext(Fts5Cursor *pCsr){ + Fts5Sorter *pSorter = pCsr->pSorter; + int rc; + + rc = sqlite3_step(pSorter->pStmt); + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_EOF); + }else if( rc==SQLITE_ROW ){ + const u8 *a; + const u8 *aBlob; + int nBlob; + int i; + int iOff = 0; + rc = SQLITE_OK; + + pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); + nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); + aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); + + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; + } + + fts5CsrNewrow(pCsr); + } + + return rc; +} + + +/* +** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors +** open on table pTab. +*/ +static void fts5TripCursors(Fts5Table *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && pCsr->base.pVtab==(sqlite3_vtab*)pTab + ){ + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); + } + } +} + +/* +** If the REQUIRE_RESEEK flag is set on the cursor passed as the first +** argument, close and reopen all Fts5IndexIter iterators that the cursor +** is using. Then attempt to move the cursor to a rowid equal to or laster +** (in the cursors sort order - ASC or DESC) than the current rowid. +** +** If the new rowid is not equal to the old, set output parameter *pbSkip +** to 1 before returning. Otherwise, leave it unchanged. +** +** Return SQLITE_OK if successful or if no reseek was required, or an +** error code if an error occurred. +*/ +static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ + int rc = SQLITE_OK; + assert( *pbSkip==0 ); + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + int bDesc = pCsr->bDesc; + i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); + + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc); + if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ + *pbSkip = 1; + } + + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); + fts5CsrNewrow(pCsr); + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + *pbSkip = 1; + } + } + return rc; +} + + +/* +** Advance the cursor to the next row in the table that matches the +** search criteria. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts5EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. +*/ +static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc; + + assert( (pCsr->ePlan<3)== + (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) + ); + assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + + if( pCsr->ePlan<3 ){ + int bSkip = 0; + if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; + rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); + CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr)); + fts5CsrNewrow(pCsr); + }else{ + switch( pCsr->ePlan ){ + case FTS5_PLAN_SPECIAL: { + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = SQLITE_OK; + break; + } + + case FTS5_PLAN_SORTED_MATCH: { + rc = fts5SorterNext(pCsr); + break; + } + + default: + rc = sqlite3_step(pCsr->pStmt); + if( rc!=SQLITE_ROW ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = sqlite3_reset(pCsr->pStmt); + }else{ + rc = SQLITE_OK; + } + break; + } + } + + return rc; +} + + +static int fts5PrepareStatement( + sqlite3_stmt **ppStmt, + Fts5Config *pConfig, + const char *zFmt, + ... +){ + sqlite3_stmt *pRet = 0; + int rc; + char *zSql; + va_list ap; + + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); + if( rc!=SQLITE_OK ){ + *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + } + sqlite3_free(zSql); + } + + va_end(ap); + *ppStmt = pRet; + return rc; +} + +static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ + Fts5Config *pConfig = pTab->pConfig; + Fts5Sorter *pSorter; + int nPhrase; + int nByte; + int rc; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); + if( pSorter==0 ) return SQLITE_NOMEM; + memset(pSorter, 0, nByte); + pSorter->nIdx = nPhrase; + + /* TODO: It would be better to have some system for reusing statement + ** handles here, rather than preparing a new one for each query. But that + ** is not possible as SQLite reference counts the virtual table objects. + ** And since the statement required here reads from this very virtual + ** table, saving it creates a circular reference. + ** + ** If SQLite a built-in statement cache, this wouldn't be a problem. */ + rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", + pConfig->zDb, pConfig->zName, zRank, pConfig->zName, + (zRankArgs ? ", " : ""), + (zRankArgs ? zRankArgs : ""), + bDesc ? "DESC" : "ASC" + ); + + pCsr->pSorter = pSorter; + if( rc==SQLITE_OK ){ + assert( pTab->pSortCsr==0 ); + pTab->pSortCsr = pCsr; + rc = fts5SorterNext(pCsr); + pTab->pSortCsr = 0; + } + + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + pCsr->pSorter = 0; + } + + return rc; +} + +static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ + int rc; + Fts5Expr *pExpr = pCsr->pExpr; + rc = sqlite3Fts5ExprFirst(pExpr, pTab->pIndex, pCsr->iFirstRowid, bDesc); + if( sqlite3Fts5ExprEof(pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + } + fts5CsrNewrow(pCsr); + return rc; +} + +/* +** Process a "special" query. A special query is identified as one with a +** MATCH expression that begins with a '*' character. The remainder of +** the text passed to the MATCH operator are used as the special query +** parameters. +*/ +static int fts5SpecialMatch( + Fts5Table *pTab, + Fts5Cursor *pCsr, + const char *zQuery +){ + int rc = SQLITE_OK; /* Return code */ + const char *z = zQuery; /* Special query text */ + int n; /* Number of bytes in text at z */ + + while( z[0]==' ' ) z++; + for(n=0; z[n] && z[n]!=' '; n++); + + assert( pTab->base.zErrMsg==0 ); + pCsr->ePlan = FTS5_PLAN_SPECIAL; + + if( 0==sqlite3_strnicmp("reads", z, n) ){ + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); + } + else if( 0==sqlite3_strnicmp("id", z, n) ){ + pCsr->iSpecial = pCsr->iCsrId; + } + else{ + /* An unrecognized directive. Return an error message. */ + pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Search for an auxiliary function named zName that can be used with table +** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary +** structure. Otherwise, if no such function exists, return NULL. +*/ +static Fts5Auxiliary *fts5FindAuxiliary(Fts5Table *pTab, const char *zName){ + Fts5Auxiliary *pAux; + + for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){ + if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + + +static int fts5FindRankFunction(Fts5Cursor *pCsr){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->pConfig; + int rc = SQLITE_OK; + Fts5Auxiliary *pAux = 0; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + if( zRankArgs ){ + char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nByte; + pCsr->nRankArg = sqlite3_column_count(pStmt); + nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; + pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte); + if( rc==SQLITE_OK ){ + int i; + for(i=0; i<pCsr->nRankArg; i++){ + pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i); + } + } + pCsr->pRankArgStmt = pStmt; + }else{ + rc = sqlite3_finalize(pStmt); + assert( rc!=SQLITE_OK ); + } + } + } + } + + if( rc==SQLITE_OK ){ + pAux = fts5FindAuxiliary(pTab, zRank); + if( pAux==0 ){ + assert( pTab->base.zErrMsg==0 ); + pTab->base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); + rc = SQLITE_ERROR; + } + } + + pCsr->pRank = pAux; + return rc; +} + + +static int fts5CursorParseRank( + Fts5Config *pConfig, + Fts5Cursor *pCsr, + sqlite3_value *pRank +){ + int rc = SQLITE_OK; + if( pRank ){ + const char *z = (const char*)sqlite3_value_text(pRank); + char *zRank = 0; + char *zRankArgs = 0; + + if( z==0 ){ + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + } + if( rc==SQLITE_OK ){ + pCsr->zRank = zRank; + pCsr->zRankArgs = zRankArgs; + CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK); + }else if( rc==SQLITE_ERROR ){ + pCsr->base.pVtab->zErrMsg = sqlite3_mprintf( + "parse error in rank function: %s", z + ); + } + }else{ + if( pConfig->zRank ){ + pCsr->zRank = (char*)pConfig->zRank; + pCsr->zRankArgs = (char*)pConfig->zRankArgs; + }else{ + pCsr->zRank = (char*)FTS5_DEFAULT_RANK; + pCsr->zRankArgs = 0; + } + } + return rc; +} + +static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** There are three possible query strategies: +** +** 1. Full-text search using a MATCH operator. +** 2. A by-rowid lookup. +** 3. A full-table scan. +*/ +static int fts5FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *zUnused, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; /* Error code */ + int iVal = 0; /* Counter for apVal[] */ + int bDesc; /* True if ORDER BY [rank|rowid] DESC */ + int bOrderByRank; /* True if ORDER BY rank */ + sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ + sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ + sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ + sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ + sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ + char **pzErrmsg = pConfig->pzErrmsg; + + UNUSED_PARAM(zUnused); + UNUSED_PARAM(nVal); + + if( pCsr->ePlan ){ + fts5FreeCursorComponents(pCsr); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); + } + + assert( pCsr->pStmt==0 ); + assert( pCsr->pExpr==0 ); + assert( pCsr->csrflags==0 ); + assert( pCsr->pRank==0 ); + assert( pCsr->zRank==0 ); + assert( pCsr->zRankArgs==0 ); + + assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); + pConfig->pzErrmsg = &pTab->base.zErrMsg; + + /* Decode the arguments passed through to this function. + ** + ** Note: The following set of if(...) statements must be in the same + ** order as the corresponding entries in the struct at the top of + ** fts5BestIndexMethod(). */ + if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; + assert( iVal==nVal ); + bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); + pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); + + /* Set the cursor upper and lower rowid limits. Only some strategies + ** actually use them. This is ok, as the xBestIndex() method leaves the + ** sqlite3_index_constraint.omit flag clear for range constraints + ** on the rowid field. */ + if( pRowidEq ){ + pRowidLe = pRowidGe = pRowidEq; + } + if( bDesc ){ + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + }else{ + pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + } + + if( pTab->pSortCsr ){ + /* If pSortCsr is non-NULL, then this call is being made as part of + ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is + ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will + ** return results to the user for this query. The current cursor + ** (pCursor) is used to execute the query issued by function + ** fts5CursorFirstSorted() above. */ + assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); + assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); + assert( pCsr->iLastRowid==LARGEST_INT64 ); + assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + pCsr->ePlan = FTS5_PLAN_SOURCE; + pCsr->pExpr = pTab->pSortCsr->pExpr; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + sqlite3Fts5ExprClearEof(pCsr->pExpr); + }else if( pMatch ){ + const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); + if( zExpr==0 ) zExpr = ""; + + rc = fts5CursorParseRank(pConfig, pCsr, pRank); + if( rc==SQLITE_OK ){ + if( zExpr[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); + }else{ + char **pzErr = &pTab->base.zErrMsg; + rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr); + if( rc==SQLITE_OK ){ + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); + }else{ + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + } + } + } + } + }else if( pConfig->zContent==0 ){ + *pConfig->pzErrmsg = sqlite3_mprintf( + "%s: table does not support scanning", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup + ** by rowid (ePlan==FTS5_PLAN_ROWID). */ + pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg + ); + if( rc==SQLITE_OK ){ + if( pCsr->ePlan==FTS5_PLAN_ROWID ){ + sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + }else{ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); + sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); + } + rc = fts5NextMethod(pCursor); + } + } + + pConfig->pzErrmsg = pzErrmsg; + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); +} + +/* +** Return the rowid that the cursor currently points to. +*/ +static i64 fts5CursorRowid(Fts5Cursor *pCsr){ + assert( pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + || pCsr->ePlan==FTS5_PLAN_SOURCE + ); + if( pCsr->pSorter ){ + return pCsr->pSorter->iRowid; + }else{ + return sqlite3Fts5ExprRowid(pCsr->pExpr); + } +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts5 +** exposes %_content.rowid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = pCsr->ePlan; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + switch( ePlan ){ + case FTS5_PLAN_SPECIAL: + *pRowid = 0; + break; + + case FTS5_PLAN_SOURCE: + case FTS5_PLAN_MATCH: + case FTS5_PLAN_SORTED_MATCH: + *pRowid = fts5CursorRowid(pCsr); + break; + + default: + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + break; + } + + return SQLITE_OK; +} + +/* +** If the cursor requires seeking (bSeekRequired flag is set), seek it. +** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. +** +** If argument bErrormsg is true and an error occurs, an error message may +** be left in sqlite3_vtab.zErrMsg. +*/ +static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ + int rc = SQLITE_OK; + + /* If the cursor does not yet have a statement handle, obtain one now. */ + if( pCsr->pStmt==0 ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + int eStmt = fts5StmtType(pCsr); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->base.zErrMsg:0) + ); + assert( rc!=SQLITE_OK || pTab->base.zErrMsg==0 ); + assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); + } + + if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ + assert( pCsr->pExpr ); + sqlite3_reset(pCsr->pStmt); + sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); + rc = sqlite3_step(pCsr->pStmt); + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + } + } + } + return rc; +} + +static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + assert( p->base.zErrMsg==0 ); + p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** This function is called to handle an FTS INSERT command. In other words, +** an INSERT statement of the form: +** +** INSERT INTO fts(fts) VALUES($pCmd) +** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) +** +** Argument pVal is the value assigned to column "fts" by the INSERT +** statement. This function returns SQLITE_OK if successful, or an SQLite +** error code if an error occurs. +** +** The commands implemented by this function are documented in the "Special +** INSERT Directives" section of the documentation. It should be updated if +** more commands are added to this function. +*/ +static int fts5SpecialInsert( + Fts5Table *pTab, /* Fts5 table object */ + const char *zCmd, /* Text inserted into table-name column */ + sqlite3_value *pVal /* Value inserted into rank column */ +){ + Fts5Config *pConfig = pTab->pConfig; + int rc = SQLITE_OK; + int bError = 0; + + if( 0==sqlite3_stricmp("delete-all", zCmd) ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5SetVtabError(pTab, + "'delete-all' may only be used with a " + "contentless or external content fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ + if( pConfig->eContent==FTS5_CONTENT_NONE ){ + fts5SetVtabError(pTab, + "'rebuild' may not be used with a contentless fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageRebuild(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ + rc = sqlite3Fts5StorageOptimize(pTab->pStorage); + }else if( 0==sqlite3_stricmp("merge", zCmd) ){ + int nMerge = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); + }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); +#ifdef SQLITE_DEBUG + }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ + pConfig->bPrefixIndex = sqlite3_value_int(pVal); +#endif + }else{ + rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError); + } + if( rc==SQLITE_OK ){ + if( bError ){ + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0); + } + } + } + return rc; +} + +static int fts5SpecialDelete( + Fts5Table *pTab, + sqlite3_value **apVal +){ + int rc = SQLITE_OK; + int eType1 = sqlite3_value_type(apVal[1]); + if( eType1==SQLITE_INTEGER ){ + sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + } + return rc; +} + +static void fts5StorageInsert( + int *pRc, + Fts5Table *pTab, + sqlite3_value **apVal, + i64 *piRowid +){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); + } + *pRc = rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +** +** A delete specifies a single argument - the rowid of the row to remove. +** +** Update and insert operations pass: +** +** 1. The "old" rowid, or NULL. +** 2. The "new" rowid. +** 3. Values for each of the nCol matchable columns. +** 4. Values for the two hidden columns (<tablename> and "rank"). +*/ +static int fts5UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts5Table *pTab = (Fts5Table*)pVtab; + Fts5Config *pConfig = pTab->pConfig; + int eType0; /* value_type() of apVal[0] */ + int rc = SQLITE_OK; /* Return code */ + + /* A transaction must be open when this is called. */ + assert( pTab->ts.eState==1 ); + + assert( pVtab->zErrMsg==0 ); + assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); + assert( nArg==1 + || sqlite3_value_type(apVal[1])==SQLITE_INTEGER + || sqlite3_value_type(apVal[1])==SQLITE_NULL + ); + assert( pTab->pConfig->pzErrmsg==0 ); + pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; + + /* Put any active cursors into REQUIRE_SEEK state. */ + fts5TripCursors(pTab); + + eType0 = sqlite3_value_type(apVal[0]); + if( eType0==SQLITE_NULL + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL + ){ + /* A "special" INSERT op. These are handled separately. */ + const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && 0==sqlite3_stricmp("delete", z) + ){ + rc = fts5SpecialDelete(pTab, apVal); + }else{ + rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); + } + }else{ + /* A regular INSERT, UPDATE or DELETE statement. The trick here is that + ** any conflict on the rowid value must be detected before any + ** modifications are made to the database file. There are 4 cases: + ** + ** 1) DELETE + ** 2) UPDATE (rowid not modified) + ** 3) UPDATE (rowid modified) + ** 4) INSERT + ** + ** Cases 3 and 4 may violate the rowid constraint. + */ + int eConflict = SQLITE_ABORT; + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + eConflict = sqlite3_vtab_on_conflict(pConfig->db); + } + + assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); + assert( nArg!=1 || eType0==SQLITE_INTEGER ); + + /* Filter out attempts to run UPDATE or DELETE on contentless tables. + ** This is not suported. */ + if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "cannot %s contentless fts5 table: %s", + (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName + ); + rc = SQLITE_ERROR; + } + + /* DELETE */ + else if( nArg==1 ){ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + } + + /* INSERT */ + else if( eType0!=SQLITE_INTEGER ){ + /* If this is a REPLACE, first remove the current entry (if any) */ + if( eConflict==SQLITE_REPLACE + && sqlite3_value_type(apVal[1])==SQLITE_INTEGER + ){ + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + + /* UPDATE */ + else{ + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ + if( iOld!=iNew ){ + if( eConflict==SQLITE_REPLACE ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + }else{ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); + } + } + }else{ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + } + } + + pTab->pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xSync() method. +*/ +static int fts5SyncMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5Table *pTab = (Fts5Table*)pVtab; + fts5CheckTransactionState(pTab, FTS5_SYNC, 0); + pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; + fts5TripCursors(pTab); + rc = sqlite3Fts5StorageSync(pTab->pStorage, 1); + pTab->pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts5BeginMethod(sqlite3_vtab *pVtab){ + fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); + fts5NewTransaction((Fts5Table*)pVtab); + return SQLITE_OK; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts5SyncMethod(). +*/ +static int fts5CommitMethod(sqlite3_vtab *pVtab){ + UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts5RollbackMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5Table *pTab = (Fts5Table*)pVtab; + fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + return rc; +} + +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); + +static void *fts5ApiUserData(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return pCsr->pAux->pUserData; +} + +static int fts5ApiColumnCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; +} + +static int fts5ApiColumnTotalSize( + Fts5Context *pCtx, + int iCol, + sqlite3_int64 *pnToken +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); +} + +static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); +} + +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); +} + +static int fts5ApiPhraseCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); +} + +static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); +} + +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + } + } + return rc; +} + +static int fts5CsrPoslist( + Fts5Cursor *pCsr, + int iPhrase, + const u8 **pa, + int *pn +){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + int rc = SQLITE_OK; + int bLive = (pCsr->pSorter==0); + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistPopulator *aPopulator; + int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); + if( aPopulator==0 ) rc = SQLITE_NOMEM; + for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ + int n; const char *z; + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aPopulator, i, z, n + ); + } + } + sqlite3_free(aPopulator); + + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } + + return rc; +} + +/* +** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated +** correctly for the current view. Return SQLITE_OK if successful, or an +** SQLite error code otherwise. +*/ +static int fts5CacheInstArray(Fts5Cursor *pCsr){ + int rc = SQLITE_OK; + Fts5PoslistReader *aIter; /* One iterator for each phrase */ + int nIter; /* Number of iterators/phrases */ + + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + if( pCsr->aInstIter==0 ){ + int nByte = sizeof(Fts5PoslistReader) * nIter; + pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); + } + aIter = pCsr->aInstIter; + + if( aIter ){ + int nInst = 0; /* Number instances seen so far */ + int i; + + /* Initialize all iterators */ + for(i=0; i<nIter && rc==SQLITE_OK; i++){ + const u8 *a; + int n; + rc = fts5CsrPoslist(pCsr, i, &a, &n); + if( rc==SQLITE_OK ){ + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); + } + } + + if( rc==SQLITE_OK ){ + while( 1 ){ + int *aInst; + int iBest = -1; + for(i=0; i<nIter; i++){ + if( (aIter[i].bEof==0) + && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) + ){ + iBest = i; + } + } + if( iBest<0 ) break; + + nInst++; + if( nInst>=pCsr->nInstAlloc ){ + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc( + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + }else{ + rc = SQLITE_NOMEM; + break; + } + } + + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } + } + + pCsr->nInstCount = nInst; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); + } + return rc; +} + +static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ + *pnInst = pCsr->nInstCount; + } + return rc; +} + +static int fts5ApiInst( + Fts5Context *pCtx, + int iIdx, + int *piPhrase, + int *piCol, + int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; +#if 0 + }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 2]; + *piOff = -1; +#endif + }else{ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 1]; + *piOff = pCsr->aInst[iIdx*3 + 2]; + } + } + return rc; +} + +static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ + return fts5CursorRowid((Fts5Cursor*)pCtx); +} + +static int fts5ColumnSizeCb( + void *pContext, /* Pointer to int */ + int tflags, + const char *pUnused, /* Buffer containing token */ + int nUnused, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int *pCnt = (int*)pContext; + UNUSED_PARAM2(pUnused, nUnused); + UNUSED_PARAM2(iUnused1, iUnused2); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } + return SQLITE_OK; +} + +static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->pConfig; + int rc = SQLITE_OK; + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ + if( pConfig->bColumnsize ){ + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); + }else if( pConfig->zContent==0 ){ + int i; + for(i=0; i<pConfig->nCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + pCsr->aColumnSize[i] = -1; + } + } + }else{ + int i; + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + const char *z; int n; + void *p = (void*)(&pCsr->aColumnSize[i]); + pCsr->aColumnSize[i] = 0; + rc = fts5ApiColumnText(pCtx, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize( + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + ); + } + } + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); + } + if( iCol<0 ){ + int i; + *pnToken = 0; + for(i=0; i<pConfig->nCol; i++){ + *pnToken += pCsr->aColumnSize[i]; + } + }else if( iCol<pConfig->nCol ){ + *pnToken = pCsr->aColumnSize[iCol]; + }else{ + *pnToken = 0; + rc = SQLITE_RANGE; + } + return rc; +} + +/* +** Implementation of the xSetAuxdata() method. +*/ +static int fts5ApiSetAuxdata( + Fts5Context *pCtx, /* Fts5 context */ + void *pPtr, /* Pointer to save as auxdata */ + void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + + /* Search through the cursors list of Fts5Auxdata objects for one that + ** corresponds to the currently executing auxiliary function. */ + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + if( pData->xDelete ){ + pData->xDelete(pData->pPtr); + } + }else{ + int rc = SQLITE_OK; + pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata)); + if( pData==0 ){ + if( xDelete ) xDelete(pPtr); + return rc; + } + pData->pAux = pCsr->pAux; + pData->pNext = pCsr->pAuxdata; + pCsr->pAuxdata = pData; + } + + pData->xDelete = xDelete; + pData->pPtr = pPtr; + return SQLITE_OK; +} + +static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + void *pRet = 0; + + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + pRet = pData->pPtr; + if( bClear ){ + pData->pPtr = 0; + pData->xDelete = 0; + } + } + + return pRet; +} + +static void fts5ApiPhraseNext( + Fts5Context *pUnused, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + UNUSED_PARAM(pUnused); + if( pIter->a>=pIter->b ){ + *piCol = -1; + *piOff = -1; + }else{ + int iVal; + pIter->a += fts5GetVarint32(pIter->a, iVal); + if( iVal==1 ){ + pIter->a += fts5GetVarint32(pIter->a, iVal); + *piCol = iVal; + *piOff = 0; + pIter->a += fts5GetVarint32(pIter->a, iVal); + } + *piOff += (iVal-2); + } +} + +static int fts5ApiPhraseFirst( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int n; + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + } + return rc; +} + +static void fts5ApiPhraseNextColumn( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + }else{ + int iIncr; + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); + *piCol += (iIncr-2); + } + }else{ + while( 1 ){ + int dummy; + if( pIter->a>=pIter->b ){ + *piCol = -1; + return; + } + if( pIter->a[0]==0x01 ) break; + pIter->a += fts5GetVarint32(pIter->a, dummy); + } + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + } +} + +static int fts5ApiPhraseFirstColumn( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int n; + if( pSorter ){ + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + n = pSorter->aIdx[iPhrase] - i1; + pIter->a = &pSorter->aPoslist[i1]; + }else{ + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + } + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); + } + }else{ + int n; + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } + } + } + + return rc; +} + + +static int fts5ApiQueryPhrase(Fts5Context*, int, void*, + int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) +); + +static const Fts5ExtensionApi sFts5Api = { + 2, /* iVersion */ + fts5ApiUserData, + fts5ApiColumnCount, + fts5ApiRowCount, + fts5ApiColumnTotalSize, + fts5ApiTokenize, + fts5ApiPhraseCount, + fts5ApiPhraseSize, + fts5ApiInstCount, + fts5ApiInst, + fts5ApiRowid, + fts5ApiColumnText, + fts5ApiColumnSize, + fts5ApiQueryPhrase, + fts5ApiSetAuxdata, + fts5ApiGetAuxdata, + fts5ApiPhraseFirst, + fts5ApiPhraseNext, + fts5ApiPhraseFirstColumn, + fts5ApiPhraseNextColumn, +}; + +/* +** Implementation of API function xQueryPhrase(). +*/ +static int fts5ApiQueryPhrase( + Fts5Context *pCtx, + int iPhrase, + void *pUserData, + int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + int rc; + Fts5Cursor *pNew = 0; + + rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); + if( rc==SQLITE_OK ){ + pNew->ePlan = FTS5_PLAN_MATCH; + pNew->iFirstRowid = SMALLEST_INT64; + pNew->iLastRowid = LARGEST_INT64; + pNew->base.pVtab = (sqlite3_vtab*)pTab; + rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr); + } + + if( rc==SQLITE_OK ){ + for(rc = fts5CursorFirst(pTab, pNew, 0); + rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; + rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) + ){ + rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + break; + } + } + } + + fts5CloseMethod((sqlite3_vtab_cursor*)pNew); + return rc; +} + +static void fts5ApiInvoke( + Fts5Auxiliary *pAux, + Fts5Cursor *pCsr, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( pCsr->pAux==0 ); + pCsr->pAux = pAux; + pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); + pCsr->pAux = 0; +} + +static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ + Fts5Cursor *pCsr; + for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->iCsrId==iCsrId ) break; + } + return pCsr; +} + +static void fts5ApiCallback( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + + Fts5Auxiliary *pAux; + Fts5Cursor *pCsr; + i64 iCsrId; + + assert( argc>=1 ); + pAux = (Fts5Auxiliary*)sqlite3_user_data(context); + iCsrId = sqlite3_value_int64(argv[0]); + + pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); + if( pCsr==0 ){ + char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else{ + fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + } +} + + +/* +** Given cursor id iId, return a pointer to the corresponding Fts5Index +** object. Or NULL If the cursor id does not exist. +** +** If successful, set *ppConfig to point to the associated config object +** before returning. +*/ +static Fts5Index *sqlite3Fts5IndexFromCsrid( + Fts5Global *pGlobal, /* FTS5 global context for db handle */ + i64 iCsrId, /* Id of cursor to find */ + Fts5Config **ppConfig /* OUT: Configuration object */ +){ + Fts5Cursor *pCsr; + Fts5Table *pTab; + + pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); + pTab = (Fts5Table*)pCsr->base.pVtab; + *ppConfig = pTab->pConfig; + + return pTab->pIndex; +} + +/* +** Return a "position-list blob" corresponding to the current position of +** cursor pCsr via sqlite3_result_blob(). A position-list blob contains +** the current position-list for each phrase in the query associated with +** cursor pCsr. +** +** A position-list blob begins with (nPhrase-1) varints, where nPhrase is +** the number of phrases in the query. Following the varints are the +** concatenated position lists for each phrase, in order. +** +** The first varint (if it exists) contains the size of the position list +** for phrase 0. The second (same disclaimer) contains the size of position +** list 1. And so on. There is no size field for the final position list, +** as it can be derived from the total size of the blob. +*/ +static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ + int i; + int rc = SQLITE_OK; + int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + Fts5Buffer val; + + memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; i<nPhrase; i++){ + const u8 *pPoslist; + int nPoslist; + nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ + const u8 *pPoslist; + int nPoslist; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; + } + + sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); + return rc; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +*/ +static int fts5ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + + if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ + if( iCol==pConfig->nCol ){ + sqlite3_result_int64(pCtx, pCsr->iSpecial); + } + }else + + if( iCol==pConfig->nCol ){ + /* User is requesting the value of the special column with the same name + ** as the table. Return the cursor integer id number. This value is only + ** useful in that it may be passed as the first argument to an FTS5 + ** auxiliary function. */ + sqlite3_result_int64(pCtx, pCsr->iCsrId); + }else if( iCol==pConfig->nCol+1 ){ + + /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ + fts5PoslistBlob(pCtx, pCsr); + }else if( + pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + ){ + if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ + fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); + } + } + }else if( !fts5IsContentless(pTab) ){ + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + } + return rc; +} + + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts5FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nUnused, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* OUT: User data for *pxFunc */ +){ + Fts5Table *pTab = (Fts5Table*)pVtab; + Fts5Auxiliary *pAux; + + UNUSED_PARAM(nUnused); + pAux = fts5FindAuxiliary(pTab, zName); + if( pAux ){ + *pxFunc = fts5ApiCallback; + *ppArg = (void*)pAux; + return 1; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + +/* +** Implementation of FTS5 xRename method. Rename an fts5 table. +*/ +static int fts5RenameMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + const char *zName /* New name of table */ +){ + Fts5Table *pTab = (Fts5Table*)pVtab; + return sqlite3Fts5StorageRename(pTab->pStorage, zName); +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5Table *pTab = (Fts5Table*)pVtab; + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); + fts5TripCursors(pTab); + return sqlite3Fts5StorageSync(pTab->pStorage, 0); +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5Table *pTab = (Fts5Table*)pVtab; + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); + fts5TripCursors(pTab); + return sqlite3Fts5StorageSync(pTab->pStorage, 0); +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5Table *pTab = (Fts5Table*)pVtab; + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); + fts5TripCursors(pTab); + return sqlite3Fts5StorageRollback(pTab->pStorage); +} + +/* +** Register a new auxiliary function with global context pGlobal. +*/ +static int fts5CreateAux( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_extension_function xFunc, /* Aux. function implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = sqlite3_overload_function(pGlobal->db, zName, -1); + if( rc==SQLITE_OK ){ + Fts5Auxiliary *pAux; + int nName; /* Size of zName in bytes, including \0 */ + int nByte; /* Bytes of space to allocate */ + + nName = (int)strlen(zName) + 1; + nByte = sizeof(Fts5Auxiliary) + nName; + pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); + if( pAux ){ + memset(pAux, 0, nByte); + pAux->zFunc = (char*)&pAux[1]; + memcpy(pAux->zFunc, zName, nName); + pAux->pGlobal = pGlobal; + pAux->pUserData = pUserData; + pAux->xFunc = xFunc; + pAux->xDestroy = xDestroy; + pAux->pNext = pGlobal->pAux; + pGlobal->pAux = pAux; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + Fts5TokenizerModule *pNew; + int nName; /* Size of zName and its \0 terminator */ + int nByte; /* Bytes of space to allocate */ + int rc = SQLITE_OK; + + nName = (int)strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); + if( pNew ){ + memset(pNew, 0, nByte); + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->x = *pTokenizer; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + }else{ + rc = SQLITE_NOMEM; + } + + return rc; +} + +static Fts5TokenizerModule *fts5LocateTokenizer( + Fts5Global *pGlobal, + const char *zName +){ + Fts5TokenizerModule *pMod = 0; + + if( zName==0 ){ + pMod = pGlobal->pDfltTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; + } + } + + return pMod; +} + +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer() method. +*/ +static int fts5FindTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void **ppUserData, + fts5_tokenizer *pTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + *pTokenizer = pMod->x; + *ppUserData = pMod->pUserData; + }else{ + memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + rc = SQLITE_ERROR; + } + + return rc; +} + +static int sqlite3Fts5GetTokenizer( + Fts5Global *pGlobal, + const char **azArg, + int nArg, + Fts5Tokenizer **ppTok, + fts5_tokenizer **ppTokApi, + char **pzErr +){ + Fts5TokenizerModule *pMod; + int rc = SQLITE_OK; + + pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + if( pMod==0 ){ + assert( nArg>0 ); + rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + }else{ + rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); + *ppTokApi = &pMod->x; + if( rc!=SQLITE_OK && pzErr ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } + } + + if( rc!=SQLITE_OK ){ + *ppTokApi = 0; + *ppTok = 0; + } + + return rc; +} + +static void fts5ModuleDestroy(void *pCtx){ + Fts5TokenizerModule *pTok, *pNextTok; + Fts5Auxiliary *pAux, *pNextAux; + Fts5Global *pGlobal = (Fts5Global*)pCtx; + + for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){ + pNextAux = pAux->pNext; + if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData); + sqlite3_free(pAux); + } + + for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){ + pNextTok = pTok->pNext; + if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData); + sqlite3_free(pTok); + } + + sqlite3_free(pGlobal); +} + +static void fts5Fts5Func( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apUnused /* Function arguments */ +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + char buf[8]; + UNUSED_PARAM2(nArg, apUnused); + assert( nArg==0 ); + assert( sizeof(buf)>=sizeof(pGlobal) ); + memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); + sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); +} + +/* +** Implementation of fts5_source_id() function. +*/ +static void fts5SourceIdFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apUnused /* Function arguments */ +){ + assert( nArg==0 ); + UNUSED_PARAM2(nArg, apUnused); + sqlite3_result_text(pCtx, "fts5: 2016-04-08 15:09:49 fe7d3b75fe1bde41511b323925af8ae1b910bc4d", -1, SQLITE_TRANSIENT); +} + +static int fts5Init(sqlite3 *db){ + static const sqlite3_module fts5Mod = { + /* iVersion */ 2, + /* xCreate */ fts5CreateMethod, + /* xConnect */ fts5ConnectMethod, + /* xBestIndex */ fts5BestIndexMethod, + /* xDisconnect */ fts5DisconnectMethod, + /* xDestroy */ fts5DestroyMethod, + /* xOpen */ fts5OpenMethod, + /* xClose */ fts5CloseMethod, + /* xFilter */ fts5FilterMethod, + /* xNext */ fts5NextMethod, + /* xEof */ fts5EofMethod, + /* xColumn */ fts5ColumnMethod, + /* xRowid */ fts5RowidMethod, + /* xUpdate */ fts5UpdateMethod, + /* xBegin */ fts5BeginMethod, + /* xSync */ fts5SyncMethod, + /* xCommit */ fts5CommitMethod, + /* xRollback */ fts5RollbackMethod, + /* xFindFunction */ fts5FindFunctionMethod, + /* xRename */ fts5RenameMethod, + /* xSavepoint */ fts5SavepointMethod, + /* xRelease */ fts5ReleaseMethod, + /* xRollbackTo */ fts5RollbackToMethod, + }; + + int rc; + Fts5Global *pGlobal = 0; + + pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + if( pGlobal==0 ){ + rc = SQLITE_NOMEM; + }else{ + void *p = (void*)pGlobal; + memset(pGlobal, 0, sizeof(Fts5Global)); + pGlobal->db = db; + pGlobal->api.iVersion = 2; + pGlobal->api.xCreateFunction = fts5CreateAux; + pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; + pGlobal->api.xFindTokenizer = fts5FindTokenizer; + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); + if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 + ); + } + } + + /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file + ** fts5_test_mi.c is compiled and linked into the executable. And call + ** its entry point to enable the matchinfo() demo. */ +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + if( rc==SQLITE_OK ){ + extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); + rc = sqlite3Fts5TestRegisterMatchinfo(db); + } +#endif + + return rc; +} + +/* +** The following functions are used to register the module with SQLite. If +** this module is being built as part of the SQLite core (SQLITE_CORE is +** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. +** +** Or, if this module is being built as a loadable extension, +** sqlite3Fts5Init() is omitted and the two standard entry points +** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. +*/ +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_fts_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_fts5_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} +#else +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3 *db){ + return fts5Init(db); +} +#endif + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +/* #include "fts5Int.h" */ + +struct Fts5Storage { + Fts5Config *pConfig; + Fts5Index *pIndex; + int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ + i64 nTotalRow; /* Total number of rows in FTS table */ + i64 *aTotalSize; /* Total sizes of each column */ + sqlite3_stmt *aStmt[11]; +}; + + +#if FTS5_STMT_SCAN_ASC!=0 +# error "FTS5_STMT_SCAN_ASC mismatch" +#endif +#if FTS5_STMT_SCAN_DESC!=1 +# error "FTS5_STMT_SCAN_DESC mismatch" +#endif +#if FTS5_STMT_LOOKUP!=2 +# error "FTS5_STMT_LOOKUP mismatch" +#endif + +#define FTS5_STMT_INSERT_CONTENT 3 +#define FTS5_STMT_REPLACE_CONTENT 4 +#define FTS5_STMT_DELETE_CONTENT 5 +#define FTS5_STMT_REPLACE_DOCSIZE 6 +#define FTS5_STMT_DELETE_DOCSIZE 7 +#define FTS5_STMT_LOOKUP_DOCSIZE 8 +#define FTS5_STMT_REPLACE_CONFIG 9 +#define FTS5_STMT_SCAN 10 + +/* +** Prepare the two insert statements - Fts5Storage.pInsertContent and +** Fts5Storage.pInsertDocsize - if they have not already been prepared. +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageGetStmt( + Fts5Storage *p, /* Storage handle */ + int eStmt, /* FTS5_STMT_XXX constant */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ + char **pzErrMsg /* OUT: Error message (if any) */ +){ + int rc = SQLITE_OK; + + /* If there is no %_docsize table, there should be no requests for + ** statements to operate on it. */ + assert( p->pConfig->bColumnsize || ( + eStmt!=FTS5_STMT_REPLACE_DOCSIZE + && eStmt!=FTS5_STMT_DELETE_DOCSIZE + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE + )); + + assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) ); + if( p->aStmt[eStmt]==0 ){ + const char *azStmt[] = { + "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", + "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + + "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ + "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ + "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ + + "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + + "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ + "SELECT %s FROM %s AS T", /* SCAN */ + }; + Fts5Config *pC = p->pConfig; + char *zSql = 0; + + switch( eStmt ){ + case FTS5_STMT_SCAN: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent + ); + break; + + case FTS5_STMT_SCAN_ASC: + case FTS5_STMT_SCAN_DESC: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, + pC->zContent, pC->zContentRowid, pC->zContentRowid, + pC->zContentRowid + ); + break; + + case FTS5_STMT_LOOKUP: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent, pC->zContentRowid + ); + break; + + case FTS5_STMT_INSERT_CONTENT: + case FTS5_STMT_REPLACE_CONTENT: { + int nCol = pC->nCol + 1; + char *zBind; + int i; + + zBind = sqlite3_malloc(1 + nCol*2); + if( zBind ){ + for(i=0; i<nCol; i++){ + zBind[i*2] = '?'; + zBind[i*2 + 1] = ','; + } + zBind[i*2-1] = '\0'; + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); + sqlite3_free(zBind); + } + break; + } + + default: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); + break; + } + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK && pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); + } + } + } + + *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); + return rc; +} + + +static int fts5ExecPrintf( + sqlite3 *db, + char **pzErr, + const char *zFormat, + ... +){ + int rc; + va_list ap; /* ... printf arguments */ + char *zSql; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, 0, 0, pzErr); + sqlite3_free(zSql); + } + + va_end(ap); + return rc; +} + +/* +** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error +** code otherwise. +*/ +static int sqlite3Fts5DropAll(Fts5Config *pConfig){ + int rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_data';" + "DROP TABLE IF EXISTS %Q.'%q_idx';" + "DROP TABLE IF EXISTS %Q.'%q_config';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName + ); + } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_content';", + pConfig->zDb, pConfig->zName + ); + } + return rc; +} + +static void fts5StorageRenameOne( + Fts5Config *pConfig, /* Current FTS5 configuration */ + int *pRc, /* IN/OUT: Error code */ + const char *zTail, /* Tail of table name e.g. "data", "config" */ + const char *zName /* New name of FTS5 table */ +){ + if( *pRc==SQLITE_OK ){ + *pRc = fts5ExecPrintf(pConfig->db, 0, + "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';", + pConfig->zDb, pConfig->zName, zTail, zName, zTail + ); + } +} + +static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ + Fts5Config *pConfig = pStorage->pConfig; + int rc = sqlite3Fts5StorageSync(pStorage, 1); + + fts5StorageRenameOne(pConfig, &rc, "data", zName); + fts5StorageRenameOne(pConfig, &rc, "idx", zName); + fts5StorageRenameOne(pConfig, &rc, "config", zName); + if( pConfig->bColumnsize ){ + fts5StorageRenameOne(pConfig, &rc, "docsize", zName); + } + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5StorageRenameOne(pConfig, &rc, "content", zName); + } + return rc; +} + +/* +** Create the shadow table named zPost, with definition zDefn. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int sqlite3Fts5CreateTable( + Fts5Config *pConfig, /* FTS5 configuration */ + const char *zPost, /* Shadow table to create (e.g. "content") */ + const char *zDefn, /* Columns etc. for shadow table */ + int bWithout, /* True for without rowid */ + char **pzErr /* OUT: Error message */ +){ + int rc; + char *zErr = 0; + + rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", + pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" + ); + if( zErr ){ + *pzErr = sqlite3_mprintf( + "fts5: error creating shadow table %q_%s: %s", + pConfig->zName, zPost, zErr + ); + sqlite3_free(zErr); + } + + return rc; +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying tables +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +static int sqlite3Fts5StorageOpen( + Fts5Config *pConfig, + Fts5Index *pIndex, + int bCreate, + Fts5Storage **pp, + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + Fts5Storage *p; /* New object */ + int nByte; /* Bytes of space to allocate */ + + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ + *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + + memset(p, 0, nByte); + p->aTotalSize = (i64*)&p[1]; + p->pConfig = pConfig; + p->pIndex = pIndex; + + if( bCreate ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + int nDefn = 32 + pConfig->nCol*10; + char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); + if( zDefn==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + int iOff; + sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); + iOff = (int)strlen(zDefn); + for(i=0; i<pConfig->nCol; i++){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); + } + sqlite3_free(zDefn); + } + + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = sqlite3Fts5CreateTable( + pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable( + pConfig, "config", "k PRIMARY KEY, v", 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + } + + if( rc ){ + sqlite3Fts5StorageClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen(). +*/ +static int sqlite3Fts5StorageClose(Fts5Storage *p){ + int rc = SQLITE_OK; + if( p ){ + int i; + + /* Finalize all SQL statements */ + for(i=0; i<ArraySize(p->aStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + + sqlite3_free(p); + } + return rc; +} + +typedef struct Fts5InsertCtx Fts5InsertCtx; +struct Fts5InsertCtx { + Fts5Storage *pStorage; + int iCol; + int szCol; /* Size of column value in tokens */ +}; + +/* +** Tokenization callback used when inserting tokens into the FTS index. +*/ +static int fts5StorageInsertCallback( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; + Fts5Index *pIdx = pCtx->pStorage->pIndex; + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); +} + +/* +** If a row with rowid iDel is present in the %_content table, add the +** delete-markers to the FTS index necessary to delete it. Do not actually +** remove the %_content row at this time though. +*/ +static int fts5StorageDeleteFromIndex( + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal +){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ + int rc; /* Return code */ + int rc2; /* sqlite3_reset() return code */ + int iCol; + Fts5InsertCtx ctx; + + if( apVal==0 ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } + } + + ctx.pStorage = p; + ctx.iCol = -1; + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ + if( pConfig->abUnindexed[iCol-1]==0 ){ + const char *zText; + int nText; + if( pSeek ){ + zText = (const char*)sqlite3_column_text(pSeek, iCol); + nText = sqlite3_column_bytes(pSeek, iCol); + }else{ + zText = (const char*)sqlite3_value_text(apVal[iCol-1]); + nText = sqlite3_value_bytes(apVal[iCol-1]); + } + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + zText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + } + } + p->nTotalRow--; + + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + + +/* +** Insert a record into the %_docsize table. Specifically, do: +** +** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); +** +** If there is no %_docsize table (as happens if the columnsize=0 option +** is specified when the FTS5 table is created), this function is a no-op. +*/ +static int fts5StorageInsertDocsize( + Fts5Storage *p, /* Storage module to write to */ + i64 iRowid, /* id value */ + Fts5Buffer *pBuf /* sz value */ +){ + int rc = SQLITE_OK; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pReplace, 1, iRowid); + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + } + return rc; +} + +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. If successful, and if +** argument bCache is true, set the p->bTotalsValid flag to indicate +** that the contents of aTotalSize[] and nTotalRow are valid until +** further notice. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ + int rc = SQLITE_OK; + if( p->bTotalsValid==0 ){ + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); + p->bTotalsValid = bCache; + } + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; i<nCol; i++){ + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + +/* +** Remove a row from the FTS table. +*/ +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ + Fts5Config *pConfig = p->pConfig; + int rc; + sqlite3_stmt *pDel = 0; + + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); + rc = fts5StorageLoadTotals(p, 1); + + /* Delete the index records */ + if( rc==SQLITE_OK ){ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + } + + /* Delete the %_docsize record */ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + /* Delete the %_content record */ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + + return rc; +} + +/* +** Delete all entries in the FTS5 index. +*/ +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ + Fts5Config *pConfig = p->pConfig; + int rc; + + /* Delete the contents of the %_data and %_docsize tables. */ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_idx';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName + ); + } + + /* Reinitialize the %_data table. This call creates the initial structure + ** and averages records. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p->pIndex); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + return rc; +} + +static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ + Fts5Buffer buf = {0,0,0}; + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pScan = 0; + Fts5InsertCtx ctx; + int rc; + + memset(&ctx, 0, sizeof(Fts5InsertCtx)); + ctx.pStorage = p; + rc = sqlite3Fts5StorageDeleteAll(p); + if( rc==SQLITE_OK ){ + rc = fts5StorageLoadTotals(p, 1); + } + + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ + i64 iRowid = sqlite3_column_int64(pScan, 0); + + sqlite3Fts5BufferZero(&buf); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + (const char*)sqlite3_column_text(pScan, ctx.iCol+1), + sqlite3_column_bytes(pScan, ctx.iCol+1), + (void*)&ctx, + fts5StorageInsertCallback + ); + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + } + sqlite3_free(buf.p); + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + return rc; +} + +static int sqlite3Fts5StorageOptimize(Fts5Storage *p){ + return sqlite3Fts5IndexOptimize(p->pIndex); +} + +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ + return sqlite3Fts5IndexMerge(p->pIndex, nMerge); +} + +static int sqlite3Fts5StorageReset(Fts5Storage *p){ + return sqlite3Fts5IndexReset(p->pIndex); +} + +/* +** Allocate a new rowid. This is used for "external content" tables when +** a NULL value is inserted into the rowid column. The new rowid is allocated +** by inserting a dummy row into the %_docsize table. The dummy will be +** overwritten later. +** +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In +** this case the user is required to provide a rowid explicitly. +*/ +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ + int rc = SQLITE_MISMATCH; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pReplace, 1); + sqlite3_bind_null(pReplace, 2); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + if( rc==SQLITE_OK ){ + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); + } + } + return rc; +} + +/* +** Insert a new row into the FTS content table. +*/ +static int sqlite3Fts5StorageContentInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 *piRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; + + /* Insert the new row into the %_content table. */ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + *piRowid = sqlite3_value_int64(apVal[1]); + }else{ + rc = fts5StorageNewRowid(p, piRowid); + } + }else{ + sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ + int i; /* Counter variable */ + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); + for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + rc = sqlite3_bind_value(pInsert, i, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + *piRowid = sqlite3_last_insert_rowid(pConfig->db); + } + + return rc; +} + +/* +** Insert new entries into the FTS index and %_docsize table. +*/ +static int sqlite3Fts5StorageIndexInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 iRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; /* Return code */ + Fts5InsertCtx ctx; /* Tokenization callback context object */ + Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ + + memset(&buf, 0, sizeof(Fts5Buffer)); + ctx.pStorage = p; + rc = fts5StorageLoadTotals(p, 1); + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + } + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), + sqlite3_value_bytes(apVal[ctx.iCol+2]), + (void*)&ctx, + fts5StorageInsertCallback + ); + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + /* Write the %_docsize record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + sqlite3_free(buf.p); + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + + return rc; +} + +static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ + Fts5Config *pConfig = p->pConfig; + char *zSql; + int rc; + + zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", + pConfig->zDb, pConfig->zName, zSuffix + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pCnt = 0; + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pCnt) ){ + *pnRow = sqlite3_column_int64(pCnt, 0); + } + rc = sqlite3_finalize(pCnt); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** Context object used by sqlite3Fts5StorageIntegrity(). +*/ +typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; +struct Fts5IntegrityCtx { + i64 iRowid; + int iCol; + int szCol; + u64 cksum; + Fts5Termset *pTermset; + Fts5Config *pConfig; +}; + + +/* +** Tokenization callback used by integrity check. +*/ +static int fts5StorageIntegrityCallback( + void *pContext, /* Pointer to Fts5IntegrityCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; + Fts5Termset *pTermset = pCtx->pTermset; + int bPresent; + int ii; + int rc = SQLITE_OK; + int iPos; + int iCol; + + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + + switch( pCtx->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + iPos = pCtx->szCol-1; + iCol = pCtx->iCol; + break; + + case FTS5_DETAIL_COLUMNS: + iPos = pCtx->iCol; + iCol = 0; + break; + + default: + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); + iPos = 0; + iCol = 0; + break; + } + + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, 0, pToken, nToken + ); + } + + for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){ + const int nChar = pCtx->pConfig->aPrefix[ii]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); + if( bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte + ); + } + } + } + + return rc; +} + +/* +** Check that the contents of the FTS index match that of the %_content +** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return +** some other SQLite error code if an error occurs while attempting to +** determine this. +*/ +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ + Fts5Config *pConfig = p->pConfig; + int rc; /* Return code */ + int *aColSize; /* Array of size pConfig->nCol */ + i64 *aTotalSize; /* Array of size pConfig->nCol */ + Fts5IntegrityCtx ctx; + sqlite3_stmt *pScan; + + memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); + ctx.pConfig = p->pConfig; + aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64))); + if( !aTotalSize ) return SQLITE_NOMEM; + aColSize = (int*)&aTotalSize[pConfig->nCol]; + memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); + + /* Generate the expected index checksum based on the contents of the + ** %_content table. This block stores the checksum in ctx.cksum. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + if( rc==SQLITE_OK ){ + int rc2; + while( SQLITE_ROW==sqlite3_step(pScan) ){ + int i; + ctx.iRowid = sqlite3_column_int64(pScan, 0); + ctx.szCol = 0; + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ + if( pConfig->abUnindexed[i] ) continue; + ctx.iCol = i; + ctx.szCol = 0; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + (const char*)sqlite3_column_text(pScan, i+1), + sqlite3_column_bytes(pScan, i+1), + (void*)&ctx, + fts5StorageIntegrityCallback + ); + } + if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } + } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + + if( rc!=SQLITE_OK ) break; + } + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* Test that the "totals" (sometimes called "averages") record looks Ok */ + if( rc==SQLITE_OK ){ + int i; + rc = fts5StorageLoadTotals(p, 0); + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ + if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; + } + } + + /* Check that the %_docsize and %_content tables contain the expected + ** number of rows. */ + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "content", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "docsize", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + + /* Pass the expected checksum down to the FTS index module. It will + ** verify, amongst other things, that it matches the checksum generated by + ** inspecting the index itself. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum); + } + + sqlite3_free(aTotalSize); + return rc; +} + +/* +** Obtain an SQLite statement handle that may be used to read data from the +** %_content table. +*/ +static int sqlite3Fts5StorageStmt( + Fts5Storage *p, + int eStmt, + sqlite3_stmt **pp, + char **pzErrMsg +){ + int rc; + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg); + if( rc==SQLITE_OK ){ + assert( p->aStmt[eStmt]==*pp ); + p->aStmt[eStmt] = 0; + } + return rc; +} + +/* +** Release an SQLite statement handle obtained via an earlier call to +** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function +** must match that passed to the sqlite3Fts5StorageStmt() call. +*/ +static void sqlite3Fts5StorageStmtRelease( + Fts5Storage *p, + int eStmt, + sqlite3_stmt *pStmt +){ + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + if( p->aStmt[eStmt]==0 ){ + sqlite3_reset(pStmt); + p->aStmt[eStmt] = pStmt; + }else{ + sqlite3_finalize(pStmt); + } +} + +static int fts5StorageDecodeSizeArray( + int *aCol, int nCol, /* Array to populate */ + const u8 *aBlob, int nBlob /* Record to read varints from */ +){ + int i; + int iOff = 0; + for(i=0; i<nCol; i++){ + if( iOff>=nBlob ) return 1; + iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); + } + return (iOff!=nBlob); +} + +/* +** Argument aCol points to an array of integers containing one entry for +** each table column. This function reads the %_docsize record for the +** specified rowid and populates aCol[] with the results. +** +** An SQLite error code is returned if an error occurs, or SQLITE_OK +** otherwise. +*/ +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + int bCorrupt = 1; + sqlite3_bind_int64(pLookup, 1, iRowid); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + const u8 *aBlob = sqlite3_column_blob(pLookup, 0); + int nBlob = sqlite3_column_bytes(pLookup, 0); + if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){ + bCorrupt = 0; + } + } + rc = sqlite3_reset(pLookup); + if( bCorrupt && rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + } + } + + return rc; +} + +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + *pnToken = 0; + if( iCol<0 ){ + int i; + for(i=0; i<p->pConfig->nCol; i++){ + *pnToken += p->aTotalSize[i]; + } + }else if( iCol<p->pConfig->nCol ){ + *pnToken = p->aTotalSize[iCol]; + }else{ + rc = SQLITE_RANGE; + } + } + return rc; +} + +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + *pnRow = p->nTotalRow; + } + return rc; +} + +/* +** Flush any data currently held in-memory to disk. +*/ +static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ + if( bCommit && p->bTotalsValid ){ + int rc = fts5StorageSaveTotals(p); + p->bTotalsValid = 0; + if( rc!=SQLITE_OK ) return rc; + } + return sqlite3Fts5IndexSync(p->pIndex, bCommit); +} + +static int sqlite3Fts5StorageRollback(Fts5Storage *p){ + p->bTotalsValid = 0; + return sqlite3Fts5IndexRollback(p->pIndex); +} + +static int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, + const char *z, + sqlite3_value *pVal, + int iVal +){ + sqlite3_stmt *pReplace = 0; + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); + if( pVal ){ + sqlite3_bind_value(pReplace, 2, pVal); + }else{ + sqlite3_bind_int(pReplace, 2, iVal); + } + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + if( rc==SQLITE_OK && pVal ){ + int iNew = p->pConfig->iCookie + 1; + rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); + if( rc==SQLITE_OK ){ + p->pConfig->iCookie = iNew; + } + } + return rc; +} + +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + +/* #include "fts5Int.h" */ + +/************************************************************************** +** Start of ascii tokenizer implementation. +*/ + +/* +** For tokenizers with no "unicode" modifier, the set of token characters +** is the same as the set of ASCII range alphanumeric characters. +*/ +static unsigned char aAsciiTokenChar[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */ +}; + +typedef struct AsciiTokenizer AsciiTokenizer; +struct AsciiTokenizer { + unsigned char aTokenChar[128]; +}; + +static void fts5AsciiAddExceptions( + AsciiTokenizer *p, + const char *zArg, + int bTokenChars +){ + int i; + for(i=0; zArg[i]; i++){ + if( (zArg[i] & 0x80)==0 ){ + p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars; + } + } +} + +/* +** Delete a "ascii" tokenizer. +*/ +static void fts5AsciiDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Create an "ascii" tokenizer. +*/ +static int fts5AsciiCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + AsciiTokenizer *p = 0; + UNUSED_PARAM(pUnused); + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = sqlite3_malloc(sizeof(AsciiTokenizer)); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + memset(p, 0, sizeof(AsciiTokenizer)); + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); + for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ + const char *zArg = azArg[i+1]; + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ + fts5AsciiAddExceptions(p, zArg, 1); + }else + if( 0==sqlite3_stricmp(azArg[i], "separators") ){ + fts5AsciiAddExceptions(p, zArg, 0); + }else{ + rc = SQLITE_ERROR; + } + } + if( rc!=SQLITE_OK ){ + fts5AsciiDelete((Fts5Tokenizer*)p); + p = 0; + } + } + } + + *ppOut = (Fts5Tokenizer*)p; + return rc; +} + + +static void asciiFold(char *aOut, const char *aIn, int nByte){ + int i; + for(i=0; i<nByte; i++){ + char c = aIn[i]; + if( c>='A' && c<='Z' ) c += 32; + aOut[i] = c; + } +} + +/* +** Tokenize some text using the ascii tokenizer. +*/ +static int fts5AsciiTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; + int rc = SQLITE_OK; + int ie; + int is = 0; + + char aFold[64]; + int nFold = sizeof(aFold); + char *pFold = aFold; + unsigned char *a = p->aTokenChar; + + UNUSED_PARAM(iUnused); + + while( is<nText && rc==SQLITE_OK ){ + int nByte; + + /* Skip any leading divider characters. */ + while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){ + is++; + } + if( is==nText ) break; + + /* Count the token characters */ + ie = is+1; + while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){ + ie++; + } + + /* Fold to lower case */ + nByte = ie-is; + if( nByte>nFold ){ + if( pFold!=aFold ) sqlite3_free(pFold); + pFold = sqlite3_malloc(nByte*2); + if( pFold==0 ){ + rc = SQLITE_NOMEM; + break; + } + nFold = nByte*2; + } + asciiFold(pFold, &pText[is], nByte); + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, pFold, nByte, is, ie); + is = ie+1; + } + + if( pFold!=aFold ) sqlite3_free(pFold); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of unicode61 tokenizer implementation. +*/ + + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (unsigned char)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct Unicode61Tokenizer Unicode61Tokenizer; +struct Unicode61Tokenizer { + unsigned char aTokenChar[128]; /* ASCII range token characters */ + char *aFold; /* Buffer to fold text into */ + int nFold; /* Size of aFold[] in bytes */ + int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int nException; + int *aiException; +}; + +static int fts5UnicodeAddExceptions( + Unicode61Tokenizer *p, /* Tokenizer object */ + const char *z, /* Characters to treat as exceptions */ + int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ +){ + int rc = SQLITE_OK; + int n = (int)strlen(z); + int *aNew; + + if( n>0 ){ + aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int)); + if( aNew ){ + int nNew = p->nException; + const unsigned char *zCsr = (const unsigned char*)z; + const unsigned char *zTerm = (const unsigned char*)&z[n]; + while( zCsr<zTerm ){ + int iCode; + int bToken; + READ_UTF8(zCsr, zTerm, iCode); + if( iCode<128 ){ + p->aTokenChar[iCode] = (unsigned char)bTokenChars; + }else{ + bToken = sqlite3Fts5UnicodeIsalnum(iCode); + assert( (bToken==0 || bToken==1) ); + assert( (bTokenChars==0 || bTokenChars==1) ); + if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ + int i; + for(i=0; i<nNew; i++){ + if( aNew[i]>iCode ) break; + } + memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int)); + aNew[i] = iCode; + nNew++; + } + } + } + p->aiException = aNew; + p->nException = nNew; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Delete a "unicode61" tokenizer. +*/ +static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok; + sqlite3_free(p->aiException); + sqlite3_free(p->aFold); + sqlite3_free(p); + } + return; +} + +/* +** Create a "unicode61" tokenizer. +*/ +static int fts5UnicodeCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; /* Return code */ + Unicode61Tokenizer *p = 0; /* New tokenizer object */ + + UNUSED_PARAM(pUnused); + + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + if( p ){ + int i; + memset(p, 0, sizeof(Unicode61Tokenizer)); + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); + p->bRemoveDiacritic = 1; + p->nFold = 64; + p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); + if( p->aFold==0 ){ + rc = SQLITE_NOMEM; + } + for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ + const char *zArg = azArg[i+1]; + if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ + rc = SQLITE_ERROR; + } + p->bRemoveDiacritic = (zArg[0]=='1'); + }else + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 1); + }else + if( 0==sqlite3_stricmp(azArg[i], "separators") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 0); + }else{ + rc = SQLITE_ERROR; + } + } + }else{ + rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + fts5UnicodeDelete((Fts5Tokenizer*)p); + p = 0; + } + *ppOut = (Fts5Tokenizer*)p; + } + return rc; +} + +/* +** Return true if, for the purposes of tokenizing with the tokenizer +** passed as the first argument, codepoint iCode is considered a token +** character (not a separator). +*/ +static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ + assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode); +} + +static int fts5UnicodeTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; + int rc = SQLITE_OK; + unsigned char *a = p->aTokenChar; + + unsigned char *zTerm = (unsigned char*)&pText[nText]; + unsigned char *zCsr = (unsigned char *)pText; + + /* Output buffer */ + char *aFold = p->aFold; + int nFold = p->nFold; + const char *pEnd = &aFold[nFold-6]; + + UNUSED_PARAM(iUnused); + + /* Each iteration of this loop gobbles up a contiguous run of separators, + ** then the next token. */ + while( rc==SQLITE_OK ){ + int iCode; /* non-ASCII codepoint read from input */ + char *zOut = aFold; + int is; + int ie; + + /* Skip any separator characters. */ + while( 1 ){ + if( zCsr>=zTerm ) goto tokenize_done; + if( *zCsr & 0x80 ) { + /* A character outside of the ascii range. Skip past it if it is + ** a separator character. Or break out of the loop if it is not. */ + is = zCsr - (unsigned char*)pText; + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p, iCode) ){ + goto non_ascii_tokenchar; + } + }else{ + if( a[*zCsr] ){ + is = zCsr - (unsigned char*)pText; + goto ascii_tokenchar; + } + zCsr++; + } + } + + /* Run through the tokenchars. Fold them into the output buffer along + ** the way. */ + while( zCsr<zTerm ){ + + /* Grow the output buffer so that there is sufficient space to fit the + ** largest possible utf-8 character. */ + if( zOut>pEnd ){ + aFold = sqlite3_malloc(nFold*2); + if( aFold==0 ){ + rc = SQLITE_NOMEM; + goto tokenize_done; + } + zOut = &aFold[zOut - p->aFold]; + memcpy(aFold, p->aFold, nFold); + sqlite3_free(p->aFold); + p->aFold = aFold; + p->nFold = nFold = nFold*2; + pEnd = &aFold[nFold-6]; + } + + if( *zCsr & 0x80 ){ + /* An non-ascii-range character. Fold it into the output buffer if + ** it is a token character, or break out of the loop if it is not. */ + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ + non_ascii_tokenchar: + iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); + if( iCode ) WRITE_UTF8(zOut, iCode); + }else{ + break; + } + }else if( a[*zCsr]==0 ){ + /* An ascii-range separator character. End of token. */ + break; + }else{ + ascii_tokenchar: + if( *zCsr>='A' && *zCsr<='Z' ){ + *zOut++ = *zCsr + 32; + }else{ + *zOut++ = *zCsr; + } + zCsr++; + } + ie = zCsr - (unsigned char*)pText; + } + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); + } + + tokenize_done: + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of porter stemmer implementation. +*/ + +/* Any tokens larger than this (in bytes) are passed through without +** stemming. */ +#define FTS5_PORTER_MAX_TOKEN 64 + +typedef struct PorterTokenizer PorterTokenizer; +struct PorterTokenizer { + fts5_tokenizer tokenizer; /* Parent tokenizer module */ + Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ + char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; +}; + +/* +** Delete a "porter" tokenizer. +*/ +static void fts5PorterDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + PorterTokenizer *p = (PorterTokenizer*)pTok; + if( p->pTokenizer ){ + p->tokenizer.xDelete(p->pTokenizer); + } + sqlite3_free(p); + } +} + +/* +** Create a "porter" tokenizer. +*/ +static int fts5PorterCreate( + void *pCtx, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + fts5_api *pApi = (fts5_api*)pCtx; + int rc = SQLITE_OK; + PorterTokenizer *pRet; + void *pUserdata = 0; + const char *zBase = "unicode61"; + + if( nArg>0 ){ + zBase = azArg[0]; + } + + pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + if( pRet ){ + memset(pRet, 0, sizeof(PorterTokenizer)); + rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + }else{ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + int nArg2 = (nArg>0 ? nArg-1 : 0); + const char **azArg2 = (nArg2 ? &azArg[1] : 0); + rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + } + + if( rc!=SQLITE_OK ){ + fts5PorterDelete((Fts5Tokenizer*)pRet); + pRet = 0; + } + *ppOut = (Fts5Tokenizer*)pRet; + return rc; +} + +typedef struct PorterContext PorterContext; +struct PorterContext { + void *pCtx; + int (*xToken)(void*, int, const char*, int, int, int); + char *aBuf; +}; + +typedef struct PorterRule PorterRule; +struct PorterRule { + const char *zSuffix; + int nSuffix; + int (*xCond)(char *zStem, int nStem); + const char *zOutput; + int nOutput; +}; + +#if 0 +static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){ + int ret = -1; + int nBuf = *pnBuf; + PorterRule *p; + + for(p=aRule; p->zSuffix; p++){ + assert( strlen(p->zSuffix)==p->nSuffix ); + assert( strlen(p->zOutput)==p->nOutput ); + if( nBuf<p->nSuffix ) continue; + if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break; + } + + if( p->zSuffix ){ + int nStem = nBuf - p->nSuffix; + if( p->xCond==0 || p->xCond(aBuf, nStem) ){ + memcpy(&aBuf[nStem], p->zOutput, p->nOutput); + *pnBuf = nStem + p->nOutput; + ret = p - aRule; + } + } + + return ret; +} +#endif + +static int fts5PorterIsVowel(char c, int bYIsVowel){ + return ( + c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y') + ); +} + +static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){ + int i; + int bCons = bPrevCons; + + /* Scan for a vowel */ + for(i=0; i<nStem; i++){ + if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break; + } + + /* Scan for a consonent */ + for(i++; i<nStem; i++){ + if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1; + } + return 0; +} + +/* porter rule condition: (m > 0) */ +static int fts5Porter_MGt0(char *zStem, int nStem){ + return !!fts5PorterGobbleVC(zStem, nStem, 0); +} + +/* porter rule condition: (m > 1) */ +static int fts5Porter_MGt1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (m = 1) */ +static int fts5Porter_MEq1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (*o) */ +static int fts5Porter_Ostar(char *zStem, int nStem){ + if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){ + return 0; + }else{ + int i; + int mask = 0; + int bCons = 0; + for(i=0; i<nStem; i++){ + bCons = !fts5PorterIsVowel(zStem[i], bCons); + assert( bCons==0 || bCons==1 ); + mask = (mask << 1) + bCons; + } + return ((mask & 0x0007)==0x0005); + } +} + +/* porter rule condition: (m > 1 and (*S or *T)) */ +static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){ + assert( nStem>0 ); + return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') + && fts5Porter_MGt1(zStem, nStem); +} + +/* porter rule condition: (*v*) */ +static int fts5Porter_Vowel(char *zStem, int nStem){ + int i; + for(i=0; i<nStem; i++){ + if( fts5PorterIsVowel(zStem[i], i>0) ){ + return 1; + } + } + return 0; +} + + +/************************************************************************** +*************************************************************************** +** GENERATED CODE STARTS HERE (mkportersteps.tcl) +*/ + +static int fts5PorterStep4(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'e': + if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'l': + if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt1(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'o': + if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 's': + if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 't': + if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'z': + if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ate", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'b': + if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ble", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ize", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + } + return ret; +} + + +static int fts5PorterStep2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ate", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "tion", 4); + *pnBuf = nBuf - 6 + 4; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ence", 4); + *pnBuf = nBuf - 4 + 4; + } + }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ance", 4); + *pnBuf = nBuf - 4 + 4; + } + } + break; + + case 'e': + if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ize", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'g': + if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "log", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'l': + if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ble", 3); + *pnBuf = nBuf - 3 + 3; + } + }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "al", 2); + *pnBuf = nBuf - 4 + 2; + } + }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ent", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "e", 1); + *pnBuf = nBuf - 3 + 1; + } + }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ous", 3); + *pnBuf = nBuf - 5 + 3; + } + } + break; + + case 'o': + if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ize", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ate", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ate", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 's': + if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ive", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ful", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ous", 3); + *pnBuf = nBuf - 7 + 3; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ive", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "ble", 3); + *pnBuf = nBuf - 6 + 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep3(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ic", 2); + *pnBuf = nBuf - 4 + 2; + } + } + break; + + case 's': + if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + } + break; + + case 'z': + if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'e': + if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ee", 2); + *pnBuf = nBuf - 3 + 2; + } + }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_Vowel(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + ret = 1; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_Vowel(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + ret = 1; + } + } + break; + + } + return ret; +} + +/* +** GENERATED CODE ENDS HERE (mkportersteps.tcl) +*************************************************************************** +**************************************************************************/ + +static void fts5PorterStep1A(char *aBuf, int *pnBuf){ + int nBuf = *pnBuf; + if( aBuf[nBuf-1]=='s' ){ + if( aBuf[nBuf-2]=='e' ){ + if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') + || (nBuf>3 && aBuf[nBuf-3]=='i' ) + ){ + *pnBuf = nBuf-2; + }else{ + *pnBuf = nBuf-1; + } + } + else if( aBuf[nBuf-2]!='s' ){ + *pnBuf = nBuf-1; + } + } +} + +static int fts5PorterCb( + void *pCtx, + int tflags, + const char *pToken, + int nToken, + int iStart, + int iEnd +){ + PorterContext *p = (PorterContext*)pCtx; + + char *aBuf; + int nBuf; + + if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through; + aBuf = p->aBuf; + nBuf = nToken; + memcpy(aBuf, pToken, nBuf); + + /* Step 1. */ + fts5PorterStep1A(aBuf, &nBuf); + if( fts5PorterStep1B(aBuf, &nBuf) ){ + if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){ + char c = aBuf[nBuf-1]; + if( fts5PorterIsVowel(c, 0)==0 + && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] + ){ + nBuf--; + }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){ + aBuf[nBuf++] = 'e'; + } + } + } + + /* Step 1C. */ + if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){ + aBuf[nBuf-1] = 'i'; + } + + /* Steps 2 through 4. */ + fts5PorterStep2(aBuf, &nBuf); + fts5PorterStep3(aBuf, &nBuf); + fts5PorterStep4(aBuf, &nBuf); + + /* Step 5a. */ + assert( nBuf>0 ); + if( aBuf[nBuf-1]=='e' ){ + if( fts5Porter_MGt1(aBuf, nBuf-1) + || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1)) + ){ + nBuf--; + } + } + + /* Step 5b. */ + if( nBuf>1 && aBuf[nBuf-1]=='l' + && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) + ){ + nBuf--; + } + + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); + + pass_through: + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); +} + +/* +** Tokenize using the porter tokenizer. +*/ +static int fts5PorterTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + PorterTokenizer *p = (PorterTokenizer*)pTokenizer; + PorterContext sCtx; + sCtx.xToken = xToken; + sCtx.pCtx = pCtx; + sCtx.aBuf = p->aBuf; + return p->tokenizer.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + ); +} + +/* +** Register all built-in tokenizers with FTS5. +*/ +static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ + struct BuiltinTokenizer { + const char *zName; + fts5_tokenizer x; + } aBuiltin[] = { + { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, + { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, + { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, + }; + + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ + rc = pApi->xCreateTokenizer(pApi, + aBuiltin[i].zName, + (void*)pApi, + &aBuiltin[i].x, + 0 + ); + } + + return rc; +} + + + +/* +** 2012 May 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + + +/* #include <assert.h> */ + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +static int sqlite3Fts5UnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]<key ); + assert( key>=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int fts5_remove_diacritic(int c){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, + 62924, 63050, 63082, 63274, 63390, + }; + char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', + 'e', 'i', 'o', 'u', 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +static int sqlite3Fts5UnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & (1 << (c-768))) : + (mask1 & (1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} + +/* +** 2015 May 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Routines for varint serialization and deserialization. +*/ + + +/* #include "fts5Int.h" */ + +/* +** This is a copy of the sqlite3GetVarint32() routine from the SQLite core. +** Except, this version does handle the single byte case that the core +** version depends on being handled before its function is called. +*/ +static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){ + u32 a,b; + + /* The 1-byte case. Overwhelmingly the most common. */ + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 0 and 127 */ + *v = a; + return 1; + } + + /* The 2-byte case */ + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 128 and 16383 */ + a &= 0x7f; + a = a<<7; + *v = a | b; + return 2; + } + + /* The 3-byte case */ + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 16384 and 2097151 */ + a &= (0x7f<<14)|(0x7f); + b &= 0x7f; + b = b<<7; + *v = a | b; + return 3; + } + + /* A 32-bit varint is used to store size information in btrees. + ** Objects are rarely larger than 2MiB limit of a 3-byte varint. + ** A 3-byte varint is sufficient, for example, to record the size + ** of a 1048569-byte BLOB or string. + ** + ** We only unroll the first 1-, 2-, and 3- byte cases. The very + ** rare larger cases can be handled by the slower 64-bit varint + ** routine. + */ + { + u64 v64; + u8 n; + p -= 2; + n = sqlite3Fts5GetVarint(p, &v64); + *v = (u32)v64; + assert( n>3 && n<=9 ); + return n; + } +} + + +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + *v = a; + return 1; + } + + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + a &= 0x7f; + a = a<<7; + a |= b; + *v = a; + return 2; + } + + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_2_0; + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_2_0; + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= SLOT_2_0; + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= SLOT_2_0; + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_4_2_0; + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= SLOT_2_0; + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +#ifdef SQLITE_NOINLINE +# define FTS5_NOINLINE SQLITE_NOINLINE +#else +# define FTS5_NOINLINE +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; + return 1; + } + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; + return 2; + } + return fts5PutVarint64(p,v); +} + + +static int sqlite3Fts5GetVarintLen(u32 iVal){ +#if 0 + if( iVal<(1 << 7 ) ) return 1; +#endif + assert( iVal>=(1 << 7) ); + if( iVal<(1 << 14) ) return 2; + if( iVal<(1 << 21) ) return 3; + if( iVal<(1 << 28) ) return 4; + return 5; +} + + +/* +** 2015 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite virtual table module implementing direct access to an +** existing FTS5 index. The module may create several different types of +** tables: +** +** col: +** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col)); +** +** One row for each term/column combination. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term within column $col. Field $cnt is set to the total number of +** instances of term $term in column $col (in any row of the fts5 table). +** +** row: +** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term)); +** +** One row for each term in the database. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term. Field $cnt is set to the total number of instances of term +** $term in the database. +*/ + + +/* #include "fts5Int.h" */ + + +typedef struct Fts5VocabTable Fts5VocabTable; +typedef struct Fts5VocabCursor Fts5VocabCursor; + +struct Fts5VocabTable { + sqlite3_vtab base; + char *zFts5Tbl; /* Name of fts5 table */ + char *zFts5Db; /* Db containing fts5 table */ + sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* FTS5 global object for this database */ + int eType; /* FTS5_VOCAB_COL or ROW */ +}; + +struct Fts5VocabCursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ + Fts5Index *pIndex; /* Associated FTS5 index */ + + int bEof; /* True if this cursor is at EOF */ + Fts5IndexIter *pIter; /* Term/rowid iterator object */ + + int nLeTerm; /* Size of zLeTerm in bytes */ + char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + + /* These are used by 'col' tables only */ + Fts5Config *pConfig; /* Fts5 table configuration */ + int iCol; + i64 *aCnt; + i64 *aDoc; + + /* Output values used by 'row' and 'col' tables */ + i64 rowid; /* This table's current rowid value */ + Fts5Buffer term; /* Current value of 'term' column */ +}; + +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 + +#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" +#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" + +/* +** Bits for the mask used as the idxNum value by xBestIndex/xFilter. +*/ +#define FTS5_VOCAB_TERM_EQ 0x01 +#define FTS5_VOCAB_TERM_GE 0x02 +#define FTS5_VOCAB_TERM_LE 0x04 + + +/* +** Translate a string containing an fts5vocab table type to an +** FTS5_VOCAB_XXX constant. If successful, set *peType to the output +** value and return SQLITE_OK. Otherwise, set *pzErr to an error message +** and return SQLITE_ERROR. +*/ +static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){ + int rc = SQLITE_OK; + char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1); + if( rc==SQLITE_OK ){ + sqlite3Fts5Dequote(zCopy); + if( sqlite3_stricmp(zCopy, "col")==0 ){ + *peType = FTS5_VOCAB_COL; + }else + + if( sqlite3_stricmp(zCopy, "row")==0 ){ + *peType = FTS5_VOCAB_ROW; + }else + { + *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); + rc = SQLITE_ERROR; + } + sqlite3_free(zCopy); + } + + return rc; +} + + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5vocab") +** argv[1] -> database name +** argv[2] -> table name +** +** then: +** +** argv[3] -> name of fts5 table +** argv[4] -> type of fts5vocab table +** +** or, for tables in the TEMP schema only. +** +** argv[3] -> name of fts5 tables database +** argv[4] -> name of fts5 table +** argv[5] -> type of fts5vocab table +*/ +static int fts5VocabInitVtab( + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Pointer to Fts5Global object */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + const char *azSchema[] = { + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")" + }; + + Fts5VocabTable *pRet = 0; + int rc = SQLITE_OK; /* Return code */ + int bDb; + + bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0); + + if( argc!=5 && bDb==0 ){ + *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); + rc = SQLITE_ERROR; + }else{ + int nByte; /* Bytes of space to allocate */ + const char *zDb = bDb ? argv[3] : argv[1]; + const char *zTab = bDb ? argv[4] : argv[3]; + const char *zType = bDb ? argv[5] : argv[4]; + int nDb = (int)strlen(zDb)+1; + int nTab = (int)strlen(zTab)+1; + int eType = 0; + + rc = fts5VocabTableType(zType, pzErr, &eType); + if( rc==SQLITE_OK ){ + assert( eType>=0 && eType<ArraySize(azSchema) ); + rc = sqlite3_declare_vtab(db, azSchema[eType]); + } + + nByte = sizeof(Fts5VocabTable) + nDb + nTab; + pRet = sqlite3Fts5MallocZero(&rc, nByte); + if( pRet ){ + pRet->pGlobal = (Fts5Global*)pAux; + pRet->eType = eType; + pRet->db = db; + pRet->zFts5Tbl = (char*)&pRet[1]; + pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; + memcpy(pRet->zFts5Tbl, zTab, nTab); + memcpy(pRet->zFts5Db, zDb, nDb); + sqlite3Fts5Dequote(pRet->zFts5Tbl); + sqlite3Fts5Dequote(pRet->zFts5Db); + } + } + + *ppVTab = (sqlite3_vtab*)pRet; + return rc; +} + + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5VocabInitVtab(). +*/ +static int fts5VocabConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5VocabCreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method. +*/ +static int fts5VocabBestIndexMethod( + sqlite3_vtab *pUnused, + sqlite3_index_info *pInfo +){ + int i; + int iTermEq = -1; + int iTermGe = -1; + int iTermLe = -1; + int idxNum = 0; + int nArg = 0; + + UNUSED_PARAM(pUnused); + + for(i=0; i<pInfo->nConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->iColumn==0 ){ /* term column */ + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i; + } + } + + if( iTermEq>=0 ){ + idxNum |= FTS5_VOCAB_TERM_EQ; + pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg; + pInfo->estimatedCost = 100; + }else{ + pInfo->estimatedCost = 1000000; + if( iTermGe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_GE; + pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + if( iTermLe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_LE; + pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + } + + pInfo->idxNum = idxNum; + + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts5VocabOpenMethod( + sqlite3_vtab *pVTab, + sqlite3_vtab_cursor **ppCsr +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; + Fts5Index *pIndex = 0; + Fts5Config *pConfig = 0; + Fts5VocabCursor *pCsr = 0; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + char *zSql = 0; + + zSql = sqlite3Fts5Mprintf(&rc, + "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", + pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl + ); + if( zSql ){ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc==SQLITE_ERROR ) rc = SQLITE_OK; + + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + i64 iId = sqlite3_column_int64(pStmt, 0); + pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig); + } + + if( rc==SQLITE_OK && pIndex==0 ){ + rc = sqlite3_finalize(pStmt); + pStmt = 0; + if( rc==SQLITE_OK ){ + pVTab->zErrMsg = sqlite3_mprintf( + "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + rc = SQLITE_ERROR; + } + } + + if( rc==SQLITE_OK ){ + int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor); + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); + } + + if( pCsr ){ + pCsr->pIndex = pIndex; + pCsr->pStmt = pStmt; + pCsr->pConfig = pConfig; + pCsr->aCnt = (i64*)&pCsr[1]; + pCsr->aDoc = &pCsr->aCnt[pConfig->nCol]; + }else{ + sqlite3_finalize(pStmt); + } + + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + pCsr->rowid = 0; + sqlite3Fts5IterClose(pCsr->pIter); + pCsr->pIter = 0; + sqlite3_free(pCsr->zLeTerm); + pCsr->nLeTerm = -1; + pCsr->zLeTerm = 0; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + fts5VocabResetCursor(pCsr); + sqlite3Fts5BufferFree(&pCsr->term); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance the cursor to the next row in the table. +*/ +static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; + int rc = SQLITE_OK; + int nCol = pCsr->pConfig->nCol; + + pCsr->rowid++; + + if( pTab->eType==FTS5_VOCAB_COL ){ + for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){ + if( pCsr->aDoc[pCsr->iCol] ) break; + } + } + + if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){ + pCsr->bEof = 1; + return SQLITE_OK; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + memset(pCsr->aCnt, 0, nCol * sizeof(i64)); + memset(pCsr->aDoc, 0, nCol * sizeof(i64)); + pCsr->iCol = 0; + + assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); + while( rc==SQLITE_OK ){ + const u8 *pPos; int nPos; /* Position list */ + i64 iPos = 0; /* 64-bit position read from poslist */ + int iOff = 0; /* Current offset within position list */ + + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; + switch( pCsr->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + pCsr->aDoc[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + if( ii>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[ii]++; + iCol = ii; + } + } + } + break; + + case FTS5_DETAIL_COLUMNS: + if( pTab->eType==FTS5_VOCAB_ROW ){ + pCsr->aDoc[0]++; + }else{ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPos<nCol ); + if( iPos>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[iPos]++; + } + } + break; + + default: + assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + break; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + } + + if( rc==SQLITE_OK ){ + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ + break; + } + if( sqlite3Fts5IterEof(pCsr->pIter) ) break; + } + } + } + } + + if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ + while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; + assert( pCsr->iCol<pCsr->pConfig->nCol ); + } + return rc; +} + +/* +** This is the xFilter implementation for the virtual table. +*/ +static int fts5VocabFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *zUnused, /* Unused */ + int nUnused, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int rc = SQLITE_OK; + + int iVal = 0; + int f = FTS5INDEX_QUERY_SCAN; + const char *zTerm = 0; + int nTerm = 0; + + sqlite3_value *pEq = 0; + sqlite3_value *pGe = 0; + sqlite3_value *pLe = 0; + + UNUSED_PARAM2(zUnused, nUnused); + + fts5VocabResetCursor(pCsr); + if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + + if( pEq ){ + zTerm = (const char *)sqlite3_value_text(pEq); + nTerm = sqlite3_value_bytes(pEq); + f = 0; + }else{ + if( pGe ){ + zTerm = (const char *)sqlite3_value_text(pGe); + nTerm = sqlite3_value_bytes(pGe); + } + if( pLe ){ + const char *zCopy = (const char *)sqlite3_value_text(pLe); + pCsr->nLeTerm = sqlite3_value_bytes(pLe); + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + if( pCsr->zLeTerm==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); + } + } + } + + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + } + if( rc==SQLITE_OK ){ + rc = fts5VocabNextMethod(pCursor); + } + + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + return pCsr->bEof; +} + +static int fts5VocabColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; + + if( iCol==0 ){ + sqlite3_result_text( + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT + ); + }else if( eType==FTS5_VOCAB_COL ){ + assert( iCol==1 || iCol==2 || iCol==3 ); + if( iCol==1 ){ + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + }else if( iCol==2 ){ + iVal = pCsr->aDoc[pCsr->iCol]; + }else{ + iVal = pCsr->aCnt[pCsr->iCol]; + } + }else{ + assert( iCol==1 || iCol==2 ); + if( iCol==1 ){ + iVal = pCsr->aDoc[0]; + }else{ + iVal = pCsr->aCnt[0]; + } + } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); + return SQLITE_OK; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. The +** rowid should be written to *pRowid. +*/ +static int fts5VocabRowidMethod( + sqlite3_vtab_cursor *pCursor, + sqlite_int64 *pRowid +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + *pRowid = pCsr->rowid; + return SQLITE_OK; +} + +static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ + static const sqlite3_module fts5Vocab = { + /* iVersion */ 2, + /* xCreate */ fts5VocabCreateMethod, + /* xConnect */ fts5VocabConnectMethod, + /* xBestIndex */ fts5VocabBestIndexMethod, + /* xDisconnect */ fts5VocabDisconnectMethod, + /* xDestroy */ fts5VocabDestroyMethod, + /* xOpen */ fts5VocabOpenMethod, + /* xClose */ fts5VocabCloseMethod, + /* xFilter */ fts5VocabFilterMethod, + /* xNext */ fts5VocabNextMethod, + /* xEof */ fts5VocabEofMethod, + /* xColumn */ fts5VocabColumnMethod, + /* xRowid */ fts5VocabRowidMethod, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + }; + void *p = (void*)pGlobal; + + return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); +} + + + + + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ + +/************** End of fts5.c ************************************************/ |