diff options
Diffstat (limited to 'vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c')
| -rw-r--r-- | vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c | 15842 |
1 files changed, 10926 insertions, 4916 deletions
diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c index df04099..a1d6a28 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c @@ -1,7 +1,7 @@ #ifndef USE_LIBSQLITE3 /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.39.2. By combining all the individual C code files into this +** version 3.42.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -124,6 +124,10 @@ #define SQLITE_4_BYTE_ALIGNED_MALLOC #endif /* defined(_MSC_VER) && !defined(_WIN64) */ +#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 +#define HAVE_LOG2 0 +#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ @@ -453,9 +457,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.2" -#define SQLITE_VERSION_NUMBER 3039002 -#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" +#define SQLITE_VERSION "3.42.0" +#define SQLITE_VERSION_NUMBER 3042000 +#define SQLITE_SOURCE_ID "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -870,6 +874,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) @@ -977,13 +982,17 @@ SQLITE_API int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -1061,7 +1070,14 @@ struct sqlite3_file { ** <li> [SQLITE_LOCK_PENDING], or ** <li> [SQLITE_LOCK_EXCLUSIVE]. ** </ul> -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +* If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -1166,9 +1182,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** ** <li>[[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1472,7 +1487,6 @@ struct sqlite3_io_methods { ** in wal mode after the client has finished copying pages from the wal ** file to the database file, but before the *-shm file is updated to ** record the fact that the pages have been checkpointed. -** </ul> ** ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect @@ -1485,10 +1499,16 @@ struct sqlite3_io_methods { ** the database is not a wal-mode db, or if there is no such connection in any ** other process. This opcode cannot be used to detect transactions opened ** by clients within the current process, only within other processes. -** </ul> ** ** <li>[[SQLITE_FCNTL_CKSM_FILE]] -** Used by the cksmvfs VFS module only. +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use interally by the +** [checksum VFS shim] only. +** +** <li>[[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control +** purges the contents of the in-memory page cache. If there is an open +** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1531,6 +1551,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKPT_START 39 #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1561,6 +1582,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; typedef struct sqlite3_api_routines sqlite3_api_routines; /* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +** <ul> +** <li> sqlite3_filename_database() +** <li> sqlite3_filename_journal() +** <li> sqlite3_filename_wal() +** <li> sqlite3_uri_parameter() +** <li> sqlite3_uri_boolean() +** <li> sqlite3_uri_int64() +** <li> sqlite3_uri_key() +** </ul> +*/ +typedef const char *sqlite3_filename; + +/* ** CAPI3REF: OS Interface Object ** ** An instance of the sqlite3_vfs object defines the interface between @@ -1738,7 +1779,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -1925,20 +1966,23 @@ SQLITE_API int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running.</b> ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -2046,6 +2090,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +** <ul> +** <li> SQLITE_CONFIG_LOG +** <li> SQLITE_CONFIG_PCACHE_HDRSZ +** </ul> +** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -2392,28 +2453,28 @@ struct sqlite3_mem_methods { ** compile-time option is not set, then the default maximum is 1073741824. ** </dl> */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ @@ -2454,7 +2515,7 @@ struct sqlite3_mem_methods { ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words ** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^</dd> @@ -2604,8 +2665,12 @@ struct sqlite3_mem_methods { ** <li> sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); ** </ol> ** Because resetting a database is destructive and irreversible, the -** process requires the use of this obscure API and multiple steps to help -** ensure that it does not happen by accident. +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. ** ** [[SQLITE_DBCONFIG_DEFENSIVE]] <dt>SQLITE_DBCONFIG_DEFENSIVE</dt> ** <dd>The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the @@ -2616,6 +2681,7 @@ struct sqlite3_mem_methods { ** <ul> ** <li> The [PRAGMA writable_schema=ON] statement. ** <li> The [PRAGMA journal_mode=OFF] statement. +** <li> The [PRAGMA schema_version=N] statement. ** <li> Writes to the [sqlite_dbpage] virtual table. ** <li> Direct writes to [shadow tables]. ** </ul> @@ -2643,7 +2709,7 @@ struct sqlite3_mem_methods { ** </dd> ** ** [[SQLITE_DBCONFIG_DQS_DML]] -** <dt>SQLITE_DBCONFIG_DQS_DML</td> +** <dt>SQLITE_DBCONFIG_DQS_DML</dt> ** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2652,7 +2718,7 @@ struct sqlite3_mem_methods { ** </dd> ** ** [[SQLITE_DBCONFIG_DQS_DDL]] -** <dt>SQLITE_DBCONFIG_DQS_DDL</td> +** <dt>SQLITE_DBCONFIG_DQS_DDL</dt> ** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2661,7 +2727,7 @@ struct sqlite3_mem_methods { ** </dd> ** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</td> +** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</dt> ** <dd>The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2681,7 +2747,7 @@ struct sqlite3_mem_methods { ** </dd> ** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</td> +** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2690,7 +2756,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2701,6 +2767,38 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. ** </dd> +** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> +** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +** </dd> +** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +** <dt>SQLITE_DBCONFIG_REVERSE_SCANORDER</dt> +** <dd>The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +** </dd> +** ** </dl> */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2721,7 +2819,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2943,8 +3043,12 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); ** ^A call to sqlite3_interrupt(D) that occurs when there are no running ** SQL statements is a no-op and has no effect on SQL statements ** that are started after the sqlite3_interrupt() call returns. +** +** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether +** or not an interrupt is currently in effect for [database connection] D. */ SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API int sqlite3_is_interrupted(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -3562,8 +3666,8 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, ** <dd>^An SQLITE_TRACE_PROFILE callback provides approximately the same ** information as is provided by the [sqlite3_profile()] callback. ** ^The P argument is a pointer to the [prepared statement] and the -** X argument points to a 64-bit integer which is the estimated of -** the number of nanosecond that the prepared statement took to run. +** X argument points to a 64-bit integer which is approximately +** the number of nanoseconds that the prepared statement took to run. ** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. ** ** [[SQLITE_TRACE_ROW]] <dt>SQLITE_TRACE_ROW</dt> @@ -3626,7 +3730,7 @@ SQLITE_API int sqlite3_trace_v2( ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to -** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** [sqlite3_step()] and [sqlite3_prepare()] and similar for ** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** @@ -3651,6 +3755,13 @@ SQLITE_API int sqlite3_trace_v2( ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** +** The progress handler callback would originally only be invoked from the +** bytecode engine. It still might be invoked during [sqlite3_prepare()] +** and similar because those routines might force a reparse of the schema +** which involves running the bytecode engine. However, beginning with +** SQLite version 3.41.0, the progress handler callback might also be +** invoked directly from [sqlite3_prepare()] while analyzing and generating +** code for complex queries. */ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); @@ -3687,13 +3798,18 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** <dl> ** ^(<dt>[SQLITE_OPEN_READONLY]</dt> -** <dd>The database is opened in read-only mode. If the database does not -** already exist, an error is returned.</dd>)^ +** <dd>The database is opened in read-only mode. If the database does +** not already exist, an error is returned.</dd>)^ ** ** ^(<dt>[SQLITE_OPEN_READWRITE]</dt> -** <dd>The database is opened for reading and writing if possible, or reading -** only if the file is write protected by the operating system. In either -** case the database must already exist, otherwise an error is returned.</dd>)^ +** <dd>The database is opened for reading and writing if possible, or +** reading only if the file is write protected by the operating +** system. In either case the database must already exist, otherwise +** an error is returned. For historical reasons, if opening in +** read-write mode fails due to OS-level permissions, an attempt is +** made to open it in read-only mode. [sqlite3_db_readonly()] can be +** used to determine whether the database is actually +** read-write.</dd>)^ ** ** ^(<dt>[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]</dt> ** <dd>The database is opened for reading and writing, and is created if @@ -3731,6 +3847,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** <dd>The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt> ** <dd>The database is opened [shared cache] disabled, overriding @@ -3746,7 +3865,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code.</dd> ** ** [[OPEN_NOFOLLOW]] ^(<dt>[SQLITE_OPEN_NOFOLLOW]</dt> -** <dd>The database filename is not allowed to be a symbolic link</dd> +** <dd>The database filename is not allowed to contain a symbolic link</dd> ** </dl>)^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the @@ -4005,10 +4124,10 @@ SQLITE_API int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -4037,9 +4156,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -SQLITE_API const char *sqlite3_filename_database(const char*); -SQLITE_API const char *sqlite3_filename_journal(const char*); -SQLITE_API const char *sqlite3_filename_wal(const char*); +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -4105,14 +4224,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -SQLITE_API void sqlite3_free_filename(char*); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -5671,10 +5790,21 @@ SQLITE_API int sqlite3_create_window_function( ** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in ** schema structures such as [CHECK constraints], [DEFAULT clauses], ** [expression indexes], [partial indexes], or [generated columns]. -** The SQLITE_DIRECTONLY flags is a security feature which is recommended -** for all [application-defined SQL functions], and especially for functions -** that have side-effects or that could potentially leak sensitive -** information. +** <p> +** The SQLITE_DIRECTONLY flag is recommended for any +** [application-defined SQL function] +** that has side-effects or that could potentially leak sensitive information. +** This will prevent attacks in which an application is tricked +** into using a database file that has had its schema surreptiously +** modified to invoke the application-defined function in ways that are +** harmful. +** <p> +** Some people say it is good practice to set SQLITE_DIRECTONLY on all +** [application-defined SQL functions], regardless of whether or not they +** are security sensitive, as doing so prevents those functions from being used +** inside of the database schema, and thus ensures that the database +** can be inspected and modified using generic tools (such as the [CLI]) +** that do not have access to the application-defined functions. ** </dd> ** ** [[SQLITE_INNOCUOUS]] <dt>SQLITE_INNOCUOUS</dt><dd> @@ -5881,6 +6011,28 @@ SQLITE_API int sqlite3_value_nochange(sqlite3_value*); SQLITE_API int sqlite3_value_frombind(sqlite3_value*); /* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); + +/* ** CAPI3REF: Finding The Subtype Of SQL Values ** METHOD: sqlite3_value ** @@ -5932,7 +6084,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -6137,9 +6289,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -6413,6 +6566,13 @@ SQLITE_API void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ SQLITE_API int sqlite3_sleep(int); @@ -6635,7 +6795,7 @@ SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); ** <li> [sqlite3_filename_wal()] ** </ul> */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -6772,7 +6932,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6893,6 +7053,11 @@ SQLITE_API void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, @@ -6991,7 +7156,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -7253,15 +7418,6 @@ SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); SQLITE_API void sqlite3_reset_auto_extension(void); /* -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - -/* ** Structures used by the virtual table interface */ typedef struct sqlite3_vtab sqlite3_vtab; @@ -7379,10 +7535,10 @@ struct sqlite3_module { ** when the omit flag is true there is no guarantee that the constraint will ** not be checked again using byte code.)^ ** -** ^The idxNum and idxPtr values are recorded and passed into the +** ^The idxNum and idxStr values are recorded and passed into the ** [xFilter] method. -** ^[sqlite3_free()] is used to free idxPtr if and only if -** needToFreeIdxPtr is true. +** ^[sqlite3_free()] is used to free idxStr if and only if +** needToFreeIdxStr is true. ** ** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in ** the correct order to satisfy the ORDER BY clause so that no separate @@ -7502,7 +7658,7 @@ struct sqlite3_index_info { ** the [sqlite3_vtab_collation()] interface. For most real-world virtual ** tables, the collating sequence of constraints does not matter (for example ** because the constraints are numeric) and so the sqlite3_vtab_collation() -** interface is no commonly needed. +** interface is not commonly needed. */ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 @@ -7662,16 +7818,6 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); /* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - -/* ** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} ** @@ -8054,9 +8200,9 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ @@ -9286,7 +9432,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9714,7 +9860,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -9790,18 +9936,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt> ** <dd>Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. ** </dd> +** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]<dt>SQLITE_VTAB_USES_ALL_SCHEMAS</dt> +** <dd>Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +** </dd> ** </dl> */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -9874,7 +10030,7 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); ** <li><p> Otherwise, "BINARY" is returned. ** </ol> */ -SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); /* ** CAPI3REF: Determine if a virtual table query is DISTINCT @@ -10031,21 +10187,20 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** is undefined and probably harmful. ** ** The X parameter in a call to sqlite3_vtab_in_first(X,P) or -** sqlite3_vtab_in_next(X,P) must be one of the parameters to the +** sqlite3_vtab_in_next(X,P) should be one of the parameters to the ** xFilter method which invokes these routines, and specifically ** a parameter that was previously selected for all-at-once IN constraint ** processing use the [sqlite3_vtab_in()] interface in the ** [xBestIndex|xBestIndex method]. ^(If the X parameter is not ** an xFilter argument that was selected for all-at-once IN constraint -** processing, then these routines return [SQLITE_MISUSE])^ or perhaps -** exhibit some other undefined or harmful behavior. +** processing, then these routines return [SQLITE_ERROR].)^ ** ** ^(Use these routines to access all values on the right-hand side ** of the IN constraint using code like the following: ** ** <blockquote><pre> ** for(rc=sqlite3_vtab_in_first(pList, &pVal); -** rc==SQLITE_OK && pVal +** rc==SQLITE_OK && pVal; ** rc=sqlite3_vtab_in_next(pList, &pVal) ** ){ ** // do something with pVal @@ -10143,6 +10298,10 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** ** <dl> ** [[SQLITE_SCANSTAT_NLOOP]] <dt>SQLITE_SCANSTAT_NLOOP</dt> ** <dd>^The [sqlite3_int64] variable pointed to by the V parameter will be @@ -10170,12 +10329,24 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]] <dt>SQLITE_SCANSTAT_SELECT</dt> +** [[SQLITE_SCANSTAT_SELECTID]] <dt>SQLITE_SCANSTAT_SELECTID</dt> ** <dd>^The "int" variable pointed to by the V parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_PARENTID]] <dt>SQLITE_SCANSTAT_PARENTID</dt> +** <dd>The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]] <dt>SQLITE_SCANSTAT_NCYCLE</dt> +** <dd>The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. ** </dl> */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -10184,12 +10355,14 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. @@ -10200,19 +10373,25 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. -** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. +** +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ @@ -10222,6 +10401,19 @@ SQLITE_API int sqlite3_stmt_scanstatus( int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ ); +SQLITE_API int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 /* ** CAPI3REF: Zero Scan-Status Counters @@ -10312,6 +10504,10 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** function is not defined for operations on WITHOUT ROWID tables, or for ** DELETE operations on rowid tables. ** +** ^The sqlite3_preupdate_hook(D,C,P) function returns the P argument from +** the previous call on the same [database connection] D, or NULL for +** the first call on D. +** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces ** provide additional information about a preupdate event. These routines @@ -10717,6 +10913,19 @@ SQLITE_API int sqlite3_deserialize( # undef double #endif +#if defined(__wasi__) +# undef SQLITE_WASI +# define SQLITE_WASI 1 +# undef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ +# ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION +# endif +# ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +# endif +#endif + #if 0 } /* End of the 'extern "C"' block */ #endif @@ -10923,16 +11132,20 @@ SQLITE_API int sqlite3session_create( SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** ** <dt>SQLITE_SESSION_OBJCONFIG_SIZE <dd> @@ -10948,12 +11161,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +** <dt>SQLITE_SESSION_OBJCONFIG_ROWID <dd> +** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -12086,9 +12308,23 @@ SQLITE_API int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +** <dt>SQLITE_CHANGESETAPPLY_IGNORENOOP <dd> +** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +** <ul> +** <li>a delete change if the row being deleted cannot be found, +** <li>an update change if the modified fields are already set to +** their new values in the conflicting row, or +** <li>an insert change if all fields of the conflicting row match +** the row being inserted. +** </ul> */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -13146,11 +13382,16 @@ struct fts5_api { /************** Continuing where we left off in sqliteInt.h ******************/ /* +** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory. +*/ +#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1 + +/* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "config.h" +#include "sqlite_cfg.h" #define SQLITECONFIG_H 1 #endif @@ -13380,8 +13621,8 @@ struct fts5_api { #endif /* -** WAL mode depends on atomic aligned 32-bit loads and stores in a few -** places. The following macros try to make this explicit. +** A few places in the code require atomic load/store of aligned +** integer values. */ #ifndef __has_extension # define __has_extension(x) 0 /* compatibility with non-clang compilers */ @@ -13437,15 +13678,22 @@ struct fts5_api { #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) +# undef SQLITE_INLINE +# define SQLITE_INLINE #endif /* @@ -14263,15 +14511,9 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ /* ** The datatype used to store estimates of the number of rows in a -** table or index. This is an unsigned integer type. For 99.9% of -** the world, a 32-bit integer is sufficient. But a 64-bit integer -** can be used at compile-time if desired. +** table or index. */ -#ifdef SQLITE_64BIT_STATS - typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ -#else - typedef u32 tRowcnt; /* 32-bit is the default */ -#endif +typedef u64 tRowcnt; /* ** Estimated quantities used for query planning are stored as 16-bit @@ -14417,9 +14659,9 @@ typedef INT16_TYPE LogEst; ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) #else -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif /* @@ -14473,15 +14715,38 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace; && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ || defined(SQLITE_ENABLE_TREETRACE)) # define TREETRACE_ENABLED 1 -# define SELECTTRACE(K,P,S,X) \ +# define TREETRACE(K,P,S,X) \ if(sqlite3TreeTrace&(K)) \ sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else -# define SELECTTRACE(K,P,S,X) +# define TREETRACE(K,P,S,X) # define TREETRACE_ENABLED 0 #endif +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +*/ + /* ** Macros for "wheretrace" */ @@ -14494,6 +14759,36 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; # define WHERETRACE(K,X) #endif +/* +** Bits for the sqlite3WhereTrace mask: +** +** (---any--) Top-level block structure +** 0x-------F High-level debug messages +** 0x----FFF- More detail +** 0xFFFF---- Low-level debug messages +** +** 0x00000001 Code generation +** 0x00000002 Solver +** 0x00000004 Solver costs +** 0x00000008 WhereLoop inserts +** +** 0x00000010 Display sqlite3_index_info xBestIndex calls +** 0x00000020 Range an equality scan metrics +** 0x00000040 IN operator decisions +** 0x00000080 WhereLoop cost adjustements +** 0x00000100 +** 0x00000200 Covering index decisions +** 0x00000400 OR optimization +** 0x00000800 Index scanner +** 0x00001000 More details associated with code generation +** 0x00002000 +** 0x00004000 Show all WHERE terms at key points +** 0x00008000 Show the full SELECT statement at key places +** +** 0x00010000 Show more detail when printing WHERE terms +** 0x00020000 Show WHERE terms returned from whereScanNext() +*/ + /* ** An instance of the following structure is used to store the busy-handler @@ -14633,6 +14928,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -14698,6 +14994,7 @@ typedef struct With With; #define MASKBIT32(n) (((unsigned int)1)<<(n)) #define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0) #define ALLBITS ((Bitmask)-1) +#define TOPBIT (((Bitmask)1)<<(BMS-1)) /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer @@ -14712,6 +15009,331 @@ typedef int VList; ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of +** +** SQLITE_OS_KV +** SQLITE_OS_OTHER +** SQLITE_OS_UNIX +** SQLITE_OS_WIN +** +** will defined to either 1 or 0. One of them will be 1. The others will be 0. +** If none of the macros are initially defined, then select either +** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform. +** +** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application +** must provide its own VFS implementation together with sqlite3_os_init() +** and sqlite3_os_end() routines. +*/ +#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ + !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +#endif +#if SQLITE_OS_OTHER+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_KV+1>1 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# define SQLITE_OMIT_LOAD_EXTENSION 1 +# define SQLITE_OMIT_WAL 1 +# define SQLITE_OMIT_DEPRECATED 1 +# undef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */ +# define SQLITE_DQS 0 +# define SQLITE_OMIT_SHARED_CACHE 1 +# define SQLITE_OMIT_AUTOINIT 1 +#endif +#if SQLITE_OS_UNIX+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_WIN+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +#endif + + +#endif /* SQLITE_OS_SETUP_H */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h +*/ +#ifndef SQLITE_MAX_PATHLEN +# define SQLITE_MAX_PATHLEN FILENAME_MAX +#endif + +/* Maximum number of symlinks that will be resolved while trying to +** expand a filename in xFullPathname() in the VFS. +*/ +#ifndef SQLITE_MAX_SYMLINK +# define SQLITE_MAX_SYMLINK 200 +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ + +/************** End of os.h **************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pager.h in the middle of sqliteInt.h *****************/ /************** Begin file pager.h *******************************************/ /* @@ -15147,7 +15769,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); ** reduce network bandwidth. ** ** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use +** standard SQLite. The other hints are provided for extensions that use ** the SQLite parser and code generator but substitute their own storage ** engine. */ @@ -15293,7 +15915,15 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,Pgno*aRoot,int nRoot,int,int*); +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ +); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); @@ -15332,6 +15962,8 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree*); + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the @@ -15448,14 +16080,14 @@ struct VdbeOp { #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS char *zComment; /* Comment to improve readability */ #endif -#ifdef VDBE_PROFILE - u32 cnt; /* Number of times this instruction was executed */ - u64 cycles; /* Total time spent executing this instruction */ -#endif #ifdef SQLITE_VDBE_COVERAGE u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; +#endif }; typedef struct VdbeOp VdbeOp; @@ -15556,48 +16188,48 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Vacuum 5 #define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 8 /* jump */ -#define OP_Gosub 9 /* jump */ -#define OP_InitCoroutine 10 /* jump */ -#define OP_Yield 11 /* jump */ -#define OP_MustBeInt 12 /* jump */ -#define OP_Jump 13 /* jump */ -#define OP_Once 14 /* jump */ -#define OP_If 15 /* jump */ -#define OP_IfNot 16 /* jump */ -#define OP_IsNullOrType 17 /* jump, synopsis: if typeof(r[P1]) IN (P3,5) goto P2 */ -#define OP_IfNullRow 18 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_Init 8 /* jump, synopsis: Start at P2 */ +#define OP_Goto 9 /* jump */ +#define OP_Gosub 10 /* jump */ +#define OP_InitCoroutine 11 /* jump */ +#define OP_Yield 12 /* jump */ +#define OP_MustBeInt 13 /* jump */ +#define OP_Jump 14 /* jump */ +#define OP_Once 15 /* jump */ +#define OP_If 16 /* jump */ +#define OP_IfNot 17 /* jump */ +#define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_SeekLT 20 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 21 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 22 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IfNotOpen 24 /* jump, synopsis: if( !csr[P1] ) goto P2 */ -#define OP_IfNoHope 25 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NoConflict 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NotFound 27 /* jump, synopsis: key=r[P3@P4] */ -#define OP_Found 28 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 29 /* jump, synopsis: intkey=r[P3] */ -#define OP_NotExists 30 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 31 /* jump */ -#define OP_IfSmaller 32 /* jump */ -#define OP_SorterSort 33 /* jump */ -#define OP_Sort 34 /* jump */ -#define OP_Rewind 35 /* jump */ -#define OP_SorterNext 36 /* jump */ -#define OP_Prev 37 /* jump */ -#define OP_Next 38 /* jump */ -#define OP_IdxLE 39 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGE 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 32 /* jump */ +#define OP_IfSmaller 33 /* jump */ +#define OP_SorterSort 34 /* jump */ +#define OP_Sort 35 /* jump */ +#define OP_Rewind 36 /* jump */ +#define OP_SorterNext 37 /* jump */ +#define OP_Prev 38 /* jump */ +#define OP_Next 39 /* jump */ +#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_RowSetRead 45 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 46 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 47 /* jump */ -#define OP_FkIfZero 48 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 49 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 48 /* jump */ +#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -15607,12 +16239,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]<r[P1] */ #define OP_Ge 57 /* jump, same as TK_GE, synopsis: IF r[P3]>=r[P1] */ #define OP_ElseEq 58 /* jump, same as TK_ESCAPE */ -#define OP_IfNotZero 59 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 60 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 61 /* jump */ -#define OP_VNext 62 /* jump */ -#define OP_Filter 63 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_Init 64 /* jump, synopsis: Start at P2 */ +#define OP_IfPos 59 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 62 /* jump */ +#define OP_VNext 63 /* jump */ +#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ #define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ #define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ #define OP_Return 67 @@ -15746,29 +16378,30 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_IN3 0x08 /* in3: P3 is an input */ #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x01, 0x00,\ -/* 8 */ 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01, 0x03,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x09, 0x09, 0x09, 0x09,\ -/* 24 */ 0x01, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x01,\ -/* 32 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x23, 0x0b, 0x01,\ -/* 48 */ 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x01, 0x01,\ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ +/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x49, 0x49, 0x49,\ +/* 24 */ 0x49, 0x01, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,\ +/* 32 */ 0x41, 0x01, 0x01, 0x01, 0x41, 0x01, 0x41, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ +/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ /* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ /* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ -/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x00, 0x00,\ -/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x26, 0x26,\ +/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40, 0x00,\ +/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x00, 0x00, 0x12, 0x00, 0x00, 0x10, 0x00, 0x00,\ -/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ -/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ -/* 136 */ 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10, 0x00,\ +/* 112 */ 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40, 0x00,\ +/* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ +/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50,\ +/* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ /* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ /* 152 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ /* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x50, 0x40,\ /* 176 */ 0x00, 0x10, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00,\ /* 184 */ 0x00, 0x00, 0x00,} @@ -15823,14 +16456,20 @@ SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn(Vdbe*,int,int,int); #endif SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); #ifndef SQLITE_OMIT_EXPLAIN -SQLITE_PRIVATE void sqlite3VdbeExplain(Parse*,u8,const char*,...); +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse*,u8,const char*,...); SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse*); SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse*); # define ExplainQueryPlan(P) sqlite3VdbeExplain P +# ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define ExplainQueryPlan2(V,P) (V = sqlite3VdbeExplain P) +# else +# define ExplainQueryPlan2(V,P) ExplainQueryPlan(P) +# endif # define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) # define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) #else # define ExplainQueryPlan(P) +# define ExplainQueryPlan2(V,P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ @@ -15846,6 +16485,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe*, int); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); @@ -15860,6 +16500,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); @@ -16001,14 +16642,22 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); #else -# define sqlite3VdbeScanStatus(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e,f) +# define sqlite3VdbeScanStatusRange(a,b,c,d) +# define sqlite3VdbeScanStatusCounters(a,b,c,d) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + #endif /* SQLITE_VDBE_H */ /************** End of vdbe.h ************************************************/ @@ -16057,7 +16706,7 @@ struct PgHdr { ** private to pcache.c and should not be accessed by other modules. ** pCache is grouped with the public elements for efficiency. */ - i16 nRef; /* Number of users of this page */ + i64 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ /* NB: pDirtyNext and pDirtyPrev are undefined if the @@ -16138,12 +16787,12 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); /* Return the total number of outstanding page references */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache*); /* Increment the reference count of an existing page */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr*); /* Return the total number of pages stored in the cache */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); @@ -16208,297 +16857,6 @@ SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ -/************** Include os.h in the middle of sqliteInt.h ********************/ -/************** Begin file os.h **********************************************/ -/* -** 2001 September 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file (together with is companion C source-code file -** "os.c") attempt to abstract the underlying operating system so that -** the SQLite library will work on both POSIX and windows systems. -** -** This header file is #include-ed by sqliteInt.h and thus ends up -** being included by every source file. -*/ -#ifndef _SQLITE_OS_H_ -#define _SQLITE_OS_H_ - -/* -** Attempt to automatically detect the operating system and setup the -** necessary pre-processor macros for it. -*/ -/************** Include os_setup.h in the middle of os.h *********************/ -/************** Begin file os_setup.h ****************************************/ -/* -** 2013 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains pre-processor directives related to operating system -** detection and/or setup. -*/ -#ifndef SQLITE_OS_SETUP_H -#define SQLITE_OS_SETUP_H - -/* -** Figure out if we are dealing with Unix, Windows, or some other operating -** system. -** -** After the following block of preprocess macros, all of SQLITE_OS_UNIX, -** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the three will be 1. The other two will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ - defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#endif /* SQLITE_OS_SETUP_H */ - -/************** End of os_setup.h ********************************************/ -/************** Continuing where we left off in os.h *************************/ - -/* If the SET_FULLSYNC macro is not defined above, then make it -** a no-op -*/ -#ifndef SET_FULLSYNC -# define SET_FULLSYNC(x,y) -#endif - -/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h -*/ -#ifndef SQLITE_MAX_PATHLEN -# define SQLITE_MAX_PATHLEN FILENAME_MAX -#endif - -/* Maximum number of symlinks that will be resolved while trying to -** expand a filename in xFullPathname() in the VFS. -*/ -#ifndef SQLITE_MAX_SYMLINK -# define SQLITE_MAX_SYMLINK 200 -#endif - -/* -** The default size of a disk sector -*/ -#ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 4096 -#endif - -/* -** Temporary files are named starting with this prefix followed by 16 random -** alphanumeric characters, and no file extension. They are stored in the -** OS's standard temporary file directory, and are deleted prior to exit. -** If sqlite is being embedded in another program, you may wish to change the -** prefix to reflect your program's name, so that if your program exits -** prematurely, old temporary files can be easily identified. This can be done -** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. -** -** 2006-10-31: The default prefix used to be "sqlite_". But then -** Mcafee started using SQLite in their anti-virus product and it -** started putting files with the "sqlite" name in the c:/temp folder. -** This annoyed many windows users. Those users would then do a -** Google search for "sqlite", find the telephone numbers of the -** developers and call to wake them up at night and complain. -** For this reason, the default name prefix is changed to be "sqlite" -** spelled backwards. So the temp files are still identified, but -** anybody smart enough to figure out the code is also likely smart -** enough to know that calling the developer will not help get rid -** of the file. -*/ -#ifndef SQLITE_TEMP_FILE_PREFIX -# define SQLITE_TEMP_FILE_PREFIX "etilqs_" -#endif - -/* -** The following values may be passed as the second argument to -** sqlite3OsLock(). The various locks exhibit the following semantics: -** -** SHARED: Any number of processes may hold a SHARED lock simultaneously. -** RESERVED: A single process may hold a RESERVED lock on a file at -** any time. Other processes may hold and obtain new SHARED locks. -** PENDING: A single process may hold a PENDING lock on a file at -** any one time. Existing SHARED locks may persist, but no new -** SHARED locks may be obtained by other processes. -** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. -** -** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a -** process that requests an EXCLUSIVE lock may actually obtain a PENDING -** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to -** sqlite3OsLock(). -*/ -#define NO_LOCK 0 -#define SHARED_LOCK 1 -#define RESERVED_LOCK 2 -#define PENDING_LOCK 3 -#define EXCLUSIVE_LOCK 4 - -/* -** File Locking Notes: (Mostly about windows but also some info for Unix) -** -** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because -** those functions are not available. So we use only LockFile() and -** UnlockFile(). -** -** LockFile() prevents not just writing but also reading by other processes. -** A SHARED_LOCK is obtained by locking a single randomly-chosen -** byte out of a specific range of bytes. The lock byte is obtained at -** random so two separate readers can probably access the file at the -** same time, unless they are unlucky and choose the same lock byte. -** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. -** There can only be one writer. A RESERVED_LOCK is obtained by locking -** a single byte of the file that is designated as the reserved lock byte. -** A PENDING_LOCK is obtained by locking a designated byte different from -** the RESERVED_LOCK byte. -** -** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, -** which means we can use reader/writer locks. When reader/writer locks -** are used, the lock is placed on the same range of bytes that is used -** for probabilistic locking in Win95/98/ME. Hence, the locking scheme -** will support two or more Win95 readers or two or more WinNT readers. -** But a single Win95 reader will lock out all WinNT readers and a single -** WinNT reader will lock out all other Win95 readers. -** -** The following #defines specify the range of bytes used for locking. -** SHARED_SIZE is the number of bytes available in the pool from which -** a random byte is selected for a shared lock. The pool of bytes for -** shared locks begins at SHARED_FIRST. -** -** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possibility of having -** clients on win95, winNT, and unix all talking to the same shared file -** and all locking correctly. To do so would require that samba (or whatever -** tool is being used for file sharing) implements locks correctly between -** windows and unix. I'm guessing that isn't likely to happen, but by -** using the same locking range we are at least open to the possibility. -** -** Locking in windows is manditory. For this reason, we cannot store -** actual data in the bytes used for locking. The pager never allocates -** the pages involved in locking therefore. SHARED_SIZE is selected so -** that all locks will fit on a single page even at the minimum page size. -** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE -** is set high so that we don't have to allocate an unused page except -** for very large databases. But one should test the page skipping logic -** by setting PENDING_BYTE low and running the entire regression suite. -** -** Changing the value of PENDING_BYTE results in a subtly incompatible -** file format. Depending on how it is changed, you might not notice -** the incompatibility right away, even running a full regression test. -** The default location of PENDING_BYTE is the first byte past the -** 1GB boundary. -** -*/ -#ifdef SQLITE_OMIT_WSD -# define PENDING_BYTE (0x40000000) -#else -# define PENDING_BYTE sqlite3PendingByte -#endif -#define RESERVED_BYTE (PENDING_BYTE+1) -#define SHARED_FIRST (PENDING_BYTE+2) -#define SHARED_SIZE 510 - -/* -** Wrapper around OS specific sqlite3_os_init() function. -*/ -SQLITE_PRIVATE int sqlite3OsInit(void); - -/* -** Functions for accessing sqlite3_file methods -*/ -SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); -SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); -SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); -SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); -SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); -#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 -SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -#ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); -SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); -SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); -#endif /* SQLITE_OMIT_WAL */ -SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); - - -/* -** Functions for accessing sqlite3_vfs methods -*/ -SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); -SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); -SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); -#ifndef SQLITE_OMIT_LOAD_EXTENSION -SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); -SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); -SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ -SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); -SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); - -/* -** Convenience functions for opening and closing files using -** sqlite3_malloc() to obtain space for the file-handle structure. -*/ -SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); - -#endif /* _SQLITE_OS_H_ */ - -/************** End of os.h **************************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include mutex.h in the middle of sqliteInt.h *****************/ /************** Begin file mutex.h *******************************************/ /* @@ -16744,6 +17102,7 @@ struct Lookaside { #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ + void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */ }; struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ @@ -17008,7 +17367,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -17034,6 +17393,7 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -17088,6 +17448,9 @@ struct sqlite3 { #define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ +#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -17172,8 +17535,14 @@ struct FuncDestructor { ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API ** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API -** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS +** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS -- opposite meanings!!! ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API +** +** Note that even though SQLITE_FUNC_UNSAFE and SQLITE_INNOCUOUS have the +** same bit value, their meanings are inverted. SQLITE_FUNC_UNSAFE is +** used internally and if set means tha the function has side effects. +** SQLITE_INNOCUOUS is used by application code and means "not unsafe". +** See multiple instances of tag-20230109-1. */ #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ #define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ @@ -17290,7 +17659,7 @@ struct FuncDestructor { {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ xPtr, 0, xFunc, 0, 0, 0, #zName, {0} } #define JFUNCTION(zName, nArg, iArg, xFunc) \ - {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|\ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|\ SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define INLINE_FUNC(zName, nArg, iArg, mFlags) \ @@ -17482,6 +17851,7 @@ struct CollSeq { #define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ +#define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -17552,6 +17922,7 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ @@ -17660,7 +18031,7 @@ struct Table { #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) # define ExprIsVtab(X) \ - ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB) + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else # define IsVirtual(X) 0 # define ExprIsVtab(X) 0 @@ -17877,10 +18248,22 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution -** algorithm to employ whenever an attempt is made to insert a non-unique +** and the value of Index.onError indicates which conflict resolution +** algorithm to employ when an attempt is made to insert a non-unique ** element. ** +** The colNotIdxed bitmask is used in combination with SrcItem.colUsed +** for a fast test to see if an index can serve as a covering index. +** colNotIdxed has a 1 bit for every column of the original table that +** is *not* available in the index. Thus the expression +** "colUsed & colNotIdxed" will be non-zero if the index is not a +** covering index. The most significant bit of of colNotIdxed will always +** be true (note-20221022-a). If a column beyond the 63rd column of the +** table is used, the "colUsed & colNotIdxed" test will always be non-zero +** and we have to assume either that the index is not covering, or use +** an alternative (slower) algorithm to determine whether or not +** the index is covering. +** ** While parsing a CREATE TABLE or CREATE INDEX statement in order to ** generate VDBE code (as opposed to parsing one read from an sqlite_schema ** table as part of parsing an existing database schema), transient instances @@ -17916,15 +18299,18 @@ struct Index { unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ + int mxSample; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif - Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ + Bitmask colNotIdxed; /* Unindexed columns in pTab */ }; /* @@ -17999,16 +18385,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -18019,15 +18404,28 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ +#ifdef SQLITE_DEBUG + Select *pSelect; /* SELECT statement that this AggInfo supports */ +#endif }; /* +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. +*/ +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) + +/* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater ** than 32767 we have to make it 32-bit. 16-bit is preferred because @@ -18192,7 +18590,7 @@ struct Expr { #define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_Win 0x008000 /* Contains window functions */ #define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_MemToken 0x020000 /* Need to sqlite3DbFree() Expr.zToken */ + /* 0x020000 // Available for reuse */ #define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ #define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ #define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ @@ -18377,6 +18775,14 @@ struct IdList { ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. ** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +** ** Union member validity: ** ** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc @@ -18416,14 +18822,14 @@ struct SrcItem { Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ } u3; - Bitmask colUsed; /* Bit N (1<<N) set if column N of pTab is used */ + Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY <zIndex>" clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ - CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */ + CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; }; @@ -18437,23 +18843,11 @@ struct OnOrUsing { }; /* -** The following structure describes the FROM clause of a SELECT statement. -** Each table or subquery in the FROM clause is a separate element of -** the SrcList.a[] array. +** This object represents one or more tables that are the source of +** content for an SQL statement. For example, a single SrcList object +** is used to hold the FROM clause of a SELECT statement. SrcList also +** represents the target tables for DELETE, INSERT, and UPDATE statements. ** -** With the addition of multiple database support, the following structure -** can also be used to describe a particular table such as the table that -** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, -** such a table must be a simple name: ID. But in SQLite, the table can -** now be identified by a database name, a dot, then the table name: ID.ID. -** -** The jointype starts out showing the join type between the current table -** and the next table on the list. The parser builds the list this way. -** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each -** jointype expresses the join between the table and the previous table. -** -** In the colUsed field, the high-order bit (bit 63) is set if the table -** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { int nSrc; /* Number of tables or subqueries in the FROM clause */ @@ -18561,7 +18955,7 @@ struct NameContext { #define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ #define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ #define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ -#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ #define NC_UEList 0x000080 /* True if uNC.pEList is used */ #define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ @@ -18690,6 +19084,7 @@ struct Select { #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ +#define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) @@ -18798,7 +19193,7 @@ struct SelectDest { int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ - char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + char *zAffSdst; /* Affinity used for SRT_Set */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; @@ -18857,11 +19252,34 @@ struct TriggerPrg { #else typedef unsigned int yDbMask; # define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) -# define DbMaskZero(M) (M)=0 -# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) -# define DbMaskAllZero(M) (M)==0 -# define DbMaskNonZero(M) (M)!=0 +# define DbMaskZero(M) ((M)=0) +# define DbMaskSet(M,I) ((M)|=(((yDbMask)1)<<(I))) +# define DbMaskAllZero(M) ((M)==0) +# define DbMaskNonZero(M) ((M)!=0) +#endif + +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + u8 aff; /* Affinity of the pExpr expression */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ #endif +}; /* ** An instance of the ParseCleanup object specifies an operation that @@ -18904,11 +19322,14 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 disableVtab; /* Disable all virtual tables for this parse */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ +#endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -18921,6 +19342,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ @@ -18944,6 +19366,9 @@ struct Parse { u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ +#endif u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ @@ -19356,15 +19781,16 @@ struct Walker { struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ int *aiCol; /* array of column indexes */ struct IdxCover *pIdxCover; /* Check for index coverage */ - struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ struct WindowRewrite *pRewrite; /* Window rewrite context */ struct WhereConst *pConst; /* WHERE clause constants */ struct RenameCtx *pRename; /* RENAME COLUMN context */ struct Table *pTab; /* Table of generated column */ + struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ - DbFixer *pFix; + DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; @@ -19634,6 +20060,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -19643,6 +20071,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif SQLITE_PRIVATE int sqlite3IsIdChar(u8); @@ -19670,6 +20100,7 @@ SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3*, void*); SQLITE_PRIVATE int sqlite3MallocSize(const void*); SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); @@ -19690,12 +20121,14 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); */ #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) -# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) +# define sqlite3StackAllocRawNN(D,N) alloca(N) # define sqlite3StackFree(D,P) +# define sqlite3StackFreeNN(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) -# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) +# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P) #endif /* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they @@ -19818,6 +20251,7 @@ SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window*); #endif SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); @@ -19832,6 +20266,10 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +SQLITE_PRIVATE void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); #endif @@ -19875,7 +20313,7 @@ SQLITE_PRIVATE const char *sqlite3ColumnColl(Column*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); SQLITE_PRIVATE void sqlite3GenerateColumnNames(Parse *pParse, Select *pSelect); SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); @@ -19982,7 +20420,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); @@ -20071,7 +20509,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr*,const SrcItem*); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -20194,7 +20632,8 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); -SQLITE_PRIVATE void sqlite3Int64ToText(i64,char*); +SQLITE_PRIVATE i64 sqlite3RealToI64(double); +SQLITE_PRIVATE int sqlite3Int64ToText(i64,char*); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); @@ -20239,11 +20678,13 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table*,int); SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); @@ -20260,6 +20701,9 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int); #ifndef SQLITE_OMIT_DESERIALIZE SQLITE_PRIVATE int sqlite3MemdbInit(void); +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs*); +#else +# define sqlite3IsMemdb(X) 0 #endif SQLITE_PRIVATE const char *sqlite3ErrStr(int); @@ -20310,7 +20754,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[]; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[]; -SQLITE_PRIVATE const char sqlite3StdTypeMap[]; SQLITE_PRIVATE const char *sqlite3StdType[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char *sqlite3aLTb; @@ -20400,7 +20843,7 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); -SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, int); +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); @@ -20514,10 +20957,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) -SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info*); -#endif +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse*); SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); @@ -20754,6 +21194,22 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*); SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); #endif +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void); +#endif + +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +SQLITE_PRIVATE sqlite3_uint64 sqlite3Hwtime(void); +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + #endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ @@ -20795,101 +21251,6 @@ SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 and x86_64 class CPUs. -*/ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - /* - ** asm() is needed for hardware timing support. Without asm(), - ** disable the sqlite3Hwtime() routine. - ** - ** sqlite3Hwtime() is only used for some obscure debugging - ** and analysis configurations, not in any deliverable, so this - ** should not be a great loss. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(SQLITE_HWTIME_H) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ - static sqlite_uint64 g_start; static sqlite_uint64 g_elapsed; #define TIMER_START g_start=sqlite3Hwtime() @@ -20985,7 +21346,7 @@ SQLITE_API extern int sqlite3_open_file_count; ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -/* #include "config.h" */ +/* #include "sqlite_cfg.h" */ #define SQLITECONFIG_H 1 #endif @@ -21150,6 +21511,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT "DISABLE_SKIPAHEAD_DISTINCT", #endif +#ifdef SQLITE_DQS + "DQS=" CTIMEOPT_VAL(SQLITE_DQS), +#endif #ifdef SQLITE_ENABLE_8_3_NAMES "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif @@ -21640,9 +22004,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - "PCACHE_SEPARATE_HEADER", -#endif #ifdef SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif @@ -21844,7 +22205,7 @@ SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 -** SQLite identifier character 0x40 +** SQLite identifier character 0x40 $, _, or non-ascii ** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper @@ -22038,7 +22399,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ 0, /* iPrngSeed */ #ifdef SQLITE_DEBUG - {0,0,0,0,0,0} /* aTune */ + {0,0,0,0,0,0}, /* aTune */ #endif }; @@ -22122,10 +22483,6 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; ** ** sqlite3StdTypeAffinity[] The affinity associated with each entry ** in sqlite3StdType[]. -** -** sqlite3StdTypeMap[] The type value (as returned from -** sqlite3_column_type() or sqlite3_value_type()) -** for each entry in sqlite3StdType[]. */ SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { @@ -22136,14 +22493,6 @@ SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { SQLITE_AFF_REAL, SQLITE_AFF_TEXT }; -SQLITE_PRIVATE const char sqlite3StdTypeMap[] = { - 0, - SQLITE_BLOB, - SQLITE_INTEGER, - SQLITE_INTEGER, - SQLITE_FLOAT, - SQLITE_TEXT -}; SQLITE_PRIVATE const char *sqlite3StdType[] = { "ANY", "BLOB", @@ -22346,7 +22695,6 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ @@ -22562,10 +22910,19 @@ typedef unsigned bft; /* Bit Field Type */ /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. +** +** aAddrRange[]: +** This array is used by ScanStatus elements associated with EQP +** notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is +** an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[] +** values should be summed to calculate the NCYCLE value. Each pair of +** integer addresses is a start and end address (both inclusive) for a range +** instructions. A start value of 0 indicates an empty range. */ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ + int aAddrRange[6]; int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ @@ -22595,7 +22952,7 @@ struct DblquoteStr { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Vdbe **ppVPrev,*pVNext; /* Linked list of VDBEs with the same Vdbe.db */ Parse *pParse; /* Parsing context used to create this Vdbe */ ynVar nVar; /* Number of entries in aVar[] */ int nMem; /* Number of memory locations currently allocated */ @@ -22621,7 +22978,7 @@ struct Vdbe { int nOp; /* Number of instructions in the program */ int nOpAlloc; /* Slots allocated for aOp[] */ Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ + Mem *pResultRow; /* Current output row */ char *zErrMsg; /* Error message written here */ VList *pVList; /* Name of variables */ #ifndef SQLITE_OMIT_TRACE @@ -22658,7 +23015,6 @@ struct Vdbe { SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif @@ -22825,6 +23181,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void*); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe*, VdbeCursor*); SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe*); @@ -23153,6 +23511,8 @@ SQLITE_API int sqlite3_db_status( sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(i=0; i<db->nDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( ALWAYS(pSchema!=0) ){ @@ -23178,6 +23538,7 @@ SQLITE_API int sqlite3_db_status( } } db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); *pHighwater = 0; @@ -23195,9 +23556,12 @@ SQLITE_API int sqlite3_db_status( int nByte = 0; /* Used to accumulate return value */ db->pnBytesFreed = &nByte; - for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pVNext){ sqlite3VdbeDelete(pVdbe); } + db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; *pHighwater = 0; /* IMP: R-64479-57858 */ @@ -23334,6 +23698,7 @@ struct DateTime { char validTZ; /* True (1) if tz is valid */ char tzSet; /* Timezone was set explicitly */ char isError; /* An overflow has occurred */ + char useSubsec; /* Display subsecond precision */ }; @@ -23533,7 +23898,7 @@ static void computeJD(DateTime *p){ p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); if( p->validTZ ){ p->iJD -= p->tz*60000; p->validYMD = 0; @@ -23648,6 +24013,11 @@ static int parseDateOrTime( }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; + }else if( (sqlite3StrICmp(zDate,"subsec")==0 + || sqlite3StrICmp(zDate,"subsecond")==0) + && sqlite3NotPureFunc(context) ){ + p->useSubsec = 1; + return setDateTimeToCurrent(context, p); } return 1; } @@ -24006,7 +24376,7 @@ static int parseModifier( i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ - int iErr; /* Guess is off by this much */ + i64 iErr; /* Guess is off by this much */ computeJD(p); iGuess = iOrigJD = p->iJD; @@ -24042,7 +24412,7 @@ static int parseModifier( */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; @@ -24062,8 +24432,22 @@ static int parseModifier( ** ** Move the date backwards to the beginning of the current day, ** or month or year. + ** + ** subsecond + ** subsec + ** + ** Show subsecond precision in the output of datetime() and + ** unixepoch() and strftime('%s'). */ - if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ){ + if( sqlite3_stricmp(z, "subsec")==0 + || sqlite3_stricmp(z, "subsecond")==0 + ){ + p->useSubsec = 1; + rc = 0; + } + break; + } if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); @@ -24261,7 +24645,11 @@ static void unixepochFunc( DateTime x; if( isDate(context, argc, argv, &x)==0 ){ computeJD(&x); - sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + if( x.useSubsec ){ + sqlite3_result_double(context, (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + } } } @@ -24277,8 +24665,8 @@ static void datetimeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - int Y, s; - char zBuf[24]; + int Y, s, n; + char zBuf[32]; computeYMD_HMS(&x); Y = x.Y; if( Y<0 ) Y = -Y; @@ -24299,15 +24687,28 @@ static void datetimeFunc( zBuf[15] = '0' + (x.m/10)%10; zBuf[16] = '0' + (x.m)%10; zBuf[17] = ':'; - s = (int)x.s; - zBuf[18] = '0' + (s/10)%10; - zBuf[19] = '0' + (s)%10; - zBuf[20] = 0; + if( x.useSubsec ){ + s = (int)1000.0*x.s; + zBuf[18] = '0' + (s/10000)%10; + zBuf[19] = '0' + (s/1000)%10; + zBuf[20] = '.'; + zBuf[21] = '0' + (s/100)%10; + zBuf[22] = '0' + (s/10)%10; + zBuf[23] = '0' + (s)%10; + zBuf[24] = 0; + n = 24; + }else{ + s = (int)x.s; + zBuf[18] = '0' + (s/10)%10; + zBuf[19] = '0' + (s)%10; + zBuf[20] = 0; + n = 20; + } if( x.Y<0 ){ zBuf[0] = '-'; - sqlite3_result_text(context, zBuf, 20, SQLITE_TRANSIENT); + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); }else{ - sqlite3_result_text(context, &zBuf[1], 19, SQLITE_TRANSIENT); + sqlite3_result_text(context, &zBuf[1], n-1, SQLITE_TRANSIENT); } } } @@ -24324,7 +24725,7 @@ static void timeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - int s; + int s, n; char zBuf[16]; computeHMS(&x); zBuf[0] = '0' + (x.h/10)%10; @@ -24333,11 +24734,24 @@ static void timeFunc( zBuf[3] = '0' + (x.m/10)%10; zBuf[4] = '0' + (x.m)%10; zBuf[5] = ':'; - s = (int)x.s; - zBuf[6] = '0' + (s/10)%10; - zBuf[7] = '0' + (s)%10; - zBuf[8] = 0; - sqlite3_result_text(context, zBuf, 8, SQLITE_TRANSIENT); + if( x.useSubsec ){ + s = (int)1000.0*x.s; + zBuf[6] = '0' + (s/10000)%10; + zBuf[7] = '0' + (s/1000)%10; + zBuf[8] = '.'; + zBuf[9] = '0' + (s/100)%10; + zBuf[10] = '0' + (s/10)%10; + zBuf[11] = '0' + (s)%10; + zBuf[12] = 0; + n = 12; + }else{ + s = (int)x.s; + zBuf[6] = '0' + (s/10)%10; + zBuf[7] = '0' + (s)%10; + zBuf[8] = 0; + n = 8; + } + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); } } @@ -24468,8 +24882,13 @@ static void strftimeFunc( break; } case 's': { - i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); - sqlite3_str_appendf(&sRes,"%lld",iS); + if( x.useSubsec ){ + sqlite3_str_appendf(&sRes,"%.3f", + (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3_str_appendf(&sRes,"%lld",iS); + } break; } case 'S': { @@ -24723,9 +25142,11 @@ SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ } SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); + assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE ); return id->pMethods->xLock(id, lockType); } SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED ); return id->pMethods->xUnlock(id, lockType); } SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ @@ -24840,6 +25261,7 @@ SQLITE_PRIVATE int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ + assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) ); rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; @@ -27155,9 +27577,13 @@ static int memsys5Roundup(int n){ if( n<=mem5.szAtom ) return mem5.szAtom; return mem5.szAtom*2; } - if( n>0x40000000 ) return 0; + if( n>0x10000000 ){ + if( n>0x40000000 ) return 0; + if( n>0x20000000 ) return 0x40000000; + return 0x20000000; + } for(iFullSz=mem5.szAtom*8; iFullSz<n; iFullSz *= 4); - if( (iFullSz/2)>=n ) return iFullSz/2; + if( (iFullSz/2)>=(i64)n ) return iFullSz/2; return iFullSz; } @@ -29059,17 +29485,33 @@ static void mallocWithAlarm(int n, void **pp){ } /* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Paranoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + +/* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ void *p; - if( n==0 || n>=0x7fffff00 ){ - /* A memory allocation of a number of bytes which is near the maximum - ** signed integer value might cause an integer overflow inside of the - ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving - ** 255 bytes of overhead. SQLite itself will never use anything near - ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + if( n==0 || n>SQLITE_MAX_ALLOCATION_SIZE ){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); @@ -29105,7 +29547,7 @@ SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, const void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd); } #else #define isLookaside(A,B) 0 @@ -29129,18 +29571,16 @@ static int lookasideMallocSize(sqlite3 *db, const void *p){ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){ assert( p!=0 ); #ifdef SQLITE_DEBUG - if( db==0 || !isLookaside(db,p) ){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - }else{ - assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - } + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else if( !isLookaside(db,p) ){ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } #endif if( db ){ - if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ + if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ assert( sqlite3_mutex_held(db->mutex) ); @@ -29196,14 +29636,11 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( p!=0 ); if( db ){ - if( db->pnBytesFreed ){ - measureAllocationSize(db, p); - return; - } if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif @@ -29214,6 +29651,7 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif @@ -29222,6 +29660,10 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ return; } } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -29229,6 +29671,43 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); } +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); if( p ) sqlite3DbFreeNN(db, p); @@ -29528,9 +30007,14 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ */ SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]<zEnd ); } +#endif while( sqlite3Isspace(zStart[0]) ) zStart++; n = (int)(zEnd - zStart); - while( ALWAYS(n>0) && sqlite3Isspace(zStart[n-1]) ) n--; + while( sqlite3Isspace(zStart[n-1]) ) n--; return sqlite3DbStrNDup(db, zStart, n); } @@ -29564,8 +30048,13 @@ SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){ } DisableLookaside; if( db->pParse ){ + Parse *pParse; sqlite3ErrorMsg(db->pParse, "out of memory"); db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } } return 0; @@ -29770,6 +30259,20 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a u64. *msd is a divisor used to extract the +** most significant digit of *val. Extract that most significant +** digit and return it. +*/ +static char et_getdigit_int(u64 *val, u64 *msd){ + u64 x = (*val)/(*msd); + *val -= x*(*msd); + if( *msd>=10 ) *msd /= 10; + return '0' + (char)(x & 15); +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Set the StrAccum object to an error mode. */ @@ -29862,6 +30365,8 @@ SQLITE_API void sqlite3_str_vappendf( char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ LONGDOUBLE_TYPE realvalue; /* Value for real types */ + sqlite_uint64 msd; /* Divisor to get most-significant-digit + ** of longvalue */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ @@ -30168,52 +30673,78 @@ SQLITE_API void sqlite3_str_vappendf( }else{ prefix = flag_prefix; } + exp = 0; if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - exp = 0; - if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; - break; - } - if( realvalue>0.0 ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( realvalue<1.0e+16 + && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue) + ){ + /* Number is a pure integer that can be represented as u64 */ + for(msd=1; msd*10<=longvalue; msd *= 10, exp++){} + if( exp>precision && xtype!=etFLOAT ){ + u64 rnd = msd/2; + int kk = precision; + while( kk-- > 0 ){ rnd /= 10; } + longvalue += rnd; + } + }else{ + msd = 0; + longvalue = 0; /* To prevent a compiler warning */ + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } + if( sqlite3IsNaN((double)realvalue) ){ + if( flag_zeropad ){ + bufpt = "null"; + length = 4; + }else{ + bufpt = "NaN"; + length = 3; + } break; } + + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + if( ALWAYS(realvalue>0.0) ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} + while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + if( flag_zeropad ){ + realvalue = 9.0; + exp = 999; + }else{ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } + } + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + } } - bufpt = buf; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } if( xtype==etGENERIC ){ flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ @@ -30230,16 +30761,18 @@ SQLITE_API void sqlite3_str_vappendf( }else{ e2 = exp; } + nsd = 16 + flag_altform2*10; + bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; if( szBufNeeded > etBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); if( bufpt==0 ) return; } } zOut = bufpt; - nsd = 16 + flag_altform2*10; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -30248,9 +30781,15 @@ SQLITE_API void sqlite3_str_vappendf( /* Digits prior to the decimal point */ if( e2<0 ){ *(bufpt++) = '0'; + }else if( msd>0 ){ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + } }else{ for(; e2>=0; e2--){ *(bufpt++) = et_getdigit(&realvalue,&nsd); + if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; } } /* The decimal point */ @@ -30264,8 +30803,14 @@ SQLITE_API void sqlite3_str_vappendf( *(bufpt++) = '0'; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + if( msd>0 ){ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + } + }else{ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ @@ -30364,13 +30909,26 @@ SQLITE_API void sqlite3_str_vappendf( } } if( precision>1 ){ + i64 nPrior = 1; width -= precision-1; if( width>1 && !flag_leftjustify ){ sqlite3_str_appendchar(pAccum, width-1, ' '); width = 0; } - while( precision-- > 1 ){ - sqlite3_str_append(pAccum, buf, length); + sqlite3_str_append(pAccum, buf, length); + precision--; + while( precision > 1 ){ + i64 nCopyBytes; + if( nPrior > precision-1 ) nPrior = precision - 1; + nCopyBytes = length*nPrior; + if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ + sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + } + if( pAccum->accError ) break; + sqlite3_str_append(pAccum, + &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); + precision -= nPrior; + nPrior *= 2; } } bufpt = buf; @@ -30598,9 +31156,9 @@ SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExp ** Return the number of bytes of text that StrAccum is able to accept ** after the attempted enlargement. The value returned might be zero. */ -SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, int N){ +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ char *zNew; - assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==SQLITE_TOOBIG); testcase(p->accError==SQLITE_NOMEM); @@ -30611,8 +31169,7 @@ SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; - i64 szNew = p->nChar; - szNew += (sqlite3_int64)N + 1; + i64 szNew = p->nChar + N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ @@ -30642,7 +31199,8 @@ SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return 0; } } - return N; + assert( N>=0 && N<=0x7fffffff ); + return (int)N; } /* @@ -30933,12 +31491,22 @@ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_li return zBuf; } SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; + StrAccum acc; va_list ap; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); va_start(ap,zFormat); - z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - return z; + zBuf[acc.nChar] = 0; + return zBuf; } /* @@ -31238,6 +31806,13 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) if( pItem->fg.isOn || (pItem->fg.isUsing==0 && pItem->u3.pOn!=0) ){ sqlite3_str_appendf(&x, " ON"); } + if( pItem->fg.isTabFunc ) sqlite3_str_appendf(&x, " isTabFunc"); + if( pItem->fg.isCorrelated ) sqlite3_str_appendf(&x, " isCorrelated"); + if( pItem->fg.isMaterialized ) sqlite3_str_appendf(&x, " isMaterialized"); + if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); + if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); + if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1); n = 0; @@ -31507,7 +32082,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -31524,6 +32099,9 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; @@ -32335,16 +32913,41 @@ SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window *p){ sqlite3TreeViewWinFunc( ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ - unsigned char t; unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -32374,53 +32977,46 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ - if( !wsdPrng.isInit ){ + if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); if( NEVER(pVfs==0) ){ - memset(k, 0, sizeof(k)); + memset(&wsdPrng.s[4], 0, 44); }else{ - sqlite3OsRandomness(pVfs, 256, k); - } - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[15] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } @@ -33446,6 +34042,26 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *z } /* +** Check for interrupts and invoke progress callback. +*/ +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse *p){ + sqlite3 *db = p->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress && (++p->nProgressSteps)>=db->nProgressOps ){ + if( db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } + p->nProgressSteps = 0; + } +#endif +} + +/* ** Add an error message to pParse->zErrMsg and increment pParse->nErr. ** ** This function should be used to report any error that occurs while @@ -33460,7 +34076,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ va_list ap; sqlite3 *db = pParse->db; assert( db!=0 ); - assert( db->pParse==pParse ); + assert( db->pParse==pParse || db->pParse->pToplevel==pParse ); db->errByteOffset = -2; va_start(ap, zFormat); zMsg = sqlite3VMPrintf(db, zFormat, ap); @@ -33902,11 +34518,14 @@ do_atof_calc: #endif /* -** Render an signed 64-bit integer as text. Store the result in zOut[]. +** Render an signed 64-bit integer as text. Store the result in zOut[] and +** return the length of the string that was stored, in bytes. The value +** returned does not include the zero terminator at the end of the output +** string. ** ** The caller must ensure that zOut[] is at least 21 bytes in size. */ -SQLITE_PRIVATE void sqlite3Int64ToText(i64 v, char *zOut){ +SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; char zTemp[22]; @@ -33917,12 +34536,15 @@ SQLITE_PRIVATE void sqlite3Int64ToText(i64 v, char *zOut){ } i = sizeof(zTemp)-2; zTemp[sizeof(zTemp)-1] = 0; - do{ - zTemp[i--] = (x%10) + '0'; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; x = x/10; - }while( x ); - if( v<0 ) zTemp[i--] = '-'; - memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* @@ -34087,7 +34709,9 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 2; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { @@ -34123,7 +34747,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ @@ -34984,6 +35608,104 @@ SQLITE_PRIVATE int sqlite3VListNameToNum(VList *pIn, const char *zName, int nNam return 0; } +/* +** High-resolution hardware timer used for debugging and testing only. +*/ +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/************** Include hwtime.h in the middle of util.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in util.c ***********************/ +#endif + /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ /* @@ -35154,12 +35876,13 @@ static HashElem *findElementWithHash( count = pH->count; } if( pHash ) *pHash = h; - while( count-- ){ + while( count ){ assert( elem!=0 ); if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; + count--; } return &nullElement; } @@ -35278,48 +36001,48 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 5 */ "Vacuum" OpHelp(""), /* 6 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), /* 7 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 8 */ "Goto" OpHelp(""), - /* 9 */ "Gosub" OpHelp(""), - /* 10 */ "InitCoroutine" OpHelp(""), - /* 11 */ "Yield" OpHelp(""), - /* 12 */ "MustBeInt" OpHelp(""), - /* 13 */ "Jump" OpHelp(""), - /* 14 */ "Once" OpHelp(""), - /* 15 */ "If" OpHelp(""), - /* 16 */ "IfNot" OpHelp(""), - /* 17 */ "IsNullOrType" OpHelp("if typeof(r[P1]) IN (P3,5) goto P2"), - /* 18 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 8 */ "Init" OpHelp("Start at P2"), + /* 9 */ "Goto" OpHelp(""), + /* 10 */ "Gosub" OpHelp(""), + /* 11 */ "InitCoroutine" OpHelp(""), + /* 12 */ "Yield" OpHelp(""), + /* 13 */ "MustBeInt" OpHelp(""), + /* 14 */ "Jump" OpHelp(""), + /* 15 */ "Once" OpHelp(""), + /* 16 */ "If" OpHelp(""), + /* 17 */ "IfNot" OpHelp(""), + /* 18 */ "IsType" OpHelp("if typeof(P1.P3) in P5 goto P2"), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 21 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 22 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 23 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 24 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), - /* 25 */ "IfNoHope" OpHelp("key=r[P3@P4]"), - /* 26 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 27 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 28 */ "Found" OpHelp("key=r[P3@P4]"), - /* 29 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 30 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 31 */ "Last" OpHelp(""), - /* 32 */ "IfSmaller" OpHelp(""), - /* 33 */ "SorterSort" OpHelp(""), - /* 34 */ "Sort" OpHelp(""), - /* 35 */ "Rewind" OpHelp(""), - /* 36 */ "SorterNext" OpHelp(""), - /* 37 */ "Prev" OpHelp(""), - /* 38 */ "Next" OpHelp(""), - /* 39 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 40 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 20 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 21 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 29 */ "Found" OpHelp("key=r[P3@P4]"), + /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 32 */ "Last" OpHelp(""), + /* 33 */ "IfSmaller" OpHelp(""), + /* 34 */ "SorterSort" OpHelp(""), + /* 35 */ "Sort" OpHelp(""), + /* 36 */ "Rewind" OpHelp(""), + /* 37 */ "SorterNext" OpHelp(""), + /* 38 */ "Prev" OpHelp(""), + /* 39 */ "Next" OpHelp(""), + /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 46 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 47 */ "Program" OpHelp(""), - /* 48 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 49 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 48 */ "Program" OpHelp(""), + /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -35329,12 +36052,12 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 56 */ "Lt" OpHelp("IF r[P3]<r[P1]"), /* 57 */ "Ge" OpHelp("IF r[P3]>=r[P1]"), /* 58 */ "ElseEq" OpHelp(""), - /* 59 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 60 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 61 */ "IncrVacuum" OpHelp(""), - /* 62 */ "VNext" OpHelp(""), - /* 63 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 64 */ "Init" OpHelp("Start at P2"), + /* 59 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 62 */ "IncrVacuum" OpHelp(""), + /* 63 */ "VNext" OpHelp(""), + /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), /* 67 */ "Return" OpHelp(""), @@ -35463,6 +36186,988 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #endif /************** End of opcodes.c *********************************************/ +/************** Begin file os_kv.c *******************************************/ +/* +** 2022-09-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an experimental VFS layer that operates on a +** Key/Value storage engine where both keys and values must be pure +** text. +*/ +/* #include <sqliteInt.h> */ +#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)) + +/***************************************************************************** +** Debugging logic +*/ + +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +#if 0 +#define SQLITE_KV_TRACE(X) printf X +#else +#define SQLITE_KV_TRACE(X) +#endif + +/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ +#if 0 +#define SQLITE_KV_LOG(X) printf X +#else +#define SQLITE_KV_LOG(X) +#endif + + +/* +** Forward declaration of objects used by this VFS implementation +*/ +typedef struct KVVfsFile KVVfsFile; + +/* A single open file. There are only two files represented by this +** VFS - the database and the rollback journal. +*/ +struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + char *aData; /* Buffer to hold page data */ +}; +#define SQLITE_KVOS_SZ 133073 + +/* +** Methods for KVVfsFile +*/ +static int kvvfsClose(sqlite3_file*); +static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); +static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); +static int kvvfsSyncDb(sqlite3_file*, int flags); +static int kvvfsSyncJrnl(sqlite3_file*, int flags); +static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsLock(sqlite3_file*, int); +static int kvvfsUnlock(sqlite3_file*, int); +static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg); +static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg); +static int kvvfsSectorSize(sqlite3_file*); +static int kvvfsDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for sqlite3_vfs +*/ +static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); +static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int kvvfsSleep(sqlite3_vfs*, int microseconds); +static int kvvfsCurrentTime(sqlite3_vfs*, double*); +static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs sqlite3OsKvvfsObject = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +/* Methods for sqlite3_file objects referencing a database file +*/ +static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlDb, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* Methods for sqlite3_file objects referencing a rollback journal +*/ +static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlJrnl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/****** Storage subsystem **************************************************/ +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +/* Forward declarations for the low-level storage engine +*/ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 + +/* Expand the key name with an appropriate prefix and put the result +** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVSTORAGE_KEY_SZ bytes. +*/ +static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut +){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); +} + +/* Write content into a key. zClass is the particular namespace of the +** underlying key/value store to use - either "local" or "session". +** +** Both zKey and zData are zero-terminated pure text strings. +** +** Return the number of errors. +*/ +static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData +){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } +} + +/* Delete a key (with its corresponding data) from the key/value +** namespace given by zClass. If the key does not previously exist, +** this routine is a no-op. +*/ +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; +} + +/* Read the value associated with a zKey from the key/value namespace given +** by zClass and put the text data associated with that key in the first +** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large +** enough to hold it all. The value put into zBuf must always be zero +** terminated, even if it gets truncated because nBuf is not large enough. +** +** Return the total number of bytes in the data, without truncation, and +** not counting the final zero terminator. Return -1 if the key does +** not exist. +** +** If nBuf<=0 then this routine simply returns the size of the data without +** actually reading it. +*/ +static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf +){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } +} + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; + +/****** Utility subroutines ************************************************/ + +/* +** Encode binary into the text encoded used to persist on disk. +** The output text is stored in aOut[], which must be at least +** nData+1 bytes in length. +** +** Return the actual length of the encoded text, not counting the +** zero terminator at the end. +** +** Encoding format +** --------------- +** +** * Non-zero bytes are encoded as upper-case hexadecimal +** +** * A sequence of one or more zero-bytes that are not at the +** beginning of the buffer are encoded as a little-endian +** base-26 number using a..z. "a" means 0. "b" means 1, +** "z" means 25. "ab" means 26. "ac" means 52. And so forth. +** +** * Because there is no overlap between the encoding characters +** of hexadecimal and base-26 numbers, it is always clear where +** one stops and the next begins. +*/ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i<nData; i++){ + unsigned char c = a[i]; + if( c!=0 ){ + aOut[j++] = "0123456789ABCDEF"[c>>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k<nData && a[i+k]==0; k++){} + i += k-1; + while( k>0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; +} + +static const signed char kvvfsHexValue[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* +** Decode the text encoding back to binary. The binary content is +** written into pOut, which must be at least nOut bytes in length. +** +** The return value is the number of bytes actually written into aOut[]. +*/ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ + int i, j; + int c; + const unsigned char *aIn = (const unsigned char*)a; + i = 0; + j = 0; + while( 1 ){ + c = kvvfsHexValue[aIn[i]]; + if( c<0 ){ + int n = 0; + int mult = 1; + c = aIn[i]; + if( c==0 ) break; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + memset(&aOut[j], 0, n); + j += n; + if( c==0 || mult==1 ) break; /* progress stalled if mult==1 */ + }else{ + aOut[j] = c<<4; + c = kvvfsHexValue[aIn[++i]]; + if( c<0 ) break; + aOut[j++] += c; + i++; + } + } + return j; +} + +/* +** Decode a complete journal file. Allocate space in pFile->aJrnl +** and store the decoding there. Or leave pFile->aJrnl set to NULL +** if an error is encountered. +** +** The first few characters of the text encoding will be a little-endian +** base-26 number (digits a..z) that is the total number of bytes +** in the decoded journal file image. This base-26 number is followed +** by a single space, then the encoding of the journal. The space +** separator is required to act as a terminator for the base-26 number. +*/ +static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ +){ + unsigned int n = 0; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( n<pFile->nJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } +} + +/* +** Read or write the "sz" element, containing the database file size. +*/ +static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); +} +static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); +} + +/****** sqlite3_io_methods methods ******************************************/ + +/* +** Close an kvvfs-file. +*/ +static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + sqlite3_free(pFile->aData); + return SQLITE_OK; +} + +/* +** Read from the -journal file. +*/ +static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Read from the database file. +*/ +static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char *aData = pFile->aData; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + int k = iOfst+iAmt; + aData[k*2] = 0; + n = kvvfsDecode(aData, &aData[2000], SQLITE_KVOS_SZ-2000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[2000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( n<iAmt ){ + memset(zBuf+n, 0, iAmt-n); + return SQLITE_IOERR_SHORT_READ; + } + return SQLITE_OK; +} + + +/* +** Write into the -journal file. +*/ +static int kvvfsWriteJrnl( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + sqlite3_int64 iEnd = iOfst+iAmt; + SQLITE_KV_LOG(("xWrite('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnl<iEnd ){ + char *aNew = sqlite3_realloc(pFile->aJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnl<iOfst ){ + memset(pFile->aJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; +} + +/* +** Write into the database file. +*/ +static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char *aData = pFile->aData; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + assert( pFile->szPage<0 || pFile->szPage==iAmt ); + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; + } + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; +} + +/* +** Truncate an kvvfs-file. +*/ +static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; +} +static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK; + } + return SQLITE_IOERR; +} + +/* +** Sync an kvvfs-file. +*/ +static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return i ? SQLITE_IOERR : SQLITE_OK; +} +static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an kvvfs-file. +*/ +static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; +} +static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Lock an kvvfs-file. +*/ +static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Unlock an kvvfs-file. +*/ +static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an kvvfs-file. +*/ +static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an kvvfs-file. +*/ +static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op)); + return SQLITE_NOTFOUND; +} +static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on database\n", op)); + if( op==SQLITE_FCNTL_SYNC ){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + int rc = SQLITE_OK; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){ + rc = SQLITE_IOERR; + } + return rc; + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector-size in bytes for an kvvfs-file. +*/ +static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; +} + +/* +** Return the device characteristic flags supported by an kvvfs-file. +*/ +static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; +} + +/****** sqlite3_vfs methods *************************************************/ + +/* +** Open an kvvfs file handle. +*/ +static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + if( zName==0 ) zName = ""; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); + if( pFile->aData==0 ){ + return SQLITE_NOMEM; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("session", "jrnl"); + } + return SQLITE_OK; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut +){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + size_t nPath; +#ifdef SQLITE_OS_KV_ALWAYS_LOCAL + zPath = "local"; +#endif + nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut<nPath+1 ) nPath = nOut - 1; + memcpy(zOut, zPath, nPath); + zOut[nPath] = 0; + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *kvvfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return 0; +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + memset(zBufOut, 0, nByte); + return nByte; +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int kvvfsSleep(sqlite3_vfs *pVfs, int nMicro){ + return SQLITE_OK; +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int kvvfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + sqlite3_int64 i = 0; + int rc; + rc = kvvfsCurrentTimeInt64(0, &i); + *pTimeOut = i/86400000.0; + return rc; +} +#include <sys/time.h> +static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ + +#if SQLITE_OS_KV +/* +** This routine is called initialize the KV-vfs as the default VFS. +*/ +SQLITE_API int sqlite3_os_init(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1); +} +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV */ + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0); +} +#endif + +/************** End of os_kv.c ***********************************************/ /************** Begin file os_unix.c *****************************************/ /* ** 2004 May 22 @@ -35538,7 +37243,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #endif /* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) # define HAVE_PREAD 1 # define HAVE_PWRITE 1 #endif @@ -35553,15 +37258,16 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* ** standard include files. */ -#include <sys/types.h> -#include <sys/stat.h> +#include <sys/types.h> /* amalgamator: keep */ +#include <sys/stat.h> /* amalgamator: keep */ #include <fcntl.h> #include <sys/ioctl.h> -#include <unistd.h> +#include <unistd.h> /* amalgamator: keep */ /* #include <time.h> */ -#include <sys/time.h> +#include <sys/time.h> /* amalgamator: keep */ #include <errno.h> -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) # include <sys/mman.h> #endif @@ -35649,9 +37355,46 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define SQLITE_MAX_SYMLINKS 100 +/* +** Remove and stub certain info for WASI (WebAssembly System +** Interface) builds. +*/ +#ifdef SQLITE_WASI +# undef HAVE_FCHMOD +# undef HAVE_FCHOWN +# undef HAVE_MREMAP +# define HAVE_MREMAP 0 +# ifndef SQLITE_DEFAULT_UNIX_VFS +# define SQLITE_DEFAULT_UNIX_VFS "unix-dotfile" + /* ^^^ should SQLITE_DEFAULT_UNIX_VFS be "unix-none"? */ +# endif +# ifndef F_RDLCK +# define F_RDLCK 0 +# define F_WRLCK 1 +# define F_UNLCK 2 +# if __LONG_MAX == 0x7fffffffL +# define F_GETLK 12 +# define F_SETLK 13 +# define F_SETLKW 14 +# else +# define F_GETLK 5 +# define F_SETLK 6 +# define F_SETLKW 7 +# endif +# endif +#else /* !SQLITE_WASI */ +# ifndef HAVE_FCHMOD +# define HAVE_FCHMOD +# endif +#endif /* SQLITE_WASI */ + +#ifdef SQLITE_WASI +# define osGetpid(X) (pid_t)1 +#else /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ -#define osGetpid(X) (pid_t)getpid() +# define osGetpid(X) (pid_t)getpid() +#endif /* ** Only set the lastErrno if the error code is a real error and not @@ -35923,7 +37666,11 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) +#if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#else + { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -35959,14 +37706,16 @@ static struct unix_syscall { #endif #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #else { "mmap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #else { "munmap", (sqlite3_syscall_ptr)0, 0 }, @@ -36152,6 +37901,9 @@ static int robust_open(const char *z, int f, mode_t m){ break; } if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + if( (f & (O_EXCL|O_CREAT))==(O_EXCL|O_CREAT) ){ + (void)osUnlink(z); + } osClose(fd); sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); @@ -37114,7 +38866,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ ** ** UNLOCKED -> SHARED ** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE +** SHARED -> EXCLUSIVE ** RESERVED -> (PENDING) -> EXCLUSIVE ** PENDING -> EXCLUSIVE ** @@ -37147,19 +38899,20 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** A RESERVED lock is implemented by grabbing a write-lock on the ** 'reserved byte'. ** - ** A process may only obtain a PENDING lock after it has obtained a - ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock - ** on the 'pending byte'. This ensures that no new SHARED locks can be - ** obtained, but existing SHARED locks are allowed to persist. A process - ** does not have to obtain a RESERVED lock on the way to a PENDING lock. - ** This property is used by the algorithm for rolling back a journal file - ** after a crash. + ** An EXCLUSIVE lock may only be requested after either a SHARED or + ** RESERVED lock is held. An EXCLUSIVE lock is implemented by obtaining + ** a write-lock on the entire 'shared byte range'. Since all other locks + ** require a read-lock on one of the bytes within this range, this ensures + ** that no other locks are held on the database. ** - ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is - ** implemented by obtaining a write-lock on the entire 'shared byte - ** range'. Since all other locks require a read-lock on one of the bytes - ** within this range, this ensures that no other locks are held on the - ** database. + ** If a process that holds a RESERVED lock requests an EXCLUSIVE, then + ** a PENDING lock is obtained first. A PENDING lock is implemented by + ** obtaining a write-lock on the 'pending byte'. This ensures that no new + ** SHARED locks can be obtained, but existing SHARED locks are allowed to + ** persist. If the call to this function fails to obtain the EXCLUSIVE + ** lock in this case, it holds the PENDING lock intead. The client may + ** then re-attempt the EXCLUSIVE lock later on, after existing SHARED + ** locks have cleared. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -37230,7 +38983,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ lock.l_len = 1L; lock.l_whence = SEEK_SET; if( eFileLock==SHARED_LOCK - || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLock<PENDING_LOCK) + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLock==RESERVED_LOCK) ){ lock.l_type = (eFileLock==SHARED_LOCK?F_RDLCK:F_WRLCK); lock.l_start = PENDING_BYTE; @@ -37241,6 +38994,9 @@ static int unixLock(sqlite3_file *id, int eFileLock){ storeLastErrno(pFile, tErrno); } goto end_lock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; } } @@ -37328,13 +39084,9 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } #endif - if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; pInode->eFileLock = eFileLock; - }else if( eFileLock==EXCLUSIVE_LOCK ){ - pFile->eFileLock = PENDING_LOCK; - pInode->eFileLock = PENDING_LOCK; } end_lock: @@ -38741,12 +40493,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** in any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** ** To avoid stomping the errno value on a failed read the lastErrno value ** is set before returning. */ @@ -41321,6 +43067,7 @@ static const char *unixTempFileDir(void){ static int unixGetTempname(int nBuf, char *zBuf){ const char *zDir; int iLimit = 0; + int rc = SQLITE_OK; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -41329,18 +43076,26 @@ static int unixGetTempname(int nBuf, char *zBuf){ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; - do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; - }while( osAccess(zBuf,0)==0 ); - return SQLITE_OK; + if( zDir==0 ){ + rc = SQLITE_IOERR_GETTEMPPATH; + }else{ + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){ + rc = SQLITE_ERROR; + break; + } + }while( osAccess(zBuf,0)==0 ); + } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) @@ -41916,12 +43671,10 @@ static void appendOnePathElement( if( zName[0]=='.' ){ if( nName==1 ) return; if( zName[1]=='.' && nName==2 ){ - if( pPath->nUsed<=1 ){ - pPath->rc = SQLITE_ERROR; - return; + if( pPath->nUsed>1 ){ + assert( pPath->zOut[0]=='/' ); + while( pPath->zOut[--pPath->nUsed]!='/' ){} } - assert( pPath->zOut[0]=='/' ); - while( pPath->zOut[--pPath->nUsed]!='/' ){} return; } } @@ -42133,7 +43886,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS +#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L struct timespec sp; sp.tv_sec = microseconds / 1000000; @@ -43515,8 +45268,16 @@ SQLITE_API int sqlite3_os_init(void){ /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ +#ifdef SQLITE_DEFAULT_UNIX_VFS + sqlite3_vfs_register(&aVfs[i], + 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS)); +#else sqlite3_vfs_register(&aVfs[i], i==0); +#endif } +#ifdef SQLITE_OS_KV_OPTIONAL + sqlite3KvvfsInit(); +#endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); #ifndef SQLITE_OMIT_WAL @@ -45479,10 +47240,12 @@ SQLITE_API int sqlite3_win32_set_directory8( const char *zValue /* New value for directory being set or reset */ ){ char **ppDirectory = 0; + int rc; #ifndef SQLITE_OMIT_AUTOINIT - int rc = sqlite3_initialize(); + rc = sqlite3_initialize(); if( rc ) return rc; #endif + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ ppDirectory = &sqlite3_data_directory; }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ @@ -45497,14 +47260,19 @@ SQLITE_API int sqlite3_win32_set_directory8( if( zValue && zValue[0] ){ zCopy = sqlite3_mprintf("%s", zValue); if ( zCopy==0 ){ - return SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM_BKPT; + goto set_directory8_done; } } sqlite3_free(*ppDirectory); *ppDirectory = zCopy; - return SQLITE_OK; + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; } - return SQLITE_ERROR; +set_directory8_done: + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } /* @@ -48279,6 +50047,19 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ } /* +** If sqlite3_temp_directory is defined, take the mutex and return true. +** +** If sqlite3_temp_directory is NULL (undefined), omit the mutex and +** return false. +*/ +static int winTempDirDefined(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + if( sqlite3_temp_directory!=0 ) return 1; + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return 0; +} + +/* ** Create a temporary file name and store the resulting pointer into pzBuf. ** The pointer returned in pzBuf must be freed via sqlite3_free(). */ @@ -48314,20 +50095,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ */ nDir = nMax - (nPre + 15); assert( nDir>0 ); - if( sqlite3_temp_directory ){ + if( winTempDirDefined() ){ int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); if( nDirLen>0 ){ if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ nDirLen++; } if( nDirLen>nDir ){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); } sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); } + #if defined(__CYGWIN__) else{ static const char *azDirs[] = { @@ -48735,7 +50519,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -48752,7 +50536,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -48772,7 +50556,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -48995,6 +50779,13 @@ static int winAccess( OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); + if( zFilename==0 ){ + *pResOut = 0; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); @@ -49116,7 +50907,7 @@ static BOOL winIsVerbatimPathname( ** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname ** bytes in size. */ -static int winFullPathname( +static int winFullPathnameNoMutex( sqlite3_vfs *pVfs, /* Pointer to vfs object */ const char *zRelative, /* Possibly relative input path */ int nFull, /* Size of output buffer in bytes */ @@ -49295,6 +51086,20 @@ static int winFullPathname( } #endif } +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + int rc; + MUTEX_LOGIC( sqlite3_mutex *pMutex; ) + MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); ) + sqlite3_mutex_enter(pMutex); + rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); + sqlite3_mutex_leave(pMutex); + return rc; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -49831,6 +51636,7 @@ static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); static int memdbSync(sqlite3_file*, int flags); static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); static int memdbLock(sqlite3_file*, int); +static int memdbUnlock(sqlite3_file*, int); /* static int memdbCheckReservedLock(sqlite3_file*, int *pResOut);// not used */ static int memdbFileControl(sqlite3_file*, int op, void *pArg); /* static int memdbSectorSize(sqlite3_file*); // not used */ @@ -49889,7 +51695,7 @@ static const sqlite3_io_methods memdb_io_methods = { memdbSync, /* xSync */ memdbFileSize, /* xFileSize */ memdbLock, /* xLock */ - memdbLock, /* xUnlock - same as xLock in this case */ + memdbUnlock, /* xUnlock */ 0, /* memdbCheckReservedLock, */ /* xCheckReservedLock */ memdbFileControl, /* xFileControl */ 0, /* memdbSectorSize,*/ /* xSectorSize */ @@ -50090,39 +51896,81 @@ static int memdbLock(sqlite3_file *pFile, int eLock){ MemFile *pThis = (MemFile*)pFile; MemStore *p = pThis->pStore; int rc = SQLITE_OK; - if( eLock==pThis->eLock ) return SQLITE_OK; + if( eLock<=pThis->eLock ) return SQLITE_OK; memdbEnter(p); - if( eLock>SQLITE_LOCK_SHARED ){ - if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ - rc = SQLITE_READONLY; - }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ - if( p->nWrLock ){ - rc = SQLITE_BUSY; - }else{ - p->nWrLock = 1; + + assert( p->nWrLock==0 || p->nWrLock==1 ); + assert( pThis->eLock<=SQLITE_LOCK_SHARED || p->nWrLock==1 ); + assert( pThis->eLock==SQLITE_LOCK_NONE || p->nRdLock>=1 ); + + if( eLock>SQLITE_LOCK_SHARED && (p->mFlags & SQLITE_DESERIALIZE_READONLY) ){ + rc = SQLITE_READONLY; + }else{ + switch( eLock ){ + case SQLITE_LOCK_SHARED: { + assert( pThis->eLock==SQLITE_LOCK_NONE ); + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nRdLock++; + } + break; + }; + + case SQLITE_LOCK_RESERVED: + case SQLITE_LOCK_PENDING: { + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( ALWAYS(pThis->eLock==SQLITE_LOCK_SHARED) ){ + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; + } + } + break; + } + + default: { + assert( eLock==SQLITE_LOCK_EXCLUSIVE ); + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( p->nRdLock>1 ){ + rc = SQLITE_BUSY; + }else if( pThis->eLock==SQLITE_LOCK_SHARED ){ + p->nWrLock = 1; + } + break; } } - }else if( eLock==SQLITE_LOCK_SHARED ){ - if( pThis->eLock > SQLITE_LOCK_SHARED ){ - assert( p->nWrLock==1 ); - p->nWrLock = 0; - }else if( p->nWrLock ){ - rc = SQLITE_BUSY; - }else{ - p->nRdLock++; + } + if( rc==SQLITE_OK ) pThis->eLock = eLock; + memdbLeave(p); + return rc; +} + +/* +** Unlock an memdb-file. +*/ +static int memdbUnlock(sqlite3_file *pFile, int eLock){ + MemFile *pThis = (MemFile*)pFile; + MemStore *p = pThis->pStore; + if( eLock>=pThis->eLock ) return SQLITE_OK; + memdbEnter(p); + + assert( eLock==SQLITE_LOCK_SHARED || eLock==SQLITE_LOCK_NONE ); + if( eLock==SQLITE_LOCK_SHARED ){ + if( ALWAYS(pThis->eLock>SQLITE_LOCK_SHARED) ){ + p->nWrLock--; } }else{ - assert( eLock==SQLITE_LOCK_NONE ); if( pThis->eLock>SQLITE_LOCK_SHARED ){ - assert( p->nWrLock==1 ); - p->nWrLock = 0; + p->nWrLock--; } - assert( p->nRdLock>0 ); p->nRdLock--; } - if( rc==SQLITE_OK ) pThis->eLock = eLock; + + pThis->eLock = eLock; memdbLeave(p); - return rc; + return SQLITE_OK; } #if 0 @@ -50232,7 +52080,7 @@ static int memdbOpen( memset(pFile, 0, sizeof(*pFile)); szName = sqlite3Strlen30(zName); - if( szName>1 && zName[0]=='/' ){ + if( szName>1 && (zName[0]=='/' || zName[0]=='\\') ){ int i; #ifndef SQLITE_MUTEX_OMIT sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); @@ -50580,6 +52428,13 @@ end_deserialize: } /* +** Return true if the VFS is the memvfs. +*/ +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs *pVfs){ + return pVfs==&memdb_vfs; +} + +/* ** This routine is called when the extension is loaded. ** Register the new VFS. */ @@ -51058,7 +52913,7 @@ bitvec_end: struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRefSum; /* Sum of ref counts over all pages */ + i64 nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ @@ -51083,12 +52938,24 @@ struct PCache { int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ # define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} - void pcacheDump(PCache *pCache){ - int N; - int i, j; - sqlite3_pcache_page *pLower; + static void pcachePageTrace(int i, sqlite3_pcache_page *pLower){ PgHdr *pPg; unsigned char *a; + int j; + if( pLower==0 ){ + printf("%3d: NULL\n", i); + }else{ + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } + } + static void pcacheDump(PCache *pCache){ + int N; + int i; + sqlite3_pcache_page *pLower; if( sqlite3PcacheTrace<2 ) return; if( pCache->pCache==0 ) return; @@ -51096,23 +52963,33 @@ struct PCache { if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf("\n"); - if( pPg->pPage==0 ){ + pcachePageTrace(i, pLower); + if( pLower && ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } } - #else +#else # define pcacheTrace(X) +# define pcachePageTrace(PGNO, X) # define pcacheDump(X) #endif /* +** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. +** This routine runs inside of assert() statements only. +*/ +#ifdef SQLITE_DEBUG +static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 1; + } + return 0; +} +#endif + +/* ** Check invariants on a PgHdr entry. Return true if everything is OK. ** Return false if any invariant is violated. ** @@ -51130,8 +53007,13 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ - assert( pCache->pDirtyTail!=pPg ); + assert( !pageOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirty list */ + }else{ + assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ + assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); + assert( pPg->pDirtyPrev==0 || pPg->pDirtyPrev->pDirtyNext==pPg ); + assert( pPg->pDirtyPrev!=0 || pCache->pDirty==pPg ); + assert( pageOnDirtyList(pCache, pPg) ); } /* WRITEABLE pages must also be DIRTY */ if( pPg->flags & PGHDR_WRITEABLE ){ @@ -51405,8 +53287,9 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + pcacheTrace(("%p.FETCH %d%s (result: %p) ",pCache,pgno, createFlag?" create":"",pRes)); + pcachePageTrace(pgno, pRes); return pRes; } @@ -51534,6 +53417,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ pcacheUnpin(p); }else{ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } } @@ -51577,6 +53461,7 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + assert( sqlite3PcachePageSanity(p) ); } assert( sqlite3PcachePageSanity(p) ); } @@ -51639,14 +53524,24 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; + sqlite3_pcache_page *pOther; assert( p->nRef>0 ); assert( newPgno>0 ); assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); + if( pOther ){ + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); + } sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } @@ -51795,14 +53690,14 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ ** This is not the total number of pages referenced, but the sum of the ** reference count for all pages. */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache *pCache){ return pCache->nRefSum; } /* ** Return the number of references to the page supplied as an argument. */ -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr *p){ return p->nRef; } @@ -51944,12 +53839,13 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. +** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER +** was defined, then the page content would be held in a separate memory +** allocation from the PgHdr1. This was intended to avoid clownshoe memory +** allocations. However, the btree layer needs a small (16-byte) overrun +** area after the page content buffer. The header serves as that overrun +** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid +** any possibility of a memory error. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. @@ -51994,30 +53890,40 @@ typedef struct PGroup PGroup; /* ** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure and is used to cache the page content. +** +** When reading a corrupt database file, it is possible that SQLite might +** read a few bytes (no more than 16 bytes) past the end of the page buffer. +** It will only read past the end of the page buffer, never write. This +** object is positioned immediately after the page buffer to serve as an +** overrun area, so that overreads are harmless. ** -** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** Variables isBulkLocal and isAnchor were once type "u8". That works, ** but causes a 2-byte gap in the structure for most architectures (since ** pointers must be either 4 or 8-byte aligned). As this structure is located ** in memory directly after the associated page data, if the database is ** corrupt, code at the b-tree layer may overread the page buffer and ** read part of this structure before the corruption is detected. This ** can cause a valgrind error if the unitialized gap is accessed. Using u16 -** ensures there is no such gap, and therefore no bytes of unitialized memory -** in the structure. +** ensures there is no such gap, and therefore no bytes of uninitialized +** memory in the structure. +** +** The pLruNext and pLruPrev pointers form a double-linked circular list +** of all pages that are unpinned. The PGroup.lru element (which should be +** the only element on the list with PgHdr1.isAnchor set to 1) forms the +** beginning and the end of the list. */ struct PgHdr1 { - sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ - unsigned int iKey; /* Key value (page number) */ - u16 isBulkLocal; /* This page from bulk local storage */ - u16 isAnchor; /* This is the PGroup.lru element */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ - /* NB: pLruPrev is only valid if pLruNext!=0 */ + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in circular LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* @@ -52343,25 +54249,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ pcache1LeaveMutex(pCache->pGroup); #endif if( benignMalloc ){ sqlite3BeginBenignMalloc(); } -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else pPg = pcache1Alloc(pCache->szAlloc); -#endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; -#ifndef SQLITE_PCACHE_SEPARATE_HEADER p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; @@ -52385,9 +54279,6 @@ static void pcache1FreePage(PgHdr1 *p){ pCache->pFree = p; }else{ pcache1Free(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - sqlite3_free(p); -#endif } (*pCache->pnPurgeable)--; } @@ -53028,23 +54919,26 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int h; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); + assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ + hOld = iOld%pCache->nHash; + pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; } *pp = pPage->pNext; - h = iNew%pCache->nHash; + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ + hNew = iNew%pCache->nHash; pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; + pPage->pNext = pCache->apHash[hNew]; + pCache->apHash[hNew] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } @@ -53151,9 +55045,6 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ && p->isAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - nFree += sqlite3MemSize(p); -#endif assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); @@ -56461,6 +58352,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState!=PAGER_READER ); + PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); + if( isOpen(pPager->fd) && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) @@ -56791,7 +58684,7 @@ end_playback: ** see if it is possible to delete the super-journal. */ assert( zSuper==&pPager->pTmpSpace[4] ); - memset(&zSuper[-4], 0, 4); + memset(pPager->pTmpSpace, 0, 4); rc = pager_delsuper(pPager, zSuper); testcase( rc!=SQLITE_OK ); } @@ -57412,7 +59305,6 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** Numeric values associated with these states are OFF==1, NORMAL=2, ** and FULL=3. */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS SQLITE_PRIVATE void sqlite3PagerSetFlags( Pager *pPager, /* The pager to set safety level for */ unsigned pgFlags /* Various flags */ @@ -57447,7 +59339,6 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( pPager->doNotSpill |= SPILLFLAG_OFF; } } -#endif /* ** The following global variable is incremented whenever the library @@ -58549,7 +60440,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ - int nUri = 0; /* Number of URI parameters */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -58597,7 +60487,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( while( *z ){ z += strlen(z)+1; z += strlen(z)+1; - nUri++; } nUriByte = (int)(&z[1] - zUri); assert( nUriByte>=1 ); @@ -58853,18 +60742,7 @@ act_like_temp_file: pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; assert( useJournal || pPager->tempFile ); - pPager->noSync = pPager->tempFile; - if( pPager->noSync ){ - assert( pPager->fullSync==0 ); - assert( pPager->extraSync==0 ); - assert( pPager->syncFlags==0 ); - assert( pPager->walSyncFlags==0 ); - }else{ - pPager->fullSync = 1; - pPager->extraSync = 0; - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); - } + sqlite3PagerSetFlags(pPager, (SQLITE_DEFAULT_SYNCHRONOUS+1)|PAGER_CACHESPILL); /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ @@ -59393,6 +61271,10 @@ static int getPageNormal( if( !isOpen(pPager->fd) || pPager->dbSize<pgno || noContent ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; + if( pgno<=pPager->dbSize ){ + sqlite3PcacheRelease(pPg); + pPg = 0; + } goto pager_acquire_err; } if( noContent ){ @@ -59557,10 +61439,12 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ /* ** Release a page reference. ** -** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be -** used if we know that the page being released is not the last page. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used +** if we know that the page being released is not the last reference to page1. ** The btree layer always holds page1 open until the end, so these first -** to routines can be used to release any page other than BtShared.pPage1. +** two routines can be used to release any page other than BtShared.pPage1. +** The assert() at tag-20230419-2 proves that this constraint is always +** honored. ** ** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine ** checks the total number of outstanding pages and if the number of @@ -59576,7 +61460,7 @@ SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ sqlite3PcacheRelease(pPg); } /* Do not use this routine to release the last reference to page1 */ - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */ } SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); @@ -59642,6 +61526,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + flags |= SQLITE_OPEN_EXCLUSIVE; nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; @@ -59677,6 +61562,7 @@ static int pager_open_journal(Pager *pPager){ if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + pPager->journalOff = 0; }else{ assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; @@ -60123,7 +62009,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -60863,7 +62749,11 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ */ SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; + if( nullIfMemDb && (pPager->memDb || sqlite3IsMemdb(pPager->pVfs)) ){ + return &zFake[4]; + }else{ + return pPager->zFilename; + } } /* @@ -61330,13 +63220,15 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ */ static int pagerExclusiveLock(Pager *pPager){ int rc; /* Return code */ + u8 eOrigLock; /* Original lock */ - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->eLock>=SHARED_LOCK ); + eOrigLock = pPager->eLock; rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ /* If the attempt to grab the exclusive lock failed, release the ** pending lock that may have been obtained instead. */ - pagerUnlockDb(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, eOrigLock); } return rc; @@ -62341,19 +64233,40 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ + if( !nativeCksum ){ + do { + s1 += BYTESWAP32(aData[0]) + s2; + s2 += BYTESWAP32(aData[1]) + s1; + aData += 2; + }while( aData<aEnd ); + }else if( nByte%64==0 ){ do { s1 += *aData++ + s2; s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; + s1 += *aData++ + s2; + s2 += *aData++ + s1; }while( aData<aEnd ); }else{ do { - s1 += BYTESWAP32(aData[0]) + s2; - s2 += BYTESWAP32(aData[1]) + s1; - aData += 2; + s1 += *aData++ + s2; + s2 += *aData++ + s1; }while( aData<aEnd ); } + assert( aData==aEnd ); aOut[0] = s1; aOut[1] = s2; @@ -65284,7 +67197,9 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; } } - assert( (int)pWal->szPage==szPage ); + if( (int)pWal->szPage!=szPage ){ + return SQLITE_CORRUPT_BKPT; /* TH3 test case: cov1/corrupt155.test */ + } /* Setup information needed to write frames into the WAL */ w.pWal = pWal; @@ -65944,7 +67859,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** byte are used. The integer consists of all bytes that have bit 8 set and ** the first byte with bit 8 clear. The most significant byte of the integer ** appears first. A variable-length integer may not be more than 9 bytes long. -** As a special case, all 8 bytes of the 9th byte are used as data. This +** As a special case, all 8 bits of the 9th byte are used as data. This ** allows a 64-bit integer to be encoded in 9 bytes. ** ** 0x00 becomes 0x00000000 @@ -66328,7 +68243,7 @@ struct BtCursor { #define BTCF_WriteFlag 0x01 /* True if a write cursor */ #define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ -#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_AtLast 0x08 /* Cursor is pointing to the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ #define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ @@ -66446,15 +68361,15 @@ struct BtCursor { ** So, this macro is defined instead. */ #ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) +#define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else -#define ISAUTOVACUUM 0 +#define ISAUTOVACUUM(pBt) 0 #endif /* -** This structure is passed around through all the sanity checking routines -** in order to keep track of some global state information. +** This structure is passed around through all the PRAGMA integrity_check +** checking routines in order to keep track of some global state information. ** ** The aRef[] array is allocated so that there is 1 bit for each page in ** the database. As the integrity-check proceeds, for each page used in @@ -66470,10 +68385,12 @@ struct IntegrityCk { Pgno nPage; /* Number of pages in the database */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ - int bOomFault; /* A memory allocation error has occurred */ + int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ + u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ - Pgno v1; /* Value for first %u substitution in zPfx */ - int v2; /* Value for second %d substitution in zPfx */ + Pgno v0; /* Value for first %u substitution in zPfx (root page) */ + Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ + int v2; /* Value for third %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ @@ -66740,6 +68657,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ Btree *p; assert( db!=0 ); + if( db->pVfs==0 && db->nDb==0 ) return 1; if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); assert( iDb>=0 && iDb<db->nDb ); if( !sqlite3_mutex_held(db->mutex) ) return 0; @@ -66935,8 +68853,8 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ int corruptPageError(int lineno, MemPage *p){ char *zMsg; sqlite3BeginBenignMalloc(); - zMsg = sqlite3_mprintf("database corruption page %d of %s", - (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + zMsg = sqlite3_mprintf("database corruption page %u of %s", + p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) ); sqlite3EndBenignMalloc(); if( zMsg ){ @@ -67745,8 +69663,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) */ SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ /* Used only by system that substitute their own storage engine */ +#ifdef SQLITE_DEBUG + if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){ + va_list ap; + Expr *pExpr; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3CursorRangeHintExprCheck; + va_start(ap, eHintType); + pExpr = va_arg(ap, Expr*); + w.u.aMem = va_arg(ap, Mem*); + va_end(ap); + assert( pExpr!=0 ); + assert( w.u.aMem!=0 ); + sqlite3WalkExpr(&w, pExpr); + } +#endif /* SQLITE_DEBUG */ } -#endif +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* ** Provide flag hints to the cursor. @@ -67831,7 +69766,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; @@ -68030,27 +69965,31 @@ static void btreeParseCellPtr( iKey = *pIter; if( iKey>=0x80 ){ u8 x; - iKey = ((iKey&0x7f)<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x =*++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<8) | (*++pIter); + iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter); } } } } } + }else{ + iKey ^= 0x204000; } + }else{ + iKey ^= 0x4000; } } pIter++; @@ -68127,10 +70066,11 @@ static void btreeParseCell( ** ** cellSizePtrNoPayload() => table internal nodes ** cellSizePtrTableLeaf() => table leaf nodes -** cellSizePtr() => all index nodes & table leaf nodes +** cellSizePtr() => index internal nodes +** cellSizeIdxLeaf() => index leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ u8 *pEnd; /* End mark for a varint */ u32 nSize; /* Size value to return */ @@ -68143,6 +70083,49 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif + assert( pPage->childPtrSize==4 ); + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIter<pEnd ); + } + pIter++; + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + assert( nSize>4 ); + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + assert( pPage->childPtrSize==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -68312,8 +70295,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = 0; - src = data = pPage->aData; + data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; @@ -68347,7 +70329,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; - }else if( NEVER(iFree+sz>usableSize) ){ + }else if( iFree+sz>usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -68367,39 +70349,38 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ cbrk = usableSize; iCellLast = usableSize - 4; iCellStart = get2byte(&data[hdr+5]); - for(i=0; i<nCell; i++){ - u8 *pAddr; /* The i-th cell pointer */ - pAddr = &data[cellOffset + i*2]; - pc = get2byte(pAddr); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - /* These conditions have already been verified in btreeInitPage() - ** if PRAGMA cell_size_check=ON. - */ - if( pc<iCellStart || pc>iCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( pc>=iCellStart && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); - cbrk -= size; - if( cbrk<iCellStart || pc+size>usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( cbrk+size<=usableSize && cbrk>=iCellStart ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - put2byte(pAddr, cbrk); - if( temp==0 ){ - if( cbrk==pc ) continue; - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); - src = temp; + if( nCell>0 ){ + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + src = temp; + for(i=0; i<nCell; i++){ + u8 *pAddr; /* The i-th cell pointer */ + pAddr = &data[cellOffset + i*2]; + pc = get2byte(pAddr); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + /* These conditions have already been verified in btreeInitPage() + ** if PRAGMA cell_size_check=ON. + */ + if( pc>iCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=0 && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrk<iCellStart || pc+size>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + memcpy(&data[cbrk], &src[pc], size); } - memcpy(&data[cbrk], &src[pc], size); } data[hdr+7] = 0; - defragment_out: +defragment_out: assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); @@ -68456,7 +70437,6 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; - testcase( pc+x>maxPC ); return &aData[pc]; }else if( x+pc > maxPC ){ /* This slot extends off the end of the usable part of the page */ @@ -68472,9 +70452,9 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ iAddr = pc; pTmp = &aData[pc]; pc = get2byte(pTmp); - if( pc<=iAddr+size ){ + if( pc<=iAddr ){ if( pc ){ - /* The next slot in the chain is not past the end of the current slot */ + /* The next slot in the chain comes before the current slot */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } return 0; @@ -68500,7 +70480,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** allocation is being made in order to insert a new cell, so we will ** also end up needing a new cell pointer. */ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ +static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ @@ -68526,13 +70506,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** integer, so a value of 0 is used in its place. */ pTmp = &data[hdr+5]; top = get2byte(pTmp); - assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ return SQLITE_CORRUPT_PAGE(pPage); } + }else if( top>(int)pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } /* If there is enough space between gap and top for one more cell pointer, @@ -68615,7 +70596,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -68626,7 +70607,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){ - if( iFreeBlk<iPtr+4 ){ + if( iFreeBlk<=iPtr ){ if( iFreeBlk==0 ) break; /* TH3: corrupt082.100 */ return SQLITE_CORRUPT_PAGE(pPage); } @@ -68672,6 +70653,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } pTmp = &data[hdr+5]; x = get2byte(pTmp); + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another @@ -68683,14 +70669,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ }else{ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[iStart], 0, iSize); - } - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); pPage->nFree += iOrigSize; return SQLITE_OK; } @@ -68702,62 +70683,67 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ ** Only the following combinations are supported. Anything different ** indicates a corrupt database files: ** -** PTF_ZERODATA -** PTF_ZERODATA | PTF_LEAF -** PTF_LEAFDATA | PTF_INTKEY -** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +** PTF_ZERODATA (0x02, 2) +** PTF_LEAFDATA | PTF_INTKEY (0x05, 5) +** PTF_ZERODATA | PTF_LEAF (0x0a, 10) +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF (0x0d, 13) */ static int decodeFlags(MemPage *pPage, int flagByte){ BtShared *pBt; /* A copy of pPage->pBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); - pPage->intKey = 1; - if( pPage->leaf ){ + pPage->max1bytePayload = pBt->max1bytePayload; + if( flagByte>=(PTF_ZERODATA | PTF_LEAF) ){ + pPage->childPtrSize = 0; + pPage->leaf = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF) ){ pPage->intKeyLeaf = 1; pPage->xCellSize = cellSizePtrTableLeaf; pPage->xParseCell = btreeParseCellPtr; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); + } + }else{ + pPage->childPtrSize = 4; + pPage->leaf = 0; + if( flagByte==(PTF_ZERODATA) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); } - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ - assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ - assert( (PTF_ZERODATA|PTF_LEAF)==10 ); - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is - ** an error. */ - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - return SQLITE_CORRUPT_PAGE(pPage); } - pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; } @@ -69108,9 +71094,7 @@ getAndInitPage_error1: pCur->pPage = pCur->apPage[pCur->iPage]; } testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT - || rc==SQLITE_IOERR_NOMEM - || rc==SQLITE_NOMEM ); + assert( pgno!=0 || rc!=SQLITE_OK ); return rc; } @@ -70546,6 +72530,9 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } } }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } if( get4byte(pCell)==iFrom ){ put4byte(pCell, iTo); break; @@ -70594,7 +72581,7 @@ static int relocatePage( if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -72052,8 +74039,6 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - BtShared *pBt = pCur->pBt; - assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage<BTCURSOR_MAX_DEPTH ); @@ -72067,7 +74052,8 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ pCur->apPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); + return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur, + pCur->curPagerFlags); } #ifdef SQLITE_DEBUG @@ -72173,7 +74159,7 @@ static int moveToRoot(BtCursor *pCur){ } sqlite3BtreeClearCursor(pCur); } - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, + rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; @@ -72297,9 +74283,25 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ +static SQLITE_NOINLINE int btreeLast(BtCursor *pCur, int *pRes){ + int rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -72320,23 +74322,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ *pRes = 0; return SQLITE_OK; } - - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - if( rc==SQLITE_OK ){ - pCur->curFlags |= BTCF_AtLast; - }else{ - pCur->curFlags &= ~BTCF_AtLast; - } - }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); - *pRes = 1; - rc = SQLITE_OK; - } - return rc; + return btreeLast(pCur, pRes); } /* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) @@ -72881,14 +74867,8 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( !pPage->isInit || sqlite3FaultSim(412) ){ - /* The only known way for this to happen is for there to be a - ** recursive SQL function that does a DELETE operation as part of a - ** SELECT which deletes content out from under an active cursor - ** in a corrupt database file where the table being DELETE-ed from - ** has pages in common with the table being queried. See TH3 - ** module cov1/btree78.test testcase 220 (2018-06-08) for an - ** example. */ + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } @@ -73064,8 +75044,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); - /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 - ** stores stores the total number of pages on the freelist. */ + /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36 + ** stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -73151,7 +75131,7 @@ static int allocateBtreePage( memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); *ppPage = pTrunk; pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ rc = SQLITE_CORRUPT_PGNO(iTrunk); @@ -73217,7 +75197,7 @@ static int allocateBtreePage( } } pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); #endif }else if( k>0 ){ /* Extract a leaf from the trunk */ @@ -73262,8 +75242,8 @@ static int allocateBtreePage( ){ int noContent; *pPgno = iPage; - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" - ": %d more free pages\n", + TRACE(("ALLOCATE: %u was leaf %u of %u on trunk %u" + ": %u more free pages\n", *pPgno, closest+1, k, pTrunk->pgno, n-1)); rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc ) goto end_allocate_page; @@ -73319,7 +75299,7 @@ static int allocateBtreePage( ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ @@ -73342,7 +75322,7 @@ static int allocateBtreePage( releasePage(*ppPage); *ppPage = 0; } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + TRACE(("ALLOCATE: %u from end of file\n", *pPgno)); } assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); @@ -73410,7 +75390,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } @@ -73470,7 +75450,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno)); goto freepage_out; } } @@ -73491,7 +75471,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ put4byte(pPage->aData, iTrunk); put4byte(&pPage->aData[4], 0); put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk)); freepage_out: if( pPage ){ @@ -73814,12 +75794,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( pPage->pBt->usableSize > (u32)(ptr-data) ); pc = get2byte(ptr); hdr = pPage->hdrOffset; -#if 0 /* Not required. Omit for efficiency */ - if( pc<hdr+pPage->nCell*2 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } -#endif testcase( pc==(u32)get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); if( pc+sz > pPage->pBt->usableSize ){ @@ -73857,23 +75831,27 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. ** -** *pRC must be SQLITE_OK when this routine is called. +** The insertCellFast() routine below works exactly the same as +** insertCell() except that it lacks the pTemp and iChild parameters +** which are assumed zero. Other than that, the two routines are the +** same. +** +** Fixes or enhancements to this routine should be reflected in +** insertCellFast()! */ -static void insertCell( +static int insertCell( MemPage *pPage, /* Page into which we are copying */ int i, /* New cell becomes the i-th cell of the page */ u8 *pCell, /* Content of the new cell */ int sz, /* Bytes of content in pCell */ u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ + Pgno iChild /* If non-zero, replace first 4 bytes with this value */ ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -73882,14 +75860,103 @@ static void insertCell( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); assert( pPage->nFree>=0 ); + assert( iChild>0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); pCell = pTemp; } - if( iChild ){ - put4byte(pCell, iChild); + put4byte(pCell, iChild); + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( NEVER(rc!=SQLITE_OK) ){ + return rc; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ return rc; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } +#endif + } + return SQLITE_OK; +} + +/* +** This variant of insertCell() assumes that the pTemp and iChild +** parameters are both zero. Use this variant in sqlite3BtreeInsert() +** for performance improvement, and also so that this variant is only +** called from that one place, and is thus inlined, and thus runs must +** faster. +** +** Fixes or enhancements to this routine should be reflected into +** the insertCell() routine. +*/ +static int insertCellFast( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz /* Bytes of content in pCell */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + assert( pPage->nOverflow==0 ); + if( sz+2>pPage->nFree ){ j = pPage->nOverflow++; /* Comparison against ArraySize-1 since we hold back one extra slot ** as a contingency. In other words, never need more than 3 overflow @@ -73908,31 +75975,20 @@ static void insertCell( }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; + return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } + if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - if( iChild ){ - /* In a corrupt database where an entry in the cell index section of - ** a btree page has a value of 3 or less, the pCell value might point - ** as many as 4 bytes in front of the start of the aData buffer for - ** the source page. Make sure this does not cause problems by not - ** reading the first 4 bytes */ - memcpy(&data[idx+4], pCell+4, sz-4); - put4byte(&data[idx], iChild); - }else{ - memcpy(&data[idx], pCell, sz); - } + memcpy(&data[idx], pCell, sz); pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); @@ -73942,13 +75998,16 @@ static void insertCell( assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } #endif } + return SQLITE_OK; } /* @@ -74049,14 +76108,16 @@ struct CellArray { ** computed. */ static void populateCellCache(CellArray *p, int idx, int N){ + MemPage *pRef = p->pRef; + u16 *szCell = p->szCell; assert( idx>=0 && idx+N<=p->nCell ); while( N>0 ){ assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + if( szCell[idx]==0 ){ + szCell[idx] = pRef->xCellSize(pRef, p->apCell[idx]); }else{ assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + szCell[idx]==pRef->xCellSize(pRef, p->apCell[idx]) ); } idx++; N--; @@ -74112,7 +76173,7 @@ static int rebuildPage( assert( i<iEnd ); j = get2byte(&aData[hdr+5]); - if( j>(u32)usableSize ){ j = 0; } + if( NEVER(j>(u32)usableSize) ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){} @@ -74256,39 +76317,50 @@ static int pageFreeArray( u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; - int i; + int i, j; int iEnd = iFirst + nCell; - u8 *pFree = 0; - int szFree = 0; + int nFree = 0; + int aOfst[10]; + int aAfter[10]; for(i=iFirst; i<iEnd; i++){ u8 *pCell = pCArray->apCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; + int iAfter; + int iOfst; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which ** cells need freeing */ sz = pCArray->szCell[i]; assert( sz>0 ); - if( pFree!=(pCell + sz) ){ - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); - } - pFree = pCell; - szFree = sz; - if( pFree+sz>pEnd ){ - return 0; + iOfst = (u16)(pCell - aData); + iAfter = iOfst+sz; + for(j=0; j<nFree; j++){ + if( aOfst[j]==iAfter ){ + aOfst[j] = iOfst; + break; + }else if( aAfter[j]==iOfst ){ + aAfter[j] = iAfter; + break; } - }else{ - pFree = pCell; - szFree += sz; + } + if( j>=nFree ){ + if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){ + for(j=0; j<nFree; j++){ + freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]); + } + nFree = 0; + } + aOfst[nFree] = iOfst; + aAfter[nFree] = iAfter; + if( &aData[iAfter]>pEnd ) return 0; + nFree++; } nRet++; } } - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + for(j=0; j<nFree; j++){ + freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]); } return nRet; } @@ -74341,9 +76413,9 @@ static int editPage( nCell -= nTail; } - pData = &aData[get2byteNotZero(&aData[hdr+5])]; + pData = &aData[get2byte(&aData[hdr+5])]; if( pData<pBegin ) goto editpage_fail; - if( pData>pPg->aDataEnd ) goto editpage_fail; + if( NEVER(pData>pPg->aDataEnd) ) goto editpage_fail; /* Add cells to the start of the page */ if( iNew<iOld ){ @@ -74487,7 +76559,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); @@ -74515,8 +76587,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ /* Insert the new divider cell into pParent. */ if( rc==SQLITE_OK ){ - insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + rc = insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno); } /* Set the right-child pointer of pParent to point to the new page. */ @@ -74625,7 +76697,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ *pRC = setChildPtrmaps(pTo); } } @@ -74703,8 +76775,6 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ - Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ - u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); @@ -75051,15 +77121,17 @@ static int balance_nonroot( d = r + 1 - leafData; (void)cachedCellSize(&b, d); do{ + int szR, szD; assert( d<nMaxCells ); assert( r<nMaxCells ); - (void)cachedCellSize(&b, r); + szR = cachedCellSize(&b, r); + szD = b.szCell[d]; if( szRight!=0 - && (bBulk || szRight+b.szCell[d]+2 > szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + && (bBulk || szRight+szD+2 > szLeft-(szR+(i==k-1?0:2)))){ break; } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; + szRight += szD + 2; + szLeft -= szR + 2; cntNew[i-1] = r; r--; d--; @@ -75080,7 +77152,7 @@ static int balance_nonroot( ** that page. */ assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); - TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 @@ -75113,7 +77185,7 @@ static int balance_nonroot( cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; @@ -75128,47 +77200,44 @@ static int balance_nonroot( ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** - ** An O(n^2) insertion sort algorithm is used, but since n is never more - ** than (NB+2) (a small constant), that should not be a problem. + ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 + ** (5), that is not a performance concern. ** ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; i<nNew; i++){ - aPgOrder[i] = aPgno[i] = apNew[i]->pgno; - aPgFlags[i] = apNew[i]->pDbPage->flags; - for(j=0; j<i; j++){ - if( NEVER(aPgno[j]==aPgno[i]) ){ - /* This branch is taken if the set of sibling pages somehow contains - ** duplicate entries. This can happen if the database is corrupt. - ** It would be simpler to detect this as part of the loop below, but - ** we do the detection here in order to avoid populating the pager - ** cache with two separate objects associated with the same - ** page number. */ - assert( CORRUPT_DB ); - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } - } + aPgno[i] = apNew[i]->pgno; + assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE ); + assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY ); } - for(i=0; i<nNew; i++){ - int iBest = 0; /* aPgno[] index of page number to use */ - for(j=1; j<nNew; j++){ - if( aPgOrder[j]<aPgOrder[iBest] ) iBest = j; - } - pgno = aPgOrder[iBest]; - aPgOrder[iBest] = 0xffffffff; - if( iBest!=i ){ - if( iBest>i ){ - sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); - } - sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); - apNew[i]->pgno = pgno; + for(i=0; i<nNew-1; i++){ + int iB = i; + for(j=i+1; j<nNew; j++){ + if( apNew[j]->pgno < apNew[iB]->pgno ) iB = j; } - } - TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " - "%d(%d nc=%d) %d(%d nc=%d)\n", + /* If apNew[i] has a page number that is bigger than any of the + ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent + ** entry that has the smallest page number (which we know to be + ** entry apNew[iB]). + */ + if( iB!=i ){ + Pgno pgnoA = apNew[i]->pgno; + Pgno pgnoB = apNew[iB]->pgno; + Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1; + u16 fgA = apNew[i]->pDbPage->flags; + u16 fgB = apNew[iB]->pDbPage->flags; + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB); + sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA); + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB); + apNew[i]->pgno = pgnoB; + apNew[iB]->pgno = pgnoA; + } + } + + TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) " + "%u(%u nc=%u) %u(%u nc=%u)\n", apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, @@ -75209,7 +77278,7 @@ static int balance_nonroot( ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; @@ -75306,7 +77375,7 @@ static int balance_nonroot( rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); } @@ -75402,7 +77471,7 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ @@ -75413,7 +77482,7 @@ static int balance_nonroot( } assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n", nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. @@ -75423,7 +77492,7 @@ static int balance_nonroot( } #if 0 - if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ + if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may @@ -75485,7 +77554,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } @@ -75498,7 +77567,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ memcpy(pChild->aiOvfl, pRoot->aiOvfl, @@ -75589,6 +77658,11 @@ static int balance(BtCursor *pCur){ }else{ break; } + }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ + rc = SQLITE_CORRUPT_BKPT; }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -75719,9 +77793,13 @@ static int btreeOverwriteContent( /* ** Overwrite the cell that cursor pCur is pointing to with fresh content -** contained in pX. +** contained in pX. In this variant, pCur is pointing to an overflow +** cell. */ -static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ +static SQLITE_NOINLINE int btreeOverwriteOverflowCell( + BtCursor *pCur, /* Cursor pointing to cell to ovewrite */ + const BtreePayload *pX /* Content to write into the cell */ +){ int iOffset; /* Next byte of pX->pData to write */ int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ int rc; /* Return code */ @@ -75730,16 +77808,12 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ Pgno ovflPgno; /* Next overflow page to write */ u32 ovflPageSize; /* Size to write on overflow page */ - if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd - || pCur->info.pPayload < pPage->aData + pPage->cellOffset - ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pCur->info.nLocal<nTotal ); /* pCur is an overflow cell */ + /* Overwrite the local portion first */ rc = btreeOverwriteContent(pPage, pCur->info.pPayload, pX, 0, pCur->info.nLocal); if( rc ) return rc; - if( pCur->info.nLocal==nTotal ) return SQLITE_OK; /* Now overwrite the overflow pages */ iOffset = pCur->info.nLocal; @@ -75769,6 +77843,29 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ return SQLITE_OK; } +/* +** Overwrite the cell that cursor pCur is pointing to with fresh content +** contained in pX. +*/ +static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ + int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ + MemPage *pPage = pCur->pPage; /* Page being written */ + + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ + return SQLITE_CORRUPT_BKPT; + } + if( pCur->info.nLocal==nTotal ){ + /* The entire cell is local */ + return btreeOverwriteContent(pPage, pCur->info.pPayload, pX, + 0, pCur->info.nLocal); + }else{ + /* The cell contains overflow content */ + return btreeOverwriteOverflowCell(pCur, pX); + } +} + /* ** Insert a new record into the BTree. The content of the new record @@ -75812,7 +77909,6 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( int idx; MemPage *pPage; Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; @@ -75831,7 +77927,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** not to clear the cursor here. */ if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + rc = saveAllCursors(p->pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; if( loc && pCur->iPage<0 ){ /* This can only happen if the schema is corrupt such that there is more @@ -75855,8 +77951,8 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( cursorOwnsBtShared(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + && p->pBt->inTransaction==TRANS_WRITE + && (p->pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); /* Assert that the caller has been consistent. If this cursor was opened @@ -75954,7 +78050,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } assert( pCur->eState==CURSOR_VALID - || (pCur->eState==CURSOR_INVALID && loc) ); + || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); @@ -75969,31 +78065,34 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( rc ) return rc; } - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit || CORRUPT_DB ); - newCell = pBt->pTmpSpace; + newCell = p->pBt->pTmpSpace; assert( newCell!=0 ); + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; - szNew = pBt->nPreformatSize; + szNew = p->pBt->nPreformatSize; if( szNew<4 ) szNew = 4; - if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); - ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + ptrmapPut(p->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + if( NEVER(rc) ) goto end_insert; } } }else{ rc = fillInCell(pPage, newCell, pX, &szNew); + if( rc ) goto end_insert; } - if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); - assert( szNew <= MX_CELL_SIZE(pBt) ); + assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; + pCur->info.nSize = 0; if( loc==0 ){ CellInfo info; assert( idx>=0 ); @@ -76012,7 +78111,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM || szNew<pPage->minLocal) + && (!ISAUTOVACUUM(p->pBt) || szNew<pPage->minLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add @@ -76042,7 +78141,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else{ assert( pPage->leaf ); } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); @@ -76066,7 +78165,6 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ - pCur->info.nSize = 0; if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); @@ -76115,7 +78213,6 @@ end_insert: ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ - int rc = SQLITE_OK; BtShared *pBt = pDest->pBt; u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ const u8 *aIn; /* Pointer to next input buffer */ @@ -76138,7 +78235,9 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 if( nIn==nRem && nIn<pDest->pPage->maxLocal ){ memcpy(aOut, aIn, nIn); pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + return SQLITE_OK; }else{ + int rc = SQLITE_OK; Pager *pSrcPager = pSrc->pBt->pPager; u8 *pPgnoOut = 0; Pgno ovflIn = 0; @@ -76190,7 +78289,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); - if( ISAUTOVACUUM && pPageOut ){ + if( ISAUTOVACUUM(pBt) && pPageOut ){ ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); } releasePage(pPageOut); @@ -76206,9 +78305,8 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 releasePage(pPageOut); sqlite3PagerUnref(pPageIn); + return rc; } - - return rc; } /* @@ -76267,6 +78365,9 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ return SQLITE_CORRUPT_BKPT; } + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ + return SQLITE_CORRUPT_BKPT; + } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete @@ -76363,7 +78464,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); if( rc==SQLITE_OK ){ - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; @@ -76963,6 +79064,41 @@ SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ #ifndef SQLITE_OMIT_INTEGRITY_CHECK /* +** Record an OOM error during integrity_check +*/ +static void checkOom(IntegrityCk *pCheck){ + pCheck->rc = SQLITE_NOMEM; + pCheck->mxErr = 0; /* Causes integrity_check processing to stop */ + if( pCheck->nErr==0 ) pCheck->nErr++; +} + +/* +** Invoke the progress handler, if appropriate. Also check for an +** interrupt. +*/ +static void checkProgress(IntegrityCk *pCheck){ + sqlite3 *db = pCheck->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + assert( db->nProgressOps>0 ); + pCheck->nStep++; + if( (pCheck->nStep % db->nProgressOps)==0 + && db->xProgress(db->pProgressArg) + ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } + } +#endif +} + +/* ** Append a message to the error message string. */ static void checkAppendMsg( @@ -76971,6 +79107,7 @@ static void checkAppendMsg( ... ){ va_list ap; + checkProgress(pCheck); if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -76979,12 +79116,13 @@ static void checkAppendMsg( sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, + pCheck->v0, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==SQLITE_NOMEM ){ - pCheck->bOomFault = 1; + checkOom(pCheck); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -77019,14 +79157,13 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage>pCheck->nPage || iPage==0 ){ - checkAppendMsg(pCheck, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %u", iPage); return 1; } - if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; setPageReferenced(pCheck, iPage); return 0; } @@ -77049,14 +79186,14 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->bOomFault = 1; - checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -77081,7 +79218,7 @@ static void checkList( if( checkRef(pCheck, iPage) ) break; N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ - checkAppendMsg(pCheck, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %u", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -77094,7 +79231,7 @@ static void checkList( #endif if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, - "freelist leaf count too big on page %d", iPage); + "freelist leaf count too big on page %u", iPage); N--; }else{ for(i=0; i<(int)n; i++){ @@ -77126,7 +79263,7 @@ static void checkList( } if( N && nErrAtStart==pCheck->nErr ){ checkAppendMsg(pCheck, - "%s is %d but should be %d", + "%s is %u but should be %u", isFreeList ? "size" : "overflow list length", expected-N, expected); } @@ -77156,7 +79293,9 @@ static void checkList( ** lower 16 bits are the index of the last byte of that range. */ static void btreeHeapInsert(u32 *aHeap, u32 x){ - u32 j, i = ++aHeap[0]; + u32 j, i; + assert( aHeap!=0 ); + i = ++aHeap[0]; aHeap[i] = x; while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ x = aHeap[j]; @@ -77233,12 +79372,14 @@ static int checkTreePage( /* Check that the page exists */ + checkProgress(pCheck); + if( pCheck->mxErr==0 ) goto end_of_check; pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; - pCheck->zPfx = "Page %u: "; - pCheck->v1 = iPage; + pCheck->zPfx = "Tree %u page %u: "; + pCheck->v0 = pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); @@ -77264,7 +79405,7 @@ static int checkTreePage( hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %u cell %d: "; + pCheck->zPfx = "Tree %u page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -77284,7 +79425,7 @@ static int checkTreePage( pgno = get4byte(&data[hdr+8]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %u at right child: "; + pCheck->zPfx = "Tree %u page %u right child: "; checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif @@ -77308,7 +79449,7 @@ static int checkTreePage( pc = get2byteAligned(pCellIdx); pCellIdx -= 2; if( pc<contentOffset || pc>usableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + checkAppendMsg(pCheck, "Offset %u out of range %u..%u", pc, contentOffset, usableSize-4); doCoverageCheck = 0; continue; @@ -77440,7 +79581,7 @@ static int checkTreePage( */ if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, - "Fragmentation of %d bytes reported as %d on page %u", + "Fragmentation of %u bytes reported as %u on page %u", nFrag, data[hdr+7], iPage); } } @@ -77478,13 +79619,14 @@ end_of_check: ** the unverified btrees. Except, if aRoot[1] is 1, then the freelist ** checks are still performed. */ -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ - int *pnErr /* Write number of errors seen to this variable */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ ){ Pgno i; IntegrityCk sCheck; @@ -77507,18 +79649,12 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); + memset(&sCheck, 0, sizeof(sCheck)); sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; - sCheck.nErr = 0; - sCheck.bOomFault = 0; - sCheck.zPfx = 0; - sCheck.v1 = 0; - sCheck.v2 = 0; - sCheck.aPgRef = 0; - sCheck.heap = 0; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; if( sCheck.nPage==0 ){ @@ -77527,12 +79663,12 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ - sCheck.bOomFault = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); if( sCheck.heap==0 ){ - sCheck.bOomFault = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } @@ -77542,7 +79678,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ if( bCkFreelist ){ - sCheck.zPfx = "Main freelist: "; + sCheck.zPfx = "Freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36])); sCheck.zPfx = 0; @@ -77559,7 +79695,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( mxInHdr = get4byte(&pBt->pPage1->aData[52]); if( mx!=mxInHdr ){ checkAppendMsg(&sCheck, - "max rootpage (%d) disagrees with header (%d)", + "max rootpage (%u) disagrees with header (%u)", mx, mxInHdr ); } @@ -77590,7 +79726,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -77598,11 +79734,11 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); } #endif } @@ -77613,16 +79749,17 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( integrity_ck_cleanup: sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); - if( sCheck.bOomFault ){ + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ){ sqlite3_str_reset(&sCheck.errMsg); - sCheck.nErr++; + *pzOut = 0; + }else{ + *pzOut = sqlite3StrAccumFinish(&sCheck.errMsg); } - *pnErr = sCheck.nErr; - if( sCheck.nErr==0 ) sqlite3_str_reset(&sCheck.errMsg); /* Make sure this analysis did not leave any unref() pages. */ assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); sqlite3BtreeLeave(p); - return sqlite3StrAccumFinish(&sCheck.errMsg); + return sCheck.rc; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -77887,6 +80024,17 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ */ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** If no transaction is active and the database is not a temp-db, clear +** the in-memory pager cache. +*/ +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->inTransaction==TRANS_NONE ){ + sqlite3PagerClearCache(pBt->pPager); + } +} + #if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** Return true if the Btree passed as the only argument is sharable. @@ -78152,13 +80300,7 @@ static int backupOnePage( assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); assert( zSrcData ); - - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset @@ -78291,7 +80433,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); pgszDest = sqlite3BtreeGetPageSize(p->pDest); destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + if( SQLITE_OK==rc + && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager)) + && pgszSrc!=pgszDest + ){ rc = SQLITE_READONLY; } @@ -78797,9 +80942,9 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ i64 x; assert( (p->flags&MEM_Int)*2==sizeof(x) ); memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); - sqlite3Int64ToText(x, zBuf); + p->n = sqlite3Int64ToText(x, zBuf); #else - sqlite3Int64ToText(p->u.i, zBuf); + p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); @@ -78807,6 +80952,7 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ + p->n = acc.nChar; } } @@ -78834,10 +80980,12 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ ** This routine is for use inside of assert() statements only. */ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ + Mem tmp; char zBuf[100]; char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; if( p->flags & MEM_Term ){ /* Insure that the string is properly zero-terminated. Pay particular ** attention to the case where p->n is odd */ @@ -78850,7 +80998,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; - vdbeMemRenderNum(sizeof(zBuf), zBuf, p); + memcpy(&tmp, p, sizeof(tmp)); + vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; i = j = 0; incr = 1; @@ -79119,7 +81268,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30NN(pMem->z); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); @@ -79359,32 +81508,35 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ } /* -** The MEM structure is already a MEM_Real. Try to also make it a -** MEM_Int if we can. +** The MEM structure is already a MEM_Real or MEM_IntReal. Try to +** make it a MEM_Int if we can. */ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ - i64 ix; assert( pMem!=0 ); - assert( pMem->flags & MEM_Real ); + assert( pMem->flags & (MEM_Real|MEM_IntReal) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - ix = doubleToInt64(pMem->u.r); - - /* Only mark the value as an integer if - ** - ** (1) the round-trip conversion real->int->real is a no-op, and - ** (2) The integer is neither the largest nor the smallest - ** possible integer (ticket #3922) - ** - ** The second and third terms in the following conditional enforces - ** the second condition under the assumption that addition overflow causes - ** values to wrap around. - */ - if( pMem->u.r==ix && ix>SMALLEST_INT64 && ix<LARGEST_INT64 ){ - pMem->u.i = ix; + if( pMem->flags & MEM_IntReal ){ MemSetTypeFlag(pMem, MEM_Int); + }else{ + i64 ix = doubleToInt64(pMem->u.r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ix<LARGEST_INT64 ){ + pMem->u.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + } } } @@ -79432,6 +81584,16 @@ SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ && i >= -2251799813685248LL && i < 2251799813685248LL); } +/* Convert a floating point value to its closest integer. Do so in +** a way that avoids 'outside the range of representable values' warnings +** from UBSAN. +*/ +SQLITE_PRIVATE i64 sqlite3RealToI64(double r){ + if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64; + if( r>=(double)LARGEST_INT64) return LARGEST_INT64; + return (i64)r; +} + /* ** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. @@ -79453,7 +81615,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) - || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); @@ -79505,6 +81667,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; return sqlite3VdbeChangeEncoding(pMem, encoding); } } @@ -80149,6 +82312,9 @@ static int valueFromFunction( if( pList ) nVal = pList->nExpr; assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) @@ -80174,8 +82340,6 @@ static int valueFromFunction( goto value_from_function_out; } - testcase( pCtx->pParse->rc==SQLITE_ERROR ); - testcase( pCtx->pParse->rc==SQLITE_OK ); memset(&ctx, 0, sizeof(ctx)); ctx.pOut = pVal; ctx.pFunc = pFunc; @@ -80188,16 +82352,16 @@ static int valueFromFunction( sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } } - pCtx->pParse->rc = rc; value_from_function_out: if( rc!=SQLITE_OK ){ pVal = 0; + pCtx->pParse->rc = rc; } if( apVal ){ for(i=0; i<nVal; i++){ @@ -80255,6 +82419,13 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ +#ifdef SQLITE_ENABLE_STAT4 + rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif sqlite3VdbeMemCast(*ppVal, aff, enc); sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } @@ -80640,6 +82811,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ return p->n; } + if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){ + return p->n; + } if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; @@ -80685,10 +82859,10 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ - db->pVdbe->pPrev = p; + db->pVdbe->ppVPrev = &p->pVNext; } - p->pNext = db->pVdbe; - p->pPrev = 0; + p->pVNext = db->pVdbe; + p->ppVPrev = &db->pVdbe; db->pVdbe = p; assert( p->eVdbeState==VDBE_INIT_STATE ); p->pParse = pParse; @@ -80770,21 +82944,28 @@ SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( #endif /* -** Swap all content between two VDBE structures. +** Swap byte-code between two VDBE structures. +** +** This happens after pB was previously run and returned +** SQLITE_SCHEMA. The statement was then reprepared in pA. +** This routine transfers the new bytecode in pA over to pB +** so that pB can be run again. The old pB byte code is +** moved back to pA so that it will be cleaned up when pA is +** finalized. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ - Vdbe tmp, *pTmp; + Vdbe tmp, *pTmp, **ppTmp; char *zTmp; assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; - pTmp = pA->pNext; - pA->pNext = pB->pNext; - pB->pNext = pTmp; - pTmp = pA->pPrev; - pA->pPrev = pB->pPrev; - pB->pPrev = pTmp; + pTmp = pA->pVNext; + pA->pVNext = pB->pVNext; + pB->pVNext = pTmp; + ppTmp = pA->ppVPrev; + pA->ppVPrev = pB->ppVPrev; + pB->ppVPrev = ppTmp; zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; @@ -80860,6 +83041,8 @@ static int growOpArray(Vdbe *v, int nOp){ */ static void test_addop_breakpoint(int pc, Op *pOp){ static int n = 0; + (void)pc; + (void)pOp; n++; } #endif @@ -80910,16 +83093,16 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i, &p->aOp[i]); test_addop_breakpoint(i, &p->aOp[i]); } #endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif #ifdef SQLITE_VDBE_COVERAGE pOp->iSrcLine = 0; #endif @@ -81036,6 +83219,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, p1, p2, p3, (char*)pCtx, P4_FUNCCTX); sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + sqlite3MayAbort(pParse); return addr; } @@ -81086,11 +83270,12 @@ SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ ** If the bPush flag is true, then make this opcode the parent for ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ -SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ -#ifndef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ + int addr = 0; +#if !defined(SQLITE_DEBUG) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ - if( pParse->explain==2 ) + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { char *zMsg; @@ -81102,13 +83287,15 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt va_end(ap); v = pParse->pVdbe; iThis = v->nOp; - sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + addr = sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } + sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0); } + return addr; } /* @@ -81216,6 +83403,9 @@ static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ int i; for(i=p->nLabelAlloc; i<nNewSize; i++) p->aLabel[i] = -1; #endif + if( nNewSize>=100 && (nNewSize/100)>(p->nLabelAlloc/100) ){ + sqlite3ProgressCheck(p); + } p->nLabelAlloc = nNewSize; p->aLabel[j] = v->nOp; } @@ -81371,6 +83561,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || opcode==OP_VDestroy || opcode==OP_VCreate || opcode==OP_ParseSchema + || opcode==OP_Function || opcode==OP_PureFunc || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ @@ -81458,11 +83649,13 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; - while(1){ - + assert( p->aOp[0].opcode==OP_Init ); + while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -81491,6 +83684,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 1; break; } + case OP_Init: { + assert( pOp->p2>=0 ); + goto resolve_p2_values_loop_exit; + } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; @@ -81513,6 +83710,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } break; @@ -81523,11 +83721,12 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } - if( pOp==p->aOp ) break; + assert( pOp>p->aOp ); pOp--; } +resolve_p2_values_loop_exit: if( aLabel ){ - sqlite3DbFreeNN(p->db, pParse->aLabel); + sqlite3DbNNFreeNN(p->db, pParse->aLabel); pParse->aLabel = 0; } pParse->nLabel = 0; @@ -81755,20 +83954,83 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } + } +} + +/* +** Add the range of instructions from addrStart to addrEnd (inclusive) to +** the set of those corresponding to the sqlite3_stmt_scanstatus() counters +** associated with the OP_Explain instruction at addrExplain. The +** sum of the sqlite3Hwtime() values for each of these instructions +** will be returned for SQLITE_SCANSTAT_NCYCLE requests. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange( + Vdbe *p, + int addrExplain, + int addrStart, + int addrEnd +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; ii<ArraySize(pScan->aAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } + } + } } } -#endif + +/* +** Set the addresses for the SQLITE_SCANSTAT_NLOOP and SQLITE_SCANSTAT_NROW +** counters for the query element associated with the OP_Explain at +** addrExplain. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters( + Vdbe *p, + int addrExplain, + int addrLoop, + int addrVisit +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } + } +} +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* @@ -81776,15 +84038,19 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( ** for a specific instruction. */ SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p1 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( addr>=0 || p->db->mallocFailed ); sqlite3VdbeGetOp(p,addr)->p2 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p3 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -81793,6 +84059,18 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ } /* +** If the previous opcode is an OP_Column that delivers results +** into register iDest, then add the OPFLAG_TYPEOFARG flag to that +** opcode. +*/ +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ + VdbeOp *pOp = sqlite3VdbeGetLastOp(p); + if( pOp->p3==iDest && pOp->opcode==OP_Column ){ + pOp->p5 |= OPFLAG_TYPEOFARG; + } +} + +/* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. */ @@ -81820,7 +84098,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ || p->aOp[addr].opcode==OP_FkIfZero ); assert( p->aOp[addr].p4type==0 ); #ifdef SQLITE_VDBE_COVERAGE - sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ + sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ #endif p->nOp--; }else{ @@ -81834,8 +84112,9 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ ** the FuncDef is not ephermal, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + assert( db!=0 ); if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFreeNN(db, pDef); + sqlite3DbNNFreeNN(db, pDef); } } @@ -81844,11 +84123,12 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ */ static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + assert( db!=0 ); freeEphemeralFunction(db, p->pFunc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); @@ -81861,7 +84141,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ case P4_INT64: case P4_DYNAMIC: case P4_INTARRAY: { - sqlite3DbFree(db, p4); + if( p4 ) sqlite3DbNNFreeNN(db, p4); break; } case P4_KEYINFO: { @@ -81900,6 +84180,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ assert( nOp>=0 ); + assert( db!=0 ); if( aOp ){ Op *pOp = &aOp[nOp-1]; while(1){ /* Exit via break */ @@ -81910,7 +84191,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( pOp==aOp ) break; pOp--; } - sqlite3DbFreeNN(db, aOp); + sqlite3DbNNFreeNN(db, aOp); } } @@ -82079,7 +84360,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){ if( p->db->mallocFailed ){ freeP4(p->db, n, pP4); }else{ - assert( pP4!=0 ); + assert( pP4!=0 || n==P4_DYNAMIC ); assert( p->nOp>0 ); pOp = &p->aOp[p->nOp-1]; assert( pOp->p4type==P4_NOTUSED ); @@ -82141,13 +84422,13 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** Set the value if the iSrcLine field for the previously coded instruction. */ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ - sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; + sqlite3VdbeGetLastOp(v)->iSrcLine = iLine; } #endif /* SQLITE_VDBE_COVERAGE */ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** Return the opcode for a given address. The address must be non-negative. +** See sqlite3VdbeGetLastOp() to get the most recently added opcode. ** ** If a memory allocation error has occurred prior to the calling of this ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode @@ -82163,9 +84444,6 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ ** zeros, which is correct. MSVC generates a warning, nevertheless. */ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ assert( p->eVdbeState==VDBE_INIT_STATE ); - if( addr<0 ){ - addr = p->nOp - 1; - } assert( (addr>=0 && addr<p->nOp) || p->db->mallocFailed ); if( p->db->mallocFailed ){ return (VdbeOp*)&dummy; @@ -82174,6 +84452,12 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ } } +/* Return the most recently added opcode +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ + return sqlite3VdbeGetOp(p, p->nOp - 1); +} + #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) /* ** Return an integer value for one of the parameters to the opcode pOp @@ -82661,7 +84945,7 @@ static void releaseMemArray(Mem *p, int N){ sqlite3VdbeMemRelease(p); p->flags = MEM_Undefined; }else if( p->szMalloc ){ - sqlite3DbFreeNN(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p->zMalloc); p->szMalloc = 0; p->flags = MEM_Undefined; } @@ -82875,7 +85159,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); - p->pResultSet = 0; if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or @@ -82932,7 +85215,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); p->nResColumn = 8; } - p->pResultSet = pMem; + p->pResultRow = pMem; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; rc = SQLITE_ERROR; @@ -83043,7 +85326,7 @@ static void *allocSpace( ** running it. */ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#if defined(SQLITE_DEBUG) int i; #endif assert( p!=0 ); @@ -83072,8 +85355,8 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->nFkConstraint = 0; #ifdef VDBE_PROFILE for(i=0; i<p->nOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; } #endif } @@ -83182,9 +85465,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); -#endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; @@ -83193,9 +85473,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); -#endif } } @@ -83210,9 +85487,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - memset(p->anExec, 0, p->nOp*sizeof(i64)); -#endif } sqlite3VdbeRewind(p); } @@ -83270,9 +85544,6 @@ static void closeCursorsInFrame(Vdbe *p){ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - v->anExec = pFrame->anExec; -#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -83653,7 +85924,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; } - p = p->pNext; + p = p->pVNext; } assert( cnt==db->nVdbeActive ); assert( nWrite==db->nVdbeWrite ); @@ -83887,6 +86158,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -84076,7 +86349,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - p->pResultSet = 0; + p->pResultRow = 0; #ifdef SQLITE_DEBUG p->nWrite = 0; #endif @@ -84104,10 +86377,12 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } for(i=0; i<p->nOp; i++){ char zHdr[100]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + cnt, + cycles, + cnt>0 ? cycles/cnt : 0 ); fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); @@ -84182,10 +86457,11 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, */ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + assert( db!=0 ); assert( p->db==0 || p->db==db ); if( p->aColName ){ releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - sqlite3DbFreeNN(db, p->aColName); + sqlite3DbNNFreeNN(db, p->aColName); } for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; @@ -84194,17 +86470,17 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ } if( p->eVdbeState!=VDBE_INIT_STATE ){ releaseMemArray(p->aVar, p->nVar); - if( p->pVList ) sqlite3DbFreeNN(db, p->pVList); - if( p->pFree ) sqlite3DbFreeNN(db, p->pFree); + if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList); + if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree); } vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->zSql); + if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { - DblquoteStr *pThis, *pNext; - for(pThis=p->pDblStr; pThis; pThis=pNext){ - pNext = pThis->pNextStr; + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; sqlite3DbFree(db, pThis); } } @@ -84228,20 +86504,17 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ assert( p!=0 ); db = p->db; + assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); if( db->pnBytesFreed==0 ){ - if( p->pPrev ){ - p->pPrev->pNext = p->pNext; - }else{ - assert( db->pVdbe==p ); - db->pVdbe = p->pNext; - } - if( p->pNext ){ - p->pNext->pPrev = p->pPrev; + assert( p->ppVPrev!=0 ); + *p->ppVPrev = p->pVNext; + if( p->pVNext ){ + p->pVNext->ppVPrev = p->ppVPrev; } } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -85196,7 +87469,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ + while( 1 /*exit-by-break*/ ){ u32 serial_type; /* RHS is an integer */ @@ -85206,7 +87479,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ @@ -85231,7 +87504,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing ** them to numberic values are. */ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else{ @@ -85312,7 +87585,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0); + rc = (serial_type!=0 && serial_type!=10); } if( rc!=0 ){ @@ -85334,8 +87607,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); + if( d1>(unsigned)nKey1 ) break; idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); + if( idx1>=(unsigned)szHdr1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corrupt index */ + } + } /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -85736,7 +88014,7 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; - for(p = db->pVdbe; p; p=p->pNext){ + for(p = db->pVdbe; p; p=p->pVNext){ p->expired = iCode+1; } } @@ -85829,6 +88107,20 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ return 1; } +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored @@ -85857,13 +88149,14 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** the vdbeUnpackRecord() function found in vdbeapi.c. */ static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + assert( db!=0 ); if( p ){ int i; for(i=0; i<nField; i++){ Mem *pMem = &p->aMem[i]; if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem); } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -85890,6 +88183,16 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); @@ -85906,8 +88209,8 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( assert( pCsr!=0 ); assert( pCsr->eCurType==CURTYPE_BTREE ); - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; @@ -85934,7 +88237,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( for(i=0; i<pCsr->nField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } - sqlite3DbFreeNN(db, preupdate.aNew); + sqlite3DbNNFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -85958,6 +88261,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( */ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* #include "opcodes.h" */ #ifndef SQLITE_OMIT_DEPRECATED /* @@ -86051,7 +88355,9 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); - rc = sqlite3VdbeFinalize(v); + assert( v->eVdbeState>=VDBE_READY_STATE ); + rc = sqlite3VdbeReset(v); + sqlite3VdbeDelete(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); } @@ -86211,7 +88517,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_NULL, /* 0x1f (not possible) */ SQLITE_FLOAT, /* 0x20 INTREAL */ SQLITE_NULL, /* 0x21 (not possible) */ - SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ SQLITE_NULL, /* 0x23 (not possible) */ SQLITE_FLOAT, /* 0x24 (not possible) */ SQLITE_NULL, /* 0x25 (not possible) */ @@ -86259,6 +88565,9 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ #endif return aType[pVal->flags&MEM_AffMask]; } +SQLITE_API int sqlite3_value_encoding(sqlite3_value *pVal){ + return pVal->enc; +} /* Return true if a parameter to xUpdate represents an unchanged column */ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ @@ -86443,7 +88752,10 @@ SQLITE_API void sqlite3_result_text64( ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + n &= ~(u64)1; + } if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -86458,7 +88770,7 @@ SQLITE_API void sqlite3_result_text16( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel); } SQLITE_API void sqlite3_result_text16be( sqlite3_context *pCtx, @@ -86467,7 +88779,7 @@ SQLITE_API void sqlite3_result_text16be( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel); } SQLITE_API void sqlite3_result_text16le( sqlite3_context *pCtx, @@ -86476,7 +88788,7 @@ SQLITE_API void sqlite3_result_text16le( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ @@ -86687,7 +88999,7 @@ static int sqlite3Step(Vdbe *p){ /* If the statement completed successfully, invoke the profile callback */ checkProfileCallback(db, p); #endif - + p->pResultRow = 0; if( rc==SQLITE_DONE && db->autoCommit ){ assert( p->rc==SQLITE_OK ); p->rc = doWalCallbacks(db); @@ -86817,6 +89129,17 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ } /* +** The destructor function for a ValueList object. This needs to be +** a separate function, unknowable to the application, to ensure that +** calls to sqlite3_vtab_in_first()/sqlite3_vtab_in_next() that are not +** preceeded by activation of IN processing via sqlite3_vtab_int() do not +** try to access a fake ValueList object inserted by a hostile extension. +*/ +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void *pToDelete){ + sqlite3_free(pToDelete); +} + +/* ** Implementation of sqlite3_vtab_in_first() (if bNext==0) and ** sqlite3_vtab_in_next() (if bNext!=0). */ @@ -86830,8 +89153,15 @@ static int valueFromValueList( *ppOut = 0; if( pVal==0 ) return SQLITE_MISUSE; - pRhs = (ValueList*)sqlite3_value_pointer(pVal, "ValueList"); - if( pRhs==0 ) return SQLITE_MISUSE; + if( (pVal->flags & MEM_Dyn)==0 || pVal->xDel!=sqlite3VdbeValueListFree ){ + return SQLITE_ERROR; + }else{ + assert( (pVal->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) ); + assert( pVal->eSubtype=='p' ); + assert( pVal->u.zPType!=0 && strcmp(pVal->u.zPType,"ValueList")==0 ); + pRhs = (ValueList*)pVal->z; + } if( bNext ){ rc = sqlite3BtreeNext(pRhs->pCsr, 0); }else{ @@ -87051,7 +89381,7 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - if( pVm==0 || pVm->pResultSet==0 ) return 0; + if( pVm==0 || pVm->pResultRow==0 ) return 0; return pVm->nResColumn; } @@ -87106,8 +89436,8 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ if( pVm==0 ) return (Mem*)columnNullValue(); assert( pVm->db ); sqlite3_mutex_enter(pVm->db->mutex); - if( pVm->pResultSet!=0 && i<pVm->nResColumn && i>=0 ){ - pOut = &pVm->pResultSet[i]; + if( pVm->pResultRow!=0 && i<pVm->nResColumn && i>=0 ){ + pOut = &pVm->pResultRow[i]; }else{ sqlite3Error(pVm->db, SQLITE_RANGE); pOut = (Mem*)columnNullValue(); @@ -87253,9 +89583,9 @@ static const void *columnName( assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N<n && N>=0 ){ + u8 prior_mallocFailed = db->mallocFailed; N += useType*n; sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -87267,7 +89597,8 @@ static const void *columnName( /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } @@ -87373,7 +89704,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ ** The error code stored in database p->db is overwritten with the return ** value in any case. */ -static int vdbeUnbind(Vdbe *p, int i){ +static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; if( vdbeSafetyNotNull(p) ){ return SQLITE_MISUSE_BKPT; @@ -87386,12 +89717,11 @@ static int vdbeUnbind(Vdbe *p, int i){ "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i<1 || i>p->nVar ){ + if( i>=(unsigned int)p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } - i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; @@ -87428,7 +89758,7 @@ static int bindText( Mem *pVar; int rc; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; @@ -87477,7 +89807,7 @@ SQLITE_API int sqlite3_bind_blob64( SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); @@ -87490,7 +89820,7 @@ SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); @@ -87500,7 +89830,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } @@ -87515,7 +89845,7 @@ SQLITE_API int sqlite3_bind_pointer( ){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); @@ -87542,7 +89872,10 @@ SQLITE_API int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + nData &= ~(u16)1; + } return bindText(pStmt, i, zData, nData, xDel, enc); } #ifndef SQLITE_OMIT_UTF16 @@ -87550,10 +89883,10 @@ SQLITE_API int sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, - int nData, + int n, void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); + return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ @@ -87593,7 +89926,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); @@ -87753,7 +90086,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; }else{ - pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pVNext; } sqlite3_mutex_leave(pDb->mutex); return pNext; @@ -87778,8 +90111,11 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ sqlite3_mutex_enter(db->mutex); v = 0; db->pnBytesFreed = (int*)&v; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; sqlite3VdbeDelete(pVdbe); db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3_mutex_leave(db->mutex); }else{ v = pVdbe->aCounter[op]; @@ -88041,23 +90377,69 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa /* ** Return status data for a single loop within query pStmt. */ -SQLITE_API int sqlite3_stmt_scanstatus( +SQLITE_API int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement being queried */ - int idx, /* Index of loop to report on */ + int iScan, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ + int flags, void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanStatus *pScan; - if( idx<0 || idx>=p->nScan ) return 1; - pScan = &p->aScan[idx]; + VdbeOp *aOp = p->aOp; + int nOp = p->nOp; + ScanStatus *pScan = 0; + int idx; + + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + + if( iScan<0 ){ + int ii; + if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ + i64 res = 0; + for(ii=0; ii<nOp; ii++){ + res += aOp[ii].nCycle; + } + *(i64*)pOut = res; + return 0; + } + return 1; + } + if( flags & SQLITE_SCANSTAT_COMPLEX ){ + idx = iScan; + pScan = &p->aScan[idx]; + }else{ + /* If the COMPLEX flag is clear, then this function must ignore any + ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ + for(idx=0; idx<p->nScan; idx++){ + pScan = &p->aScan[idx]; + if( pScan->zName ){ + iScan--; + if( iScan<0 ) break; + } + } + } + if( idx>=p->nScan ) return 1; + switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + if( pScan->addrLoop>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_NVISIT: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + if( pScan->addrVisit>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_EST: { @@ -88076,7 +90458,7 @@ SQLITE_API int sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -88084,12 +90466,51 @@ SQLITE_API int sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + case SQLITE_SCANSTAT_PARENTID: { + if( pScan->addrExplain ){ + *(int*)pOut = aOp[ pScan->addrExplain ].p2; }else{ *(int*)pOut = -1; } break; } + case SQLITE_SCANSTAT_NCYCLE: { + i64 res = 0; + if( pScan->aAddrRange[0]==0 ){ + res = -1; + }else{ + int ii; + for(ii=0; ii<ArraySize(pScan->aAddrRange); ii+=2){ + int iIns = pScan->aAddrRange[ii]; + int iEnd = pScan->aAddrRange[ii+1]; + if( iIns==0 ) break; + if( iIns>0 ){ + while( iIns<=iEnd ){ + res += aOp[iIns].nCycle; + iIns++; + } + }else{ + int iOp; + for(iOp=0; iOp<nOp; iOp++){ + Op *pOp = &aOp[iOp]; + if( pOp->p1!=iEnd ) continue; + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ + continue; + } + res += aOp[iOp].nCycle; + } + } + } + } + *(i64*)pOut = res; + break; + } default: { return 1; } @@ -88098,11 +90519,28 @@ SQLITE_API int sqlite3_stmt_scanstatus( } /* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int iScan, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); +} + +/* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); + int ii; + for(ii=0; ii<p->nOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ @@ -88438,6 +90876,9 @@ SQLITE_API int sqlite3_found_count = 0; */ static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ static int n = 0; + (void)pc; + (void)pOp; + (void)v; n++; } #endif @@ -88619,7 +91060,8 @@ static VdbeCursor *allocateCursor( ** return false. */ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ - i64 iValue = (double)rValue; + i64 iValue; + iValue = sqlite3RealToI64(rValue); if( sqlite3RealSameAsInt(rValue,iValue) ){ *piValue = iValue; return 1; @@ -88675,6 +91117,10 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** always preferred, even if the affinity is REAL, because ** an integer representation is more space efficient on disk. ** +** SQLITE_AFF_FLEXNUM: +** If the value is text, then try to convert it into a number of +** some kind (integer or real) but do not make any other changes. +** ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** @@ -88689,11 +91135,11 @@ static void applyAffinity( ){ if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); + || affinity==SQLITE_AFF_NUMERIC || affinity==SQLITE_AFF_FLEXNUM ); if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags & MEM_Real)==0 ){ + if( (pRec->flags & (MEM_Real|MEM_IntReal))==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); - }else{ + }else if( affinity<=SQLITE_AFF_REAL ){ sqlite3VdbeIntegerAffinity(pRec); } } @@ -88781,17 +91227,18 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + assert( (pMem->flags & MEM_Null)==0 + || pMem->db==0 || pMem->db->mallocFailed ); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){ testcase( pMem->flags & MEM_Int ); testcase( pMem->flags & MEM_Real ); testcase( pMem->flags & MEM_IntReal ); - return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); - } - if( pMem->flags & (MEM_Str|MEM_Blob) ){ - testcase( pMem->flags & MEM_Str ); - testcase( pMem->flags & MEM_Blob ); - return computeNumericType(pMem); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null); } + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); return 0; } @@ -88920,17 +91367,6 @@ SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ # define REGISTER_TRACE(R,M) #endif - -#ifdef VDBE_PROFILE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/* #include "hwtime.h" */ - -#endif - #ifndef NDEBUG /* ** This function is only called from within an assert() expression. It @@ -88990,8 +91426,10 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ }else if( p->flags & MEM_Real ){ h += sqlite3VdbeIntValue(p); }else if( p->flags & (MEM_Str|MEM_Blob) ){ - h += p->n; - if( p->flags & MEM_Zero ) h += p->u.nZero; + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); } } return h; @@ -89020,11 +91458,10 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ){ Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp = aOp; /* Current operation */ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) - Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif #ifdef SQLITE_DEBUG + Op *pOrigOp; /* Value of pOp at the top of the loop */ int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ + u8 iCompareIsInit = 0; /* iCompare is initialized */ #endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ @@ -89040,13 +91477,16 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ assert( p->eVdbeState==VDBE_RUN_STATE ); /* sqlite3_step() verifies this */ - sqlite3VdbeEnter(p); + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeEnter(p); + } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; @@ -89067,7 +91507,6 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); - p->pResultSet = 0; db->busyHandler.nBusy = 0; if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); @@ -89104,12 +91543,18 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( rc==SQLITE_OK ); assert( pOp>=aOp && pOp<&aOp[p->nOp]); -#ifdef VDBE_PROFILE - start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); -#endif nVmStep++; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; + +#if defined(VDBE_PROFILE) + pOp->nExec++; + pnCycle = &pOp->nCycle; + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; + *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -89171,7 +91616,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } } #endif -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG pOrigOp = pOp; #endif @@ -89455,6 +91900,12 @@ case OP_Halt: { #ifdef SQLITE_DEBUG if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } #endif + + /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates + ** something is wrong with the code generator. Raise an assertion in order + ** to bring this to the attention of fuzzers and other testing tools. */ + assert( pOp->p1!=SQLITE_INTERNAL ); + if( p->pFrame && pOp->p1==SQLITE_OK ){ /* Halt the sub-program. Return control to the parent frame. */ pFrame = p->pFrame; @@ -89896,10 +92347,10 @@ case OP_ResultRow: { assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); p->cacheCtr = (p->cacheCtr + 2)|1; - p->pResultSet = &aMem[pOp->p1]; + p->pResultRow = &aMem[pOp->p1]; #ifdef SQLITE_DEBUG { - Mem *pMem = p->pResultSet; + Mem *pMem = p->pResultRow; int i; for(i=0; i<pOp->p2; i++){ assert( memIsValid(&pMem[i]) ); @@ -90036,7 +92487,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ @@ -90045,12 +92495,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - type1 = numericType(pIn1); + type1 = pIn1->flags; pIn2 = &aMem[pOp->p2]; - type2 = numericType(pIn2); + type2 = pIn2->flags; pOut = &aMem[pOp->p3]; - flags = pIn1->flags | pIn2->flags; if( (type1 & type2 & MEM_Int)!=0 ){ +int_math: iA = pIn1->u.i; iB = pIn2->u.i; switch( pOp->opcode ){ @@ -90072,9 +92522,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); - }else if( (flags & MEM_Null)!=0 ){ + }else if( ((type1 | type2) & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ + type1 = numericType(pIn1); + type2 = numericType(pIn2); + if( (type1 & type2 & MEM_Int)!=0 ) goto int_math; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -90427,7 +92880,6 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = pIn1->flags; flags3 = pIn3->flags; if( (flags1 & flags3 & MEM_Int)!=0 ){ - assert( (pOp->p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_TEXT || CORRUPT_DB ); /* Common case of comparison of two integers */ if( pIn3->u.i > pIn1->u.i ){ if( sqlite3aGTb[pOp->opcode] ){ @@ -90435,18 +92887,21 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ goto jump_to_p2; } iCompare = +1; + VVA_ONLY( iCompareIsInit = 1; ) }else if( pIn3->u.i < pIn1->u.i ){ if( sqlite3aLTb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } iCompare = -1; + VVA_ONLY( iCompareIsInit = 1; ) }else{ if( sqlite3aEQb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } iCompare = 0; + VVA_ONLY( iCompareIsInit = 1; ) } VdbeBranchTaken(0, (pOp->p5 & SQLITE_NULLEQ)?2:3); break; @@ -90478,6 +92933,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ goto jump_to_p2; } iCompare = 1; /* Operands are not equal */ + VVA_ONLY( iCompareIsInit = 1; ) break; } }else{ @@ -90488,14 +92944,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( (flags1 | flags3)&MEM_Str ){ if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); - testcase( flags3==pIn3->flags ); + assert( flags3==pIn3->flags || CORRUPT_DB ); flags3 = pIn3->flags; } if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } - }else if( affinity==SQLITE_AFF_TEXT ){ + }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); @@ -90503,7 +92959,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); - if( pIn1==pIn3 ) flags3 = flags1 | MEM_Str; + if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); @@ -90534,6 +92990,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ res2 = sqlite3aGTb[pOp->opcode]; } iCompare = res; + VVA_ONLY( iCompareIsInit = 1; ) /* Undo any changes made by applyAffinity() to the input registers. */ assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); @@ -90572,6 +93029,7 @@ case OP_ElseEq: { /* same as TK_ESCAPE, jump */ break; } #endif /* SQLITE_DEBUG */ + assert( iCompareIsInit ); VdbeBranchTaken(iCompare==0, 2); if( iCompare==0 ) goto jump_to_p2; break; @@ -90666,6 +93124,7 @@ case OP_Compare: { pColl = pKeyInfo->aColl[i]; bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + VVA_ONLY( iCompareIsInit = 1; ) if( iCompare ){ if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) @@ -90683,13 +93142,14 @@ case OP_Compare: { /* Opcode: Jump P1 P2 P3 * * ** ** Jump to the instruction at address P1, P2, or P3 depending on whether -** in the most recent OP_Compare instruction the P1 vector was less than +** in the most recent OP_Compare instruction the P1 vector was less than, ** equal to, or greater than the P2 vector, respectively. ** ** This opcode must immediately follow an OP_Compare opcode. */ case OP_Jump: { /* jump */ assert( pOp>aOp && pOp[-1].opcode==OP_Compare ); + assert( iCompareIsInit ); if( iCompare<0 ){ VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ @@ -90889,19 +93349,96 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ break; } -/* Opcode: IsNullOrType P1 P2 P3 * * -** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2 +/* Opcode: IsType P1 P2 P3 P4 P5 +** Synopsis: if typeof(P1.P3) in P5 goto P2 +** +** Jump to P2 if the type of a column in a btree is one of the types specified +** by the P5 bitmask. +** +** P1 is normally a cursor on a btree for which the row decode cache is +** valid through at least column P3. In other words, there should have been +** a prior OP_Column for column P3 or greater. If the cursor is not valid, +** then this opcode might give spurious results. +** The the btree row has fewer than P3 columns, then use P4 as the +** datatype. +** +** If P1 is -1, then P3 is a register number and the datatype is taken +** from the value in that register. +** +** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant +** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. +** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. +** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** +** Take the jump to address P2 if and only if the datatype of the +** value determined by P1 and P3 corresponds to one of the bits in the +** P5 bitmask. ** -** Jump to P2 if the value in register P1 is NULL or has a datatype P3. -** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT, -** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT. */ -case OP_IsNullOrType: { /* jump, in1 */ - int doTheJump; - pIn1 = &aMem[pOp->p1]; - doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3; - VdbeBranchTaken( doTheJump, 2); - if( doTheJump ) goto jump_to_p2; +case OP_IsType: { /* jump */ + VdbeCursor *pC; + u16 typeMask; + u32 serialType; + + assert( pOp->p1>=(-1) && pOp->p1<p->nCursor ); + assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) ); + if( pOp->p1>=0 ){ + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=0 ); + if( pOp->p3<pC->nHdrParsed ){ + serialType = pC->aType[pOp->p3]; + if( serialType>=12 ){ + if( serialType&1 ){ + typeMask = 0x04; /* SQLITE_TEXT */ + }else{ + typeMask = 0x08; /* SQLITE_BLOB */ + } + }else{ + static const unsigned char aMask[] = { + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2, + 0x01, 0x01, 0x10, 0x10 + }; + testcase( serialType==0 ); + testcase( serialType==1 ); + testcase( serialType==2 ); + testcase( serialType==3 ); + testcase( serialType==4 ); + testcase( serialType==5 ); + testcase( serialType==6 ); + testcase( serialType==7 ); + testcase( serialType==8 ); + testcase( serialType==9 ); + testcase( serialType==10 ); + testcase( serialType==11 ); + typeMask = aMask[serialType]; + } + }else{ + typeMask = 1 << (pOp->p4.i - 1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + }else{ + assert( memIsValid(&aMem[pOp->p3]) ); + typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2); + if( typeMask & pOp->p5 ){ + goto jump_to_p2; + } break; } @@ -90944,11 +93481,14 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** If it is, then set register P3 to NULL and jump immediately to P2. ** If P1 is not on a NULL row, then fall through without making any ** changes. +** +** If P1 is not an open cursor, then this opcode is a no-op. */ case OP_IfNullRow: { /* jump */ + VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - assert( p->apCsr[pOp->p1]!=0 ); - if( p->apCsr[pOp->p1]->nullRow ){ + pC = p->apCsr[pOp->p1]; + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } @@ -90999,7 +93539,7 @@ case OP_Offset: { /* out3 */ ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional ** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) +** from this record. If there are less than (P2+1) ** values in the record, extract a NULL. ** ** The value extracted is stored in register P3. @@ -91008,12 +93548,14 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then -** the result is guaranteed to only be used as the argument of a length() -** or typeof() function, respectively. The loading of large blobs can be -** skipped for length() and all content loading can be skipped for typeof(). +** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed +** to only be used by the length() function or the equivalent. The content +** of large blobs is not loaded, thus saving CPU cycles. If the +** OPFLAG_TYPEOFARG bit is set then the result will only be used by the +** typeof() function or the IS NULL or IS NOT NULL operators or the +** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { +case OP_Column: { /* ncycle */ u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ @@ -91362,7 +93904,7 @@ case OP_TypeCheck: { } case COLTYPE_REAL: { testcase( (pIn1->flags & (MEM_Real|MEM_IntReal))==MEM_Real ); - testcase( (pIn1->flags & (MEM_Real|MEM_IntReal))==MEM_IntReal ); + assert( (pIn1->flags & MEM_IntReal)==0 ); if( pIn1->flags & MEM_Int ){ /* When applying REAL affinity, if the result is still an MEM_Int ** that will fit in 6 bytes, then change the type to MEM_IntReal @@ -91441,7 +93983,7 @@ case OP_Affinity: { }else{ pIn1->u.r = (double)pIn1->u.i; pIn1->flags |= MEM_Real; - pIn1->flags &= ~MEM_Int; + pIn1->flags &= ~(MEM_Int|MEM_Str); } } REGISTER_TRACE((int)(pIn1-aMem), pIn1); @@ -92365,7 +94907,7 @@ case OP_SetCookie: { ** ** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; u32 p2; @@ -92386,7 +94928,7 @@ case OP_ReopenIdx: { } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); @@ -92480,7 +95022,7 @@ open_cursor_set_hints: ** ** Duplicate ephemeral cursors are used for self-joins of materialized views. */ -case OP_OpenDup: { +case OP_OpenDup: { /* ncycle */ VdbeCursor *pOrig; /* The original cursor to be duplicated */ VdbeCursor *pCx; /* The new cursor */ @@ -92542,8 +95084,8 @@ case OP_OpenDup: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; @@ -92701,7 +95243,7 @@ case OP_OpenPseudo: { ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; @@ -92818,10 +95360,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group */ -case OP_SeekLE: /* jump, in3, group */ -case OP_SeekGE: /* jump, in3, group */ -case OP_SeekGT: { /* jump, in3, group */ +case OP_SeekLT: /* jump, in3, group, ncycle */ +case OP_SeekLE: /* jump, in3, group, ncycle */ +case OP_SeekGE: /* jump, in3, group, ncycle */ +case OP_SeekGT: { /* jump, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -92950,7 +95492,13 @@ case OP_SeekGT: { /* jump, in3, group */ r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; i<r.nField; i++) assert( memIsValid(&r.aMem[i]) ); } + { + int i; + for(i=0; i<r.nField; i++){ + assert( memIsValid(&r.aMem[i]) ); + if( i>0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]); + } + } #endif r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); @@ -93013,7 +95561,7 @@ seek_not_found: } -/* Opcode: SeekScan P1 P2 * * * +/* Opcode: SeekScan P1 P2 * * P5 ** Synopsis: Scan-ahead up to P1 rows ** ** This opcode is a prefix opcode to OP_SeekGE. In other words, this @@ -93023,8 +95571,8 @@ seek_not_found: ** This opcode uses the P1 through P4 operands of the subsequent ** OP_SeekGE. In the text that follows, the operands of the subsequent ** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only -** the P1 and P2 operands of this opcode are also used, and are called -** This.P1 and This.P2. +** the P1, P2 and P5 operands of this opcode are also used, and are called +** This.P1, This.P2 and This.P5. ** ** This opcode helps to optimize IN operators on a multi-column index ** where the IN operator is on the later terms of the index by avoiding @@ -93034,32 +95582,54 @@ seek_not_found: ** ** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which ** is the desired entry that we want the cursor SeekGE.P1 to be pointing -** to. Call this SeekGE.P4/P5 row the "target". +** to. Call this SeekGE.P3/P4 row the "target". ** ** If the SeekGE.P1 cursor is not currently pointing to a valid row, ** then this opcode is a no-op and control passes through into the OP_SeekGE. ** ** If the SeekGE.P1 cursor is pointing to a valid row, then that row ** might be the target row, or it might be near and slightly before the -** target row. This opcode attempts to position the cursor on the target -** row by, perhaps by invoking sqlite3BtreeStep() on the cursor -** between 0 and This.P1 times. -** -** There are three possible outcomes from this opcode:<ol> -** -** <li> If after This.P1 steps, the cursor is still pointing to a place that -** is earlier in the btree than the target row, then fall through -** into the subsquence OP_SeekGE opcode. -** -** <li> If the cursor is successfully moved to the target row by 0 or more -** sqlite3BtreeNext() calls, then jump to This.P2, which will land just -** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. -** -** <li> If the cursor ends up past the target row (indicating the the target -** row does not exist in the btree) then jump to SeekOP.P2. +** target row, or it might be after the target row. If the cursor is +** currently before the target row, then this opcode attempts to position +** the cursor on or after the target row by invoking sqlite3BtreeStep() +** on the cursor between 1 and This.P1 times. +** +** The This.P5 parameter is a flag that indicates what to do if the +** cursor ends up pointing at a valid row that is past the target +** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If +** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 +** case occurs when there are no inequality constraints to the right of +** the IN constraing. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** occurs when there are inequality constraints to the right of the IN +** operator. In that case, the This.P2 will point either directly to or +** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for +** loop terminate. +** +** Possible outcomes from this opcode:<ol> +** +** <li> If the cursor is initally not pointed to any valid row, then +** fall through into the subsequent OP_SeekGE opcode. +** +** <li> If the cursor is left pointing to a row that is before the target +** row, even after making as many as This.P1 calls to +** sqlite3BtreeNext(), then also fall through into OP_SeekGE. +** +** <li> If the cursor is left pointing at the target row, either because it +** was at the target row to begin with or because one or more +** sqlite3BtreeNext() calls moved the cursor to the target row, +** then jump to This.P2.., +** +** <li> If the cursor started out before the target row and a call to +** to sqlite3BtreeNext() moved the cursor off the end of the index +** (indicating that the target row definitely does not exist in the +** btree) then jump to SeekGE.P2, ending the loop. +** +** <li> If the cursor ends up on a valid row that is past the target row +** (indicating that the target row does not exist in the btree) then +** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. ** </ol> */ -case OP_SeekScan: { +case OP_SeekScan: { /* ncycle */ VdbeCursor *pC; int res; int nStep; @@ -93067,14 +95637,25 @@ case OP_SeekScan: { assert( pOp[1].opcode==OP_SeekGE ); - /* pOp->p2 points to the first instruction past the OP_IdxGT that - ** follows the OP_SeekGE. */ + /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the + ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first + ** opcode past the OP_SeekGE itself. */ assert( pOp->p2>=(int)(pOp-aOp)+2 ); - assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE ); - testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); - assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); - assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); - assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); +#ifdef SQLITE_DEBUG + if( pOp->p5==0 ){ + /* There are no inequality constraints following the IN constraint. */ + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT + || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + }else{ + /* There are inequality constraints. */ + assert( pOp->p2==(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_SeekGE ); + } +#endif assert( pOp->p1>0 ); pC = p->apCsr[pOp[1].p1]; @@ -93108,8 +95689,9 @@ case OP_SeekScan: { while(1){ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); if( rc ) goto abort_due_to_error; - if( res>0 ){ + if( res>0 && pOp->p5==0 ){ seekscan_search_fail: + /* Jump to SeekGE.P2, ending the loop */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then skip\n", pOp->p1 - nStep); @@ -93119,7 +95701,8 @@ case OP_SeekScan: { pOp++; goto jump_to_p2; } - if( res==0 ){ + if( res>=0 ){ + /* Jump to This.P2, bypassing the OP_SeekGE opcode */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then success\n", pOp->p1 - nStep); @@ -93139,6 +95722,7 @@ case OP_SeekScan: { break; } nStep--; + pC->cacheStatus = CACHE_STALE; rc = sqlite3BtreeNext(pC->uc.pCursor, 0); if( rc ){ if( rc==SQLITE_DONE ){ @@ -93168,7 +95752,7 @@ case OP_SeekScan: { ** ** P1 must be a valid b-tree cursor. */ -case OP_SeekHit: { +case OP_SeekHit: { /* ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; @@ -93195,12 +95779,16 @@ case OP_SeekHit: { /* Opcode: IfNotOpen P1 P2 * * * ** Synopsis: if( !csr[P1] ) goto P2 ** -** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +** If cursor P1 is not open or if P1 is set to a NULL row using the +** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through. */ case OP_IfNotOpen: { /* jump */ + VdbeCursor *pCur; + assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); - if( !p->apCsr[pOp->p1] ){ + pCur = p->apCsr[pOp->p1]; + VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); + if( pCur==0 || pCur->nullRow ){ goto jump_to_p2_and_check_for_interrupt; } break; @@ -93296,7 +95884,7 @@ case OP_IfNotOpen: { /* jump */ ** ** See also: NotFound, Found, NotExists */ -case OP_IfNoHope: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; @@ -93310,9 +95898,9 @@ case OP_IfNoHope: { /* jump, in3 */ /* Fall through into OP_NotFound */ /* no break */ deliberate_fall_through } -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; int ii; VdbeCursor *pC; @@ -93442,7 +96030,7 @@ case OP_Found: { /* jump, in3 */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_SeekRowid: { /* jump, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -93467,7 +96055,7 @@ case OP_SeekRowid: { /* jump, in3 */ } /* Fall through into OP_NotExists */ /* no break */ deliberate_fall_through -case OP_NotExists: /* jump, in3 */ +case OP_NotExists: /* jump, in3, ncycle */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); @@ -93747,8 +96335,11 @@ case OP_Insert: { if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pOp->p5 & OPFLAG_LASTROWID)==0 || (pOp->p5 & OPFLAG_NCHANGE)!=0 ); + if( pOp->p5 & OPFLAG_NCHANGE ){ + p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + } assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); x.pData = pData->z; x.nData = pData->n; @@ -93759,6 +96350,7 @@ case OP_Insert: { x.nZero = 0; } x.pKey = 0; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), seekResult @@ -94090,7 +96682,7 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; @@ -94189,8 +96781,8 @@ case OP_NullRow: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_SeekEnd: -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -94291,17 +96883,22 @@ case OP_Sort: { /* jump */ ** If the table or index is not empty, fall through to the following ** instruction. ** +** If P2 is zero, that is an assertion that the P1 table is never +** empty and hence the jump will never be taken. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); assert( pOp->p5==0 ); + assert( pOp->p2>=0 && pOp->p2<p->nOp ); + pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); @@ -94321,9 +96918,10 @@ case OP_Rewind: { /* jump */ } if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; - assert( pOp->p2>0 && pOp->p2<p->nOp ); - VdbeBranchTaken(res!=0,2); - if( res ) goto jump_to_p2; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } break; } @@ -94389,9 +96987,11 @@ case OP_SorterNext: { /* jump */ rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_Prev: /* jump */ +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - assert( pOp->p5<ArraySize(p->aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); @@ -94402,9 +97002,11 @@ case OP_Prev: /* jump */ rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); goto next_tail; -case OP_Next: /* jump */ +case OP_Next: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); - assert( pOp->p5<ArraySize(p->aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); @@ -94592,8 +97194,8 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_DeferredSeek: -case OP_IdxRowid: { /* out2 */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ VdbeCursor *pC; /* The P1 index cursor */ VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ i64 rowid; /* Rowid that P1 current points to */ @@ -94611,10 +97213,10 @@ case OP_IdxRowid: { /* out2 */ ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ rc = sqlite3VdbeCursorRestore(pC); - /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happens for an IdxRowid - ** or Seek opcode */ - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + /* sqlite3VdbeCursorRestore() may fail if the cursor has been disturbed + ** since it was last positioned and an error (e.g. OOM or an IO error) + ** occurs while trying to reposition it. */ + if( rc!=SQLITE_OK ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ @@ -94655,8 +97257,8 @@ case OP_IdxRowid: { /* out2 */ ** seek operation now, without further delay. If the cursor seek has ** already occurred, this instruction is a no-op. */ -case OP_FinishSeek: { - VdbeCursor *pC; /* The P1 index cursor */ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ assert( pOp->p1>=0 && pOp->p1<p->nCursor ); pC = p->apCsr[pOp->p1]; @@ -94711,10 +97313,10 @@ case OP_FinishSeek: { ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -95125,13 +97727,14 @@ case OP_IntegrityCk: { pIn1 = &aMem[pOp->p1]; assert( pOp->p5<db->nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - z = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, + (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); - }else if( z==0 ){ - goto no_mem; + }else if( rc ){ + sqlite3_free(z); + goto abort_due_to_error; }else{ pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); @@ -95335,9 +97938,6 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pFrame->anExec = p->anExec; -#endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif @@ -95374,9 +97974,6 @@ case OP_Program: { /* jump */ memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = 0; -#endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not ** try to reuse register values from the first use. */ @@ -95516,7 +98113,7 @@ case OP_IfPos: { /* jump, in1 */ ** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** ** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3] ** holds the offset counter. The opcode computes the combined value ** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] ** value computed is the total number of rows that will need to be @@ -95778,6 +98375,7 @@ case OP_AggFinal: { } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } @@ -96133,7 +98731,7 @@ case OP_VDestroy: { ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -96180,7 +98778,7 @@ case OP_VOpen: { ** cursor. Register P3 is used to hold the values returned by ** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). */ -case OP_VInitIn: { /* out2 */ +case OP_VInitIn: { /* out2, ncycle */ VdbeCursor *pC; /* The cursor containing the RHS values */ ValueList *pRhs; /* New ValueList object to put in reg[P2] */ @@ -96191,7 +98789,7 @@ case OP_VInitIn: { /* out2 */ pRhs->pOut = &aMem[pOp->p3]; pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; - sqlite3VdbeMemSetPointer(pOut, pRhs, "ValueList", sqlite3_free); + sqlite3VdbeMemSetPointer(pOut, pRhs, "ValueList", sqlite3VdbeValueListFree); break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -96217,7 +98815,7 @@ case OP_VInitIn: { /* out2 */ ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; @@ -96277,7 +98875,7 @@ case OP_VFilter: { /* jump */ ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are ** unused by OP_VColumn. */ -case OP_VColumn: { +case OP_VColumn: { /* ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; Mem *pDest; @@ -96329,7 +98927,7 @@ case OP_VColumn: { ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; @@ -96912,11 +99510,13 @@ default: { /* This is really OP_Noop, OP_Explain */ *****************************************************************************/ } -#ifdef VDBE_PROFILE - { - u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - if( endTime>start ) pOrigOp->cycles += endTime - start; - pOrigOp->cnt++; +#if defined(VDBE_PROFILE) + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; } #endif @@ -96993,6 +99593,18 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: +#if defined(VDBE_PROFILE) + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } +#endif + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ nProgressLimit += db->nProgressOps; @@ -97004,7 +99616,9 @@ vdbe_return: } #endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; - sqlite3VdbeLeave(p); + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeLeave(p); + } assert( rc!=SQLITE_OK || nExtraDelete==0 || sqlite3_strlike("DELETE%",p->zSql,0)!=0 ); @@ -97382,7 +99996,7 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); sqlite3ParseObjectReset(&sParse); rc = sqlite3ApiExit(db, rc); @@ -97541,7 +100155,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ((Vdbe*)p->pStmt)->rc = SQLITE_OK; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); @@ -100411,6 +103025,9 @@ static int bytecodevtabConnect( ");" }; + (void)argc; + (void)argv; + (void)pzErr; rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); if( rc==SQLITE_OK ){ pNew = sqlite3_malloc( sizeof(*pNew) ); @@ -100646,6 +103263,7 @@ static int bytecodevtabFilter( bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; int rc = SQLITE_OK; + (void)idxStr; bytecodevtabCursorClear(pCur); pCur->iRowid = 0; @@ -101114,6 +103732,8 @@ SQLITE_PRIVATE int sqlite3JournalOpen( ){ MemJournal *p = (MemJournal*)pJfd; + assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) ); + /* Zero the file-handle object. If nSpill was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls @@ -101555,9 +104175,7 @@ static void resolveAlias( pExpr->y.pWin->pOwner = pExpr; } } - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprDelete, - pDup); + sqlite3ExprDeferredDelete(pParse, pDup); } } @@ -101661,6 +104279,32 @@ static void extendFJMatch( } /* +** Return TRUE (non-zero) if zTab is a valid name for the schema table pTab. +*/ +static SQLITE_NOINLINE int isValidSchemaTableName( + const char *zTab, /* Name as it appears in the SQL */ + Table *pTab, /* The schema table we are trying to match */ + Schema *pSchema /* non-NULL if a database qualifier is present */ +){ + const char *zLegacy; + assert( pTab!=0 ); + assert( pTab->tnum==1 ); + if( sqlite3StrNICmp(zTab, "sqlite_", 7)!=0 ) return 0; + zLegacy = pTab->zName; + if( strcmp(zLegacy+7, &LEGACY_TEMP_SCHEMA_TABLE[7])==0 ){ + if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ + return 1; + } + if( pSchema==0 ) return 0; + if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + }else{ + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + } + return 0; +} + +/* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up ** that name in the set of source tables in pSrcList and make the pExpr ** expression node refer back to that source column. The following changes @@ -101813,15 +104457,17 @@ static int lookupName( } assert( zDb==0 || zTab!=0 ); if( zTab ){ - const char *zTabName; if( zDb ){ if( pTab->pSchema!=pSchema ) continue; if( pSchema==0 && strcmp(zDb,"*")!=0 ) continue; } - zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; - assert( zTabName!=0 ); - if( sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; + if( pItem->zAlias!=0 ){ + if( sqlite3StrICmp(zTab, pItem->zAlias)!=0 ){ + continue; + } + }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ + if( pTab->tnum!=1 ) continue; + if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ @@ -101897,7 +104543,8 @@ static int lookupName( assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 - && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + && ALWAYS(zTab==0 + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -101964,6 +104611,7 @@ static int lookupName( if( pParse->bReturning ){ eNewExprOp = TK_REGISTER; pExpr->op2 = TK_COLUMN; + pExpr->iColumn = iCol; pExpr->iTable = pNC->uNC.iBaseReg + (pTab->nCol+1)*pExpr->iTable + sqlite3TableColumnToStorage(pTab, iCol) + 1; }else{ @@ -102376,14 +105024,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ testcase( ExprHasProperty(pExpr, EP_OuterON) ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->op==TK_NOTNULL ){ - pExpr->u.zToken = "true"; - ExprSetProperty(pExpr, EP_IsTrue); - }else{ - pExpr->u.zToken = "false"; - ExprSetProperty(pExpr, EP_IsFalse); - } - pExpr->op = TK_TRUEFALSE; + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && i<ArraySize(anRef); p=p->pNext, i++){ p->nRef = anRef[i]; } @@ -102685,8 +105329,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; } + pNC->ncFlags |= NC_Subquery; } break; } @@ -103640,50 +106284,122 @@ SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table *pTab, int iCol){ */ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ int op; - while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ - assert( pExpr->op==TK_COLLATE - || pExpr->op==TK_IF_NULL_ROW - || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); - pExpr = pExpr->pLeft; - assert( pExpr!=0 ); - } op = pExpr->op; - if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ - assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ + while( 1 /* exit-by-break */ ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); } - } - if( op==TK_SELECT ){ - assert( ExprUseXSelect(pExpr) ); - assert( pExpr->x.pSelect!=0 ); - assert( pExpr->x.pSelect->pEList!=0 ); - assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); - } + if( op==TK_SELECT ){ + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } #ifndef SQLITE_OMIT_CAST - if( op==TK_CAST ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken, 0); - } + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } #endif - if( op==TK_SELECT_COLUMN ){ - assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); - assert( pExpr->iColumn < pExpr->iTable ); - assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); - return sqlite3ExprAffinity( - pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr - ); - } - if( op==TK_VECTOR ){ - assert( ExprUseXList(pExpr) ); - return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); + assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + if( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + op = pExpr->op; + continue; + } + if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; } return pExpr->affExpr; } /* +** Make a guess at all the possible datatypes of the result that could +** be returned by an expression. Return a bitmask indicating the answer: +** +** 0x01 Numeric +** 0x02 Text +** 0x04 Blob +** +** If the expression must return NULL, then 0x00 is returned. +*/ +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr){ + while( pExpr ){ + switch( pExpr->op ){ + case TK_COLLATE: + case TK_IF_NULL_ROW: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + break; + } + case TK_NULL: { + pExpr = 0; + break; + } + case TK_STRING: { + return 0x02; + } + case TK_BLOB: { + return 0x04; + } + case TK_CONCAT: { + return 0x06; + } + case TK_VARIABLE: + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + return 0x07; + } + case TK_COLUMN: + case TK_AGG_COLUMN: + case TK_SELECT: + case TK_CAST: + case TK_SELECT_COLUMN: + case TK_VECTOR: { + int aff = sqlite3ExprAffinity(pExpr); + if( aff>=SQLITE_AFF_NUMERIC ) return 0x05; + if( aff==SQLITE_AFF_TEXT ) return 0x06; + return 0x07; + } + case TK_CASE: { + int res = 0; + int ii; + ExprList *pList = pExpr->x.pList; + assert( ExprUseXList(pExpr) && pList!=0 ); + assert( pList->nExpr > 0); + for(ii=1; ii<pList->nExpr; ii+=2){ + res |= sqlite3ExprDataType(pList->a[ii].pExpr); + } + if( pList->nExpr % 2 ){ + res |= sqlite3ExprDataType(pList->a[pList->nExpr-1].pExpr); + } + return res; + } + default: { + return 0x01; + } + } /* End of switch(op) */ + } /* End of while(pExpr) */ + return 0x00; +} + +/* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to a new Expr node that ** implements the COLLATE operator. @@ -103770,18 +106486,17 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ + int j; assert( ExprUseYTab(p) ); - if( p->y.pTab!=0 ){ - /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = p->iColumn; - if( j>=0 ){ - const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - } - break; + assert( p->y.pTab!=0 ); + if( (j = p->iColumn)>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); } + break; } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; @@ -103803,11 +106518,10 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ }else{ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( ExprUseXList(p) ); - assert( p->x.pList==0 || p->pRight==0 ); - if( p->x.pList!=0 && !db->mallocFailed ){ + assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); + if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; - for(i=0; ALWAYS(i<p->x.pList->nExpr); i++){ + for(i=0; i<p->x.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; @@ -104366,7 +107080,9 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){ */ static void exprSetHeight(Expr *p){ int nHeight = p->pLeft ? p->pLeft->nHeight : 0; - if( p->pRight && p->pRight->nHeight>nHeight ) nHeight = p->pRight->nHeight; + if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){ + nHeight = p->pRight->nHeight; + } if( ExprUseXSelect(p) ){ heightOfSelect(p->x.pSelect, &nHeight); }else if( p->x.pList ){ @@ -104509,15 +107225,26 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); }else{ + assert( ExprUseXList(pRoot) ); + assert( pRoot->x.pSelect==0 ); if( pRight ){ pRoot->pRight = pRight; pRoot->flags |= EP_Propagate & pRight->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + pRoot->nHeight = pRight->nHeight+1; + }else{ + pRoot->nHeight = 1; +#endif } if( pLeft ){ pRoot->pLeft = pLeft; pRoot->flags |= EP_Propagate & pLeft->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pLeft->nHeight>=pRoot->nHeight ){ + pRoot->nHeight = pLeft->nHeight+1; + } +#endif } - exprSetHeight(pRoot); } } @@ -104626,9 +107353,9 @@ SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprLis ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. ** -** If one side or the other of the AND is known to be false, then instead -** of returning an AND expression, just return a constant expression with -** a value of false. +** If one side or the other of the AND is known to be false, and neither side +** is part of an ON clause, then instead of returning an AND expression, +** just return a constant expression with a value of false. */ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ sqlite3 *db = pParse->db; @@ -104636,14 +107363,17 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) - && !IN_RENAME_OBJECT - ){ - sqlite3ExprDeferredDelete(pParse, pLeft); - sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + u32 f = pLeft->flags | pRight->flags; + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } } } @@ -104803,6 +107533,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n */ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); + assert( db!=0 ); assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -104834,12 +107565,8 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ #endif } } - if( ExprHasProperty(p, EP_MemToken) ){ - assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3DbFree(db, p->u.zToken); - } if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ @@ -104870,8 +107597,9 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** pExpr to the pParse->pConstExpr list with a register number of 0. */ SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - pParse->pConstExpr = - sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprDelete, + pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -104945,7 +107673,6 @@ static int dupedExprStructSize(const Expr *p, int flags){ }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); assert( !ExprHasProperty(p, EP_OuterON) ); - assert( !ExprHasProperty(p, EP_MemToken) ); assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; @@ -105049,7 +107776,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; ExprClearVVAProperties(pNew); @@ -105625,12 +108352,13 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i = pList->nExpr; struct ExprList_item *pItem = pList->a; assert( pList->nExpr>0 ); + assert( db!=0 ); do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zEName); + if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName); pItem++; }while( --i>0 ); - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); @@ -105890,12 +108618,17 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ } /* -** Check pExpr to see if it is an invariant constraint on data source pSrc. +** Check pExpr to see if it is an constraint on the single data source +** pSrc = &pSrcList->a[iSrc]. In other words, check to see if pExpr +** constrains pSrc but does not depend on any other tables or data +** sources anywhere else in the query. Return true (non-zero) if pExpr +** is a constraint on pSrc only. +** ** This is an optimization. False negatives will perhaps cause slower ** queries, but false positives will yield incorrect answers. So when in ** doubt, return 0. ** -** To be an invariant constraint, the following must be true: +** To be an single-source constraint, the following must be true: ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** @@ -105906,13 +108639,31 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (4) If pSrc is the right operand of a LEFT JOIN, then... ** (4a) pExpr must come from an ON clause.. - (4b) and specifically the ON clause associated with the LEFT JOIN. +** (4b) and specifically the ON clause associated with the LEFT JOIN. ** ** (5) If pSrc is not the right operand of a LEFT JOIN or the left ** operand of a RIGHT JOIN, then pExpr must be from the WHERE ** clause, not an ON clause. +** +** (6) Either: +** +** (6a) pExpr does not originate in an ON or USING clause, or +** +** (6b) The ON or USING clause from which pExpr is derived is +** not to the left of a RIGHT JOIN (or FULL JOIN). +** +** Without this restriction, accepting pExpr as a single-table +** constraint might move the the ON/USING filter expression +** from the left side of a RIGHT JOIN over to the right side, +** which leads to incorrect answers. See also restriction (9) +** on push-down. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc){ +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( + Expr *pExpr, /* The constraint */ + const SrcList *pSrcList, /* Complete FROM clause */ + int iSrc /* Which element of pSrcList to use */ +){ + const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ return 0; /* rule (3) */ } @@ -105922,6 +108673,19 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc }else{ if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (5) */ } + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) /* (6a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (6b) */ + ){ + int jj; + for(jj=0; jj<iSrc; jj++){ + if( pExpr->w.iJoin==pSrcList->a[jj].iCursor ){ + if( (pSrcList->a[jj].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* restriction (6) */ + } + break; + } + } + } return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ } @@ -106164,7 +108928,7 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient +** or if the SELECT statement needs to be materialized into a transient ** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY @@ -106450,7 +109214,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex( CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); int j; - assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); for(j=0; j<nExpr; j++){ if( pIdx->aiColumn[j]!=pRhs->iColumn ) continue; assert( pIdx->azColl[j] ); @@ -106808,6 +109571,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } if( addrOnce ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iTab); sqlite3VdbeJumpHere(v, addrOnce); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); @@ -106843,6 +109607,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ SelectDest dest; /* How to deal with SELECT result */ int nReg; /* Registers to allocate */ Expr *pLimit; /* New limit expression */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif Vdbe *v = pParse->pVdbe; assert( v!=0 ); @@ -106895,8 +109662,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** In both cases, the query is augmented with "LIMIT 1". Any ** preexisting limit is discarded in place of the new LIMIT 1. */ - ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", addrOnce?"":"CORRELATED ", pSel->selId)); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, -1); nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); pParse->nMem += nReg; @@ -106921,7 +109689,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pLimit = sqlite3PExpr(pParse, TK_NE, sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); } - sqlite3ExprDelete(db, pSel->pLimit->pLeft); + sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ @@ -106939,6 +109707,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ if( addrOnce ){ sqlite3VdbeJumpHere(v, addrOnce); } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); @@ -107347,6 +110116,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( ){ int iAddr; Vdbe *v = pParse->pVdbe; + int nErr = pParse->nErr; assert( v!=0 ); assert( pParse->iSelfTab!=0 ); if( pParse->iSelfTab>0 ){ @@ -107359,6 +110129,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); + if( pParse->nErr>nErr ) pParse->db->errByteOffset = -1; } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ @@ -107374,10 +110145,8 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( ){ Column *pCol; assert( v!=0 ); - if( pTab==0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); - return; - } + assert( pTab!=0 ); + assert( iCol!=XN_EXPR ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); @@ -107435,7 +110204,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( assert( pParse->pVdbe!=0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; } return iReg; @@ -107504,7 +110273,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ ** so that a subsequent copy will not be merged into this one. */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ - if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ } } @@ -107614,10 +110383,13 @@ static int exprCodeInlineFunction( ** the type affinity of the argument. This is used for testing of ** the SQLite type logic. */ - const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + const char *azAff[] = { "blob", "text", "numeric", "integer", + "real", "flexnum" }; char aff; assert( nFarg==1 ); aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + assert( aff<=SQLITE_AFF_NONE + || (aff>=SQLITE_AFF_BLOB && aff<=SQLITE_AFF_FLEXNUM) ); sqlite3VdbeLoadString(v, target, (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); break; @@ -107627,6 +110399,64 @@ static int exprCodeInlineFunction( return target; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxEpr; p; p=p->pIENext){ + u8 exprAff; + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + assert( p->aff>=SQLITE_AFF_BLOB && p->aff<=SQLITE_AFF_NUMERIC ); + exprAff = sqlite3ExprAffinity(pExpr); + if( (exprAff<=SQLITE_AFF_BLOB && p->aff!=SQLITE_AFF_BLOB) + || (exprAff==SQLITE_AFF_TEXT && p->aff!=SQLITE_AFF_TEXT) + || (exprAff>=SQLITE_AFF_NUMERIC && p->aff!=SQLITE_AFF_NUMERIC) + ){ + /* Affinity mismatch on a generated column */ + continue; + } + + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxEpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -107655,6 +110485,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxEpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; @@ -107664,16 +110499,29 @@ expr_code_doover: AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol; assert( pAggInfo!=0 ); - assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn ); + assert( pExpr->iAgg>=0 ); + if( pExpr->iAgg>=pAggInfo->nColumn ){ + /* Happens when the left table of a RIGHT JOIN is null and + ** is using an expression index */ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); +#ifdef SQLITE_VDBE_COVERAGE + /* Verify that the OP_Null above is exercised by tests + ** tag-20230325-2 */ + sqlite3VdbeAddOp2(v, OP_NotNull, target, 1); + VdbeCoverageNeverTaken(v); +#endif + break; + } pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); }else{ VdbeComment((v,"%s.%s", @@ -107683,6 +110531,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through @@ -107700,13 +110553,10 @@ expr_code_doover: int aff; iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - }else{ - aff = pExpr->affExpr; - } + assert( pExpr->y.pTab!=0 ); + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); if( aff>SQLITE_AFF_BLOB ){ - static const char zAff[] = "B\000C\000D\000E"; + static const char zAff[] = "B\000C\000D\000E\000F"; assert( SQLITE_AFF_BLOB=='A' ); assert( SQLITE_AFF_TEXT=='B' ); sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, @@ -107766,12 +110616,10 @@ expr_code_doover: } } assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); - if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } return iReg; } case TK_INTEGER: { @@ -107838,11 +110686,8 @@ expr_code_doover: #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( inReg==target ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); @@ -107985,7 +110830,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -108174,17 +111019,16 @@ expr_code_doover: return target; } case TK_COLLATE: { - if( !ExprHasProperty(pExpr, EP_Collate) - && ALWAYS(pExpr->pLeft) - && pExpr->pLeft->op==TK_FUNCTION - ){ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } - sqlite3VdbeAddOp1(v, OP_ClrSubtype, inReg); - return inReg; + if( !ExprHasProperty(pExpr, EP_Collate) ){ + /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called + ** "SOFT-COLLATE" that is added to constraints that are pushed down + ** from outer queries into sub-queries by the push-down optimization. + ** Clear subtypes as subtypes may not cross a subquery boundary. + */ + assert( pExpr->pLeft ); + sqlite3ExprCode(pParse, pExpr->pLeft, target); + sqlite3VdbeAddOp1(v, OP_ClrSubtype, target); + return target; }else{ pExpr = pExpr->pLeft; goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */ @@ -108270,16 +111114,34 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; u8 okConstFactor = pParse->okConstFactor; - addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); - /* Temporarily disable factoring of constant expressions, since - ** even though expressions may appear to be constant, they are not - ** really constant because they originate from the right-hand side - ** of a LEFT JOIN. */ - pParse->okConstFactor = 0; - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + AggInfo *pAggInfo = pExpr->pAggInfo; + if( pAggInfo ){ + assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn ); + if( !pAggInfo->directMode ){ + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); + break; + } + if( pExpr->pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pAggInfo->aCol[pExpr->iAgg].iSorterColumn, + target); + inReg = target; + break; + } + } + addrINR = sqlite3VdbeAddOp3(v, OP_IfNullRow, pExpr->iTable, 0, target); + /* The OP_IfNullRow opcode above can overwrite the result register with + ** NULL. So we have to ensure that the result register is not a value + ** that is suppose to be a constant. Two defenses are needed: + ** (1) Temporarily disable factoring of constant expressions + ** (2) Make sure the computed value really is stored in register + ** "target" and not someplace else. + */ + pParse->okConstFactor = 0; /* note (1) above */ + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( target==inReg ); pParse->okConstFactor = okConstFactor; sqlite3VdbeJumpHere(v, addrINR); - sqlite3VdbeChangeP3(v, addrINR, inReg); break; } @@ -108516,7 +111378,9 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){ + if( ALWAYS(pExpr) + && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + ){ op = OP_Copy; }else{ op = OP_SCopy; @@ -108611,7 +111475,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( inReg!=target+i ){ VdbeOp *pOp; if( copyOp==OP_Copy - && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i && pOp->p5==0 /* The do-not-merge flag must be clear */ @@ -108810,6 +111674,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -108984,6 +111849,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -109137,7 +112003,13 @@ SQLITE_PRIVATE int sqlite3ExprCompare( if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } assert( !ExprHasProperty(pA, EP_IntValue) ); assert( !ExprHasProperty(pB, EP_IntValue) ); @@ -109439,10 +112311,10 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); if( (pLeft->op==TK_COLUMN - && pLeft->y.pTab!=0 + && ALWAYS(pLeft->y.pTab!=0) && IsVirtual(pLeft->y.pTab)) || (pRight->op==TK_COLUMN - && pRight->y.pTab!=0 + && ALWAYS(pRight->y.pTab!=0) && IsVirtual(pRight->y.pTab)) ){ return WRC_Prune; @@ -109647,6 +112519,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ Walker w; struct RefSrcList x; + assert( pParse->db!=0 ); memset(&w, 0, sizeof(w)); memset(&x, 0, sizeof(x)); w.xExprCallback = exprRefToSrcList; @@ -109663,7 +112536,7 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); } #endif - sqlite3DbFree(pParse->db, x.aiExclude); + if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude); if( w.eCode & 0x01 ){ return 1; }else if( w.eCode ){ @@ -109681,10 +112554,8 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList ** it does, make a copy. This is done because the pExpr argument is ** subject to change. ** -** The copy is stored on pParse->pConstExpr with a register number of 0. -** This will cause the expression to be deleted automatically when the -** Parse object is destroyed, but the zero register number means that it -** will not generate any code in the preamble. +** The copy is scheduled for deletion using the sqlite3ExprDeferredDelete() +** which builds on the sqlite3ParserAddCleanup() mechanism. */ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) @@ -109694,10 +112565,11 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; - assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); - if( pExpr->op==TK_AGG_COLUMN ){ - assert( iAgg>=0 && iAgg<pAggInfo->nColumn ); - if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + assert( iAgg>=0 ); + if( pExpr->op!=TK_AGG_FUNCTION ){ + if( iAgg<pAggInfo->nColumn + && pAggInfo->aCol[iAgg].pCExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; @@ -109705,8 +112577,10 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ } } }else{ - assert( iAgg>=0 && iAgg<pAggInfo->nFunc ); - if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + assert( pExpr->op==TK_AGG_FUNCTION ); + if( ALWAYS(iAgg<pAggInfo->nFunc) + && pAggInfo->aFunc[iAgg].pFExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; @@ -109762,6 +112636,73 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ } /* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + assert( pAggInfo->iFirstReg==0 ); + pCol = pAggInfo->aCol; + for(k=0; k<pAggInfo->nColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; j<n; j++, pTerm++){ + Expr *pE = pTerm->pExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( pExpr->pAggInfo==0 || pExpr->pAggInfo==pAggInfo ); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + +/* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates ** for additional information. @@ -109774,76 +112715,64 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + assert( pParse->iSelfTab==0 ); + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; + for(i=0; i<pSrcList->nSrc; i++){ + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + } + if( i>=pSrcList->nSrc ) break; + if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ + if( pParse->nErr ){ return WRC_Abort; } + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + if( pParse->nErr ){ return WRC_Abort; } + assert( pAggInfo->aCol!=0 ); + assert( tmp.iAgg<pAggInfo->nColumn ); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } + case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_IF_NULL_ROW ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; i<pSrcList->nSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; k<pAggInfo->nColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; j<n; j++, pTerm++){ - Expr *pE = pTerm->pExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return WRC_Prune; + return WRC_Continue; } case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 @@ -109868,7 +112797,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, @@ -109998,6 +112926,37 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ } /* +** Make sure sufficient registers have been allocated so that +** iReg is a valid register number. +*/ +SQLITE_PRIVATE void sqlite3TouchRegister(Parse *pParse, int iReg){ + if( pParse->nMem<iReg ) pParse->nMem = iReg; +} + +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +/* +** Return the latest reusable register in the set of all registers. +** The value returned is no less than iMin. If any register iMin or +** greater is in permanent use, then return one more than that last +** permanent register. +*/ +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ + const ExprList *pList = pParse->pConstExpr; + if( pList ){ + int i; + for(i=0; i<pList->nExpr; i++){ + if( pList->a[i].u.iConstExprReg>=iMin ){ + iMin = pList->a[i].u.iConstExprReg + 1; + } + } + } + pParse->nTempReg = 0; + pParse->nRangeReg = 0; + return iMin; +} +#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */ + +/* ** Validate that no temporary register falls within the range of ** iFirst..iLast, inclusive. This routine is only call from within assert() ** statements. @@ -110016,6 +112975,14 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ return 0; } } + if( pParse->pConstExpr ){ + ExprList *pList = pParse->pConstExpr; + for(i=0; i<pList->nExpr; i++){ + int iReg = pList->a[i].u.iConstExprReg; + if( iReg==0 ) continue; + if( iReg>=iFirst && iReg<=iLast ) return 0; + } + } return 1; } #endif /* SQLITE_DEBUG */ @@ -110765,13 +113732,14 @@ static void renameTokenCheckAll(Parse *pParse, const void *pPtr){ assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( pParse->nErr==0 ){ const RenameToken *p; - u8 i = 0; + u32 i = 1; for(p=pParse->pRename; p; p=p->pNext){ if( p->p ){ assert( p->p!=pPtr ); - i += *(u8*)(p->p); + i += *(u8*)(p->p) | 1; } } + assert( i>0 ); } } #else @@ -111303,6 +114271,19 @@ static int renameEditSql( } /* +** Set all pEList->a[].fg.eEName fields in the expression-list to val. +*/ +static void renameSetENames(ExprList *pEList, int val){ + if( pEList ){ + int i; + for(i=0; i<pEList->nExpr; i++){ + assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); + pEList->a[i].fg.eEName = val; + } + } +} + +/* ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming ** it was read from the schema of database zDb. Return SQLITE_OK if ** successful. Otherwise, return an SQLite error code and leave an error @@ -111349,7 +114330,17 @@ static int renameResolveTrigger(Parse *pParse){ pSrc = 0; rc = SQLITE_NOMEM; }else{ + /* pStep->pExprList contains an expression-list used for an UPDATE + ** statement. So the a[].zEName values are the RHS of the + ** "<col> = <expr>" clauses of the UPDATE statement. So, before + ** running SelectPrep(), change all the eEName values in + ** pStep->pExprList to ENAME_SPAN (from their current value of + ** ENAME_NAME). This is to prevent any ids in ON() clauses that are + ** part of pSrc from being incorrectly resolved against the + ** a[].zEName values as if they were column aliases. */ + renameSetENames(pStep->pExprList, ENAME_SPAN); sqlite3SelectPrep(pParse, pSel, 0); + renameSetENames(pStep->pExprList, ENAME_NAME); rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK; assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList ); assert( pSrc==pSel->pSrc ); @@ -113257,6 +116248,7 @@ static void analyzeVdbeCommentIndexWithColumnName( if( NEVER(i==XN_ROWID) ){ VdbeComment((v,"%s.rowid",pIdx->zName)); }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); }else{ VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); @@ -113297,11 +116289,15 @@ static void analyzeOneTable( int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_STAT4 + int doOnce = 1; /* Flag for a one-time computation */ +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK Table *pStat1 = 0; #endif - pParse->nMem = MAX(pParse->nMem, iMem); + sqlite3TouchRegister(pParse, iMem); + assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) ); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -113407,7 +116403,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + sqlite3TouchRegister(pParse, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -113579,7 +116575,35 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol); + if( doOnce ){ + int mxCol = nCol; + Index *pX; + + /* Compute the maximum number of columns in any index */ + for(pX=pTab->pIndex; pX; pX=pX->pNext){ + int nColX; /* Number of columns in pX */ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){ + nColX = pX->nKeyCol; + }else{ + nColX = pX->nColumn; + } + if( nColX>mxCol ) mxCol = nColX; + } + + /* Allocate space to compute results for the largest index */ + sqlite3TouchRegister(pParse, regCol+mxCol); + doOnce = 0; +#ifdef SQLITE_DEBUG + /* Verify that the call to sqlite3ClearTempRegCache() below + ** really is needed. + ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) + */ + testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); +#endif + sqlite3ClearTempRegCache(pParse); /* tag-20230325-1 */ + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); + } + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) ); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); @@ -113660,6 +116684,11 @@ static void analyzeDatabase(Parse *pParse, int iDb){ for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); +#ifdef SQLITE_ENABLE_STAT4 + iMem = sqlite3FirstAvailableRegister(pParse, iMem); +#else + assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) ); +#endif } loadAnalysis(pParse, iDb); } @@ -113900,6 +116929,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ + assert( db!=0 ); + assert( pIdx!=0 ); #ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; @@ -113909,7 +116940,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } sqlite3DbFree(db, pIdx->aSample); } - if( db && db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; } @@ -114045,6 +117076,10 @@ static int loadStatTbl( pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); assert( pIdx==0 || pIdx->nSample==0 ); if( pIdx==0 ) continue; + if( pIdx->aSample!=0 ){ + /* The same index appears in sqlite_stat4 under multiple names */ + continue; + } assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ nIdxCol = pIdx->nKeyCol; @@ -114052,6 +117087,7 @@ static int loadStatTbl( nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; + pIdx->mxSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -114091,6 +117127,11 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; + if( pIdx->nSample>=pIdx->mxSample ){ + /* Too many slots used because the same index appears in + ** sqlite_stat4 using multiple names */ + continue; + } /* This next condition is true if data has already been loaded from ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; @@ -114134,11 +117175,12 @@ static int loadStat4(sqlite3 *db, const char *zDb){ const Table *pStat4; assert( db->lookaside.bDisable ); - if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 + if( OptimizationEnabled(db, SQLITE_Stat4) + && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 && IsOrdinaryTable(pStat4) ){ rc = loadStatTbl(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); @@ -114328,7 +117370,7 @@ static void attachFunc( char *zErr = 0; unsigned int flags; Db *aNew; /* New array of Db pointers */ - Db *pNew; /* Db object for the newly attached database */ + Db *pNew = 0; /* Db object for the newly attached database */ char *zErrDyn = 0; sqlite3_vfs *pVfs; @@ -114348,13 +117390,26 @@ static void attachFunc( /* This is not a real ATTACH. Instead, this routine is being called ** from sqlite3_deserialize() to close database db->init.iDb and ** reopen it as a MemDB */ + Btree *pNewBt = 0; pVfs = sqlite3_vfs_find("memdb"); if( pVfs==0 ) return; - pNew = &db->aDb[db->init.iDb]; - if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt); - pNew->pBt = 0; - pNew->pSchema = 0; - rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNewBt, 0, SQLITE_OPEN_MAIN_DB); + if( rc==SQLITE_OK ){ + Schema *pNewSchema = sqlite3SchemaGet(db, pNewBt); + if( pNewSchema ){ + /* Both the Btree and the new Schema were allocated successfully. + ** Close the old db and update the aDb[] slot with the new memdb + ** values. */ + pNew = &db->aDb[db->init.iDb]; + if( ALWAYS(pNew->pBt) ) sqlite3BtreeClose(pNew->pBt); + pNew->pBt = pNewBt; + pNew->pSchema = pNewSchema; + }else{ + sqlite3BtreeClose(pNewBt); + rc = SQLITE_NOMEM; + } + } + if( rc ) goto attach_error; }else{ /* This is a real ATTACH ** @@ -114467,7 +117522,7 @@ static void attachFunc( } #endif if( rc ){ - if( !REOPEN_AS_MEMDB(db) ){ + if( ALWAYS(!REOPEN_AS_MEMDB(db)) ){ int iDb = db->nDb - 1; assert( iDb>=2 ); if( db->aDb[iDb].pBt ){ @@ -114584,6 +117639,8 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ) goto attach_end; + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -115259,6 +118316,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + int iDb, i; assert( pParse->pToplevel==0 ); db = pParse->db; @@ -115288,7 +118346,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ if( pParse->bReturning ){ Returning *pReturning = pParse->u1.pReturning; int addrRewind; - int i; int reg; if( pReturning->nRetCol ){ @@ -115325,76 +118382,69 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( db->mallocFailed==0 - && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) - ){ - int iDb, i; - assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); - sqlite3VdbeJumpHere(v, 0); - assert( db->nDb>0 ); - iDb = 0; - do{ - Schema *pSchema; - if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - pSchema = db->aDb[iDb].pSchema; - sqlite3VdbeAddOp4Int(v, - OP_Transaction, /* Opcode */ - iDb, /* P1 */ - DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pSchema->schema_cookie, /* P3 */ - pSchema->iGeneration /* P4 */ - ); - if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); - }while( ++iDb<db->nDb ); + assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + assert( db->nDb>0 ); + iDb = 0; + do{ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + }while( ++iDb<db->nDb ); #ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=0; i<pParse->nVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; i<pParse->nVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. - */ - if( pParse->pConstExpr ){ - ExprList *pEL = pParse->pConstExpr; - pParse->okConstFactor = 0; - for(i=0; i<pEL->nExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - if( iReg>0 ){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); - } - } + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; i<pEL->nExpr; i++){ + int iReg = pEL->a[i].u.iConstExprReg; + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); } + } - if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; - if( pRet->nRetCol ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); - } + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + if( pRet->nRetCol ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); } - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); } /* Get the VDBE program ready for execution @@ -115433,6 +118483,7 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; + if( pParse->eParseMode ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ va_start(ap, zFormat); zSql = sqlite3VMPrintf(db, zFormat, ap); @@ -115579,7 +118630,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ - if( pParse->disableVtab==0 && db->init.busy==0 ){ + if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){ Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); @@ -115592,7 +118643,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( #endif if( flags & LOCATE_NOERR ) return 0; pParse->checkSchema = 1; - }else if( IsVirtual(p) && pParse->disableVtab ){ + }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){ p = 0; } @@ -115901,16 +118952,17 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); + assert( db!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; i<pTable->nCol; i++, pCol++){ assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); sqlite3DbFree(db, pCol->zCnName); } - sqlite3DbFree(db, pTable->aCol); + sqlite3DbNNFreeNN(db, pTable->aCol); if( IsOrdinaryTable(pTable) ){ sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); } - if( db==0 || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pTable->aCol = 0; pTable->nCol = 0; if( IsOrdinaryTable(pTable) ){ @@ -115947,7 +118999,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ ** a Table object that was going to be marked ephemeral. So do not check ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + assert( db!=0 ); + if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -115957,7 +119010,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); - if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -115971,7 +119024,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ if( IsOrdinaryTable(pTable) ){ sqlite3FkDelete(db, pTable); } -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE else if( IsVirtual(pTable) ){ sqlite3VtabClear(db, pTable); } @@ -115994,8 +119047,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ } SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ + assert( db!=0 ); if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } @@ -116573,7 +119627,7 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ if( pParse->pNewTrigger ){ sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); }else{ - assert( pParse->bReturning==0 ); + assert( pParse->bReturning==0 || pParse->ifNotExists ); } pParse->bReturning = 1; pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); @@ -116599,7 +119653,8 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); + assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 + || pParse->nErr || pParse->ifNotExists ); if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); @@ -117127,6 +120182,14 @@ SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType if( pCol->colFlags & COLFLAG_PRIMKEY ){ makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ } + if( ALWAYS(pExpr) && pExpr->op==TK_ID ){ + /* The value of a generated column needs to be a real expression, not + ** just a reference to another column, in order for covering index + ** optimizations to work correctly. So if the value is not an expression, + ** turn it into one by adding a unary "+" operator. */ + pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); + } + if( pExpr && pExpr->op!=TK_RAISE ) pExpr->affExpr = pCol->affinity; sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; @@ -117263,7 +120326,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", - /* SQLITE_AFF_REAL */ " REAL" + /* SQLITE_AFF_REAL */ " REAL", + /* SQLITE_AFF_FLEXNUM */ " NUM", }; int len; const char *zType; @@ -117279,10 +120343,12 @@ static char *createTableStmt(sqlite3 *db, Table *p){ testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); + testcase( pCol->affinity==SQLITE_AFF_FLEXNUM ); zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -117399,7 +120465,8 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ /* Recompute the colNotIdxed field of the Index. ** ** colNotIdxed is a bitmask that has a 0 bit representing each indexed -** columns that are within the first 63 columns of the table. The +** columns that are within the first 63 columns of the table and a 1 for +** all other bits (all columns that are not in the index). The ** high-order bit of colNotIdxed is always 1. All unindexed columns ** of the table have a 1. ** @@ -117427,7 +120494,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){ } } pIdx->colNotIdxed = ~m; - assert( (pIdx->colNotIdxed>>63)==1 ); + assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */ } /* @@ -117696,6 +120763,7 @@ SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ ** not pass them into code generator routines by mistake. */ static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + (void)pWalker; ExprSetVVAProperty(pExpr, EP_Immutable); return WRC_Continue; } @@ -118168,7 +121236,7 @@ create_view_fail: ** the columns of the view in the pTable structure. Return the number ** of errors. If an error is seen leave an error message in pParse->zErrMsg. */ -SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ +static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ @@ -118193,9 +121261,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #ifndef SQLITE_OMIT_VIEW /* A positive nCol means the columns names for this view are - ** already known. + ** already known. This routine is not called unless either the + ** table is virtual or nCol is zero. */ - if( pTable->nCol>0 ) return 0; + assert( pTable->nCol<=0 ); /* A negative nCol is a special marker meaning that we are currently ** trying to compute the column names. If we enter this routine with @@ -118261,8 +121330,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ && pTable->nCol==pSel->pEList->nExpr ){ assert( db->mallocFailed==0 ); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, - SQLITE_AFF_NONE); + sqlite3SubqueryColumnTypes(pParse, pTable, pSel, SQLITE_AFF_NONE); } }else{ /* CREATE VIEW name AS... without an argument list. Construct @@ -118291,6 +121359,11 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #endif /* SQLITE_OMIT_VIEW */ return nErr; } +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + assert( pTable!=0 ); + if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; + return viewGetColumnNames(pParse, pTable); +} #endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ #ifndef SQLITE_OMIT_VIEW @@ -119156,7 +122229,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( !IN_RENAME_OBJECT ){ if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ + if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){ sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); goto exit_create_index; } @@ -119309,6 +122382,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); @@ -119320,6 +122394,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ pIndex->bHasVCol = 1; + pIndex->bHasExpr = 1; } } pIndex->aiColumn[i] = (i16)j; @@ -119809,12 +122884,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token * */ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; + assert( db!=0 ); if( pList==0 ) return; assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; i<pList->nId; i++){ sqlite3DbFree(db, pList->a[i].zName); } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -120017,11 +123093,12 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; SrcItem *pItem; + assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; i<pList->nSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); @@ -120032,7 +123109,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3ExprDelete(db, pItem->u3.pOn); } } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -120979,6 +124056,7 @@ SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ ** strings is BINARY. */ db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); + sqlite3ExpirePreparedStatements(db, 1); } /* @@ -121284,19 +124362,21 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ Hash temp2; HashElem *pElem; Schema *pSchema = (Schema *)p; + sqlite3 xdb; + memset(&xdb, 0, sizeof(xdb)); temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; sqlite3HashInit(&pSchema->trigHash); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); + sqlite3DeleteTable(&xdb, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); @@ -121395,18 +124475,42 @@ SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char * ** 1) It is a virtual table and no implementation of the xUpdate method ** has been provided ** -** 2) It is a system table (i.e. sqlite_schema), this call is not +** 2) A trigger is currently being coded and the table is a virtual table +** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and +** the table is not SQLITE_VTAB_INNOCUOUS. +** +** 3) It is a system table (i.e. sqlite_schema), this call is not ** part of a nested parse and writable_schema pragma has not ** been specified ** -** 3) The table is a shadow table, the database connection is in +** 4) The table is a shadow table, the database connection is in ** defensive mode, and the current sqlite3_prepare() ** is for a top-level SQL statement. */ +static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ + return 1; + } + + /* Within triggers: + ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY + ** virtual tables + ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS + ** virtual tables if PRAGMA trusted_schema=ON. + */ + if( pParse->pToplevel!=0 + && pTab->u.vtab.p->eVtabRisk > + ((pParse->db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + return 0; +} static int tabIsReadOnly(Parse *pParse, Table *pTab){ sqlite3 *db; if( IsVirtual(pTab) ){ - return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + return vtabIsReadOnly(pParse, pTab); } if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; db = pParse->db; @@ -121418,17 +124522,21 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ } /* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; +** Check to make sure the given table is writable. +** +** If pTab is not writable -> generate an error message and return 1. +** If pTab is writable but other errors have occurred -> return 1. +** If pTab is writable and no prior errors -> return 0; */ -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ if( tabIsReadOnly(pParse, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); return 1; } #ifndef SQLITE_OMIT_VIEW - if( !viewOk && IsView(pTab) ){ + if( IsView(pTab) + && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) + ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); return 1; } @@ -121682,7 +124790,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( goto delete_from_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto delete_from_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -121781,16 +124889,17 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1); + sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1); + }else{ + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } } }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + if( sNC.ncFlags & NC_Subquery ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ @@ -121983,7 +125092,7 @@ delete_from_cleanup: sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); #endif - sqlite3DbFree(db, aToOpen); + if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -123066,7 +126175,7 @@ static int patternCompare( ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ + if( c<0x80 ){ char zStop[3]; int bMatch; if( noCase ){ @@ -123149,7 +126258,13 @@ static int patternCompare( ** non-zero if there is no match. */ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + if( zString==0 ){ + return zGlobPattern!=0; + }else if( zGlobPattern==0 ){ + return 1; + }else { + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + } } /* @@ -123157,7 +126272,13 @@ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ ** a miss - like strcmp(). */ SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + if( zStr==0 ){ + return zPattern!=0; + }else if( zPattern==0 ){ + return 1; + }else{ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + } } /* @@ -123396,7 +126517,7 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } case SQLITE_BLOB: { char const *zBlob = sqlite3_value_blob(pValue); - int nBlob = sqlite3_value_bytes(pValue); + i64 nBlob = sqlite3_value_bytes(pValue); assert( zBlob==sqlite3_value_blob(pValue) ); /* No encoding change */ sqlite3StrAccumEnlarge(pStr, nBlob*2 + 4); if( pStr->accError==0 ){ @@ -123538,6 +126659,96 @@ static void hexFunc( } /* +** Buffer zStr contains nStr bytes of utf-8 encoded text. Return 1 if zStr +** contains character ch, or 0 if it does not. +*/ +static int strContainsChar(const u8 *zStr, int nStr, u32 ch){ + const u8 *zEnd = &zStr[nStr]; + const u8 *z = zStr; + while( z<zEnd ){ + u32 tst = Utf8Read(z); + if( tst==ch ) return 1; + } + return 0; +} + +/* +** The unhex() function. This function may be invoked with either one or +** two arguments. In both cases the first argument is interpreted as text +** a text value containing a set of pairs of hexadecimal digits which are +** decoded and returned as a blob. +** +** If there is only a single argument, then it must consist only of an +** even number of hexadeximal digits. Otherwise, return NULL. +** +** Or, if there is a second argument, then any character that appears in +** the second argument is also allowed to appear between pairs of hexadecimal +** digits in the first argument. If any other character appears in the +** first argument, or if one of the allowed characters appears between +** two hexadecimal digits that make up a single byte, NULL is returned. +** +** The following expressions are all true: +** +** unhex('ABCD') IS x'ABCD' +** unhex('AB CD') IS NULL +** unhex('AB CD', ' ') IS x'ABCD' +** unhex('A BCD', ' ') IS NULL +*/ +static void unhexFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + const u8 *zPass = (const u8*)""; + int nPass = 0; + const u8 *zHex = sqlite3_value_text(argv[0]); + int nHex = sqlite3_value_bytes(argv[0]); +#ifdef SQLITE_DEBUG + const u8 *zEnd = zHex ? &zHex[nHex] : 0; +#endif + u8 *pBlob = 0; + u8 *p = 0; + + assert( argc==1 || argc==2 ); + if( argc==2 ){ + zPass = sqlite3_value_text(argv[1]); + nPass = sqlite3_value_bytes(argv[1]); + } + if( !zHex || !zPass ) return; + + p = pBlob = contextMalloc(pCtx, (nHex/2)+1); + if( pBlob ){ + u8 c; /* Most significant digit of next byte */ + u8 d; /* Least significant digit of next byte */ + + while( (c = *zHex)!=0x00 ){ + while( !sqlite3Isxdigit(c) ){ + u32 ch = Utf8Read(zHex); + assert( zHex<=zEnd ); + if( !strContainsChar(zPass, nPass, ch) ) goto unhex_null; + c = *zHex; + if( c==0x00 ) goto unhex_done; + } + zHex++; + assert( *zEnd==0x00 ); + assert( zHex<=zEnd ); + d = *(zHex++); + if( !sqlite3Isxdigit(d) ) goto unhex_null; + *(p++) = (sqlite3HexToInt(c)<<4) | sqlite3HexToInt(d); + } + } + + unhex_done: + sqlite3_result_blob(pCtx, pBlob, (p - pBlob), sqlite3_free); + return; + + unhex_null: + sqlite3_free(pBlob); + return; +} + + +/* ** The zeroblob(N) function returns a zero-filled blob of size N bytes. */ static void zeroblobFunc( @@ -123742,7 +126953,7 @@ static void trimFunc( /* ** The "unknown" function is automatically substituted in place of ** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN -** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** when the SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION compile-time option is used. ** When the "sqlite3" command-line shell is built using this functionality, ** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries ** involving application-defined functions to be examined in a generic @@ -123754,6 +126965,9 @@ static void unknownFunc( sqlite3_value **argv ){ /* no-op */ + (void)context; + (void)argc; + (void)argv; } #endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ @@ -124383,6 +127597,18 @@ static double xCeil(double x){ return ceil(x); } static double xFloor(double x){ return floor(x); } /* +** Some systems do not have log2() and log10() in their standard math +** libraries. +*/ +#if defined(HAVE_LOG10) && HAVE_LOG10==0 +# define log10(X) (0.4342944819032517867*log(X)) +#endif +#if defined(HAVE_LOG2) && HAVE_LOG2==0 +# define log2(X) (1.442695040888963456*log(X)) +#endif + + +/* ** Implementation of SQL functions: ** ** ln(X) - natural logarithm @@ -124420,17 +127646,15 @@ static void logFunc( } ans = log(x)/b; }else{ - ans = log(x); switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ case 1: - /* Convert from natural logarithm to log base 10 */ - ans /= M_LN10; + ans = log10(x); break; case 2: - /* Convert from natural logarithm to log base 2 */ - ans /= M_LN2; + ans = log2(x); break; default: + ans = log(x); break; } } @@ -124499,6 +127723,7 @@ static void piFunc( sqlite3_value **argv ){ assert( argc==0 ); + (void)argv; sqlite3_result_double(context, M_PI); } @@ -124599,6 +127824,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(hex, 1, 0, 0, hexFunc ), + FUNCTION(unhex, 1, 0, 0, unhexFunc ), + FUNCTION(unhex, 2, 0, 0, unhexFunc ), INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), @@ -126029,22 +129256,22 @@ static Trigger *fkActionTrigger( if( action==OE_Restrict ){ int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - Token tFrom; - Token tDb; + SrcList *pSrc; Expr *pRaise; - tFrom.z = zFrom; - tFrom.n = nFrom; - tDb.z = db->aDb[iDb].zDbSName; - tDb.n = sqlite3Strlen30(tDb.z); - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ pRaise->affExpr = OE_Abort; } + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + assert( pSrc->nSrc==1 ); + pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); + pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom), + pSrc, pWhere, 0, 0, 0, 0, 0 ); @@ -126151,11 +129378,12 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pNext; /* Copy of pFKey->pNextFrom */ assert( IsOrdinaryTable(pTab) ); + assert( db!=0 ); for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ @@ -126259,43 +129487,68 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); - if( !pIdx->zColAff ){ - sqlite3OomFault(db); - return 0; + sqlite3OomFault(db); + return 0; + } + for(n=0; n<pIdx->nColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } - for(n=0; n<pIdx->nColumn; n++){ - i16 x = pIdx->aiColumn[n]; - char aff; - if( x>=0 ){ - aff = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - aff = SQLITE_AFF_INTEGER; - }else{ - assert( x==XN_EXPR ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff<SQLITE_AFF_BLOB ) aff = SQLITE_AFF_BLOB; + if( aff>SQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; + return pIdx->zColAff; +} +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx); + return pIdx->zColAff; +} + + +/* +** Compute an affinity string for a table. Space is obtained +** from sqlite3DbMalloc(). The caller is responsible for freeing +** the space when done. +*/ +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ + char *zColAff; + zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1); + if( zColAff ){ + int i, j; + for(i=j=0; i<pTab->nCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; } - if( aff<SQLITE_AFF_BLOB ) aff = SQLITE_AFF_BLOB; - if( aff>SQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; - pIdx->zColAff[n] = aff; } - pIdx->zColAff[n] = 0; + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); } - - return pIdx->zColAff; + return zColAff; } /* @@ -126339,7 +129592,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** Apply the type checking to that array of registers. */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i, j; + int i; char *zColAff; if( pTab->tabFlags & TF_Strict ){ if( iReg==0 ){ @@ -126348,7 +129601,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** OP_MakeRecord is found */ VdbeOp *pPrev; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); - pPrev = sqlite3VdbeGetOp(v, -1); + pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; @@ -126362,22 +129615,11 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } zColAff = pTab->zColAff; if( zColAff==0 ){ - sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + zColAff = sqlite3TableAffinityStr(0, pTab); if( !zColAff ){ - sqlite3OomFault(db); + sqlite3OomFault(sqlite3VdbeDb(v)); return; } - - for(i=j=0; i<pTab->nCol; i++){ - assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 ); - if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ - zColAff[j++] = pTab->aCol[i].affinity; - } - } - do{ - zColAff[j--] = 0; - }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); @@ -126386,7 +129628,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); }else{ - assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord + assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); sqlite3VdbeChangeP4(v, -1, zColAff, i); } @@ -126472,7 +129714,7 @@ SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( */ sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); if( (pTab->tabFlags & TF_HasStored)!=0 ){ - pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Affinity ){ /* Change the OP_Affinity argument to '@' (NONE) for all stored ** columns. '@' is the no-op affinity and those columns have not @@ -126971,7 +130213,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* Cannot insert into a read-only table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto insert_cleanup; } @@ -127378,7 +130620,12 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); } }else{ - sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + Expr *pX = pList->a[k].pExpr; + int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore); + if( y!=iRegStore ){ + sqlite3VdbeAddOp2(v, + ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore); + } } } @@ -127413,7 +130660,7 @@ SQLITE_PRIVATE void sqlite3Insert( } /* Copy the new data already generated. */ - assert( pTab->nNVCol>0 ); + assert( pTab->nNVCol>0 || pParse->nErr>0 ); sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); #ifndef SQLITE_OMIT_GENERATED_COLUMNS @@ -127515,7 +130762,9 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert ); - sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + if( db->flags & SQLITE_ForeignKeys ){ + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + } /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE ** constraints or (b) there are no triggers and this table is not a @@ -127599,7 +130848,7 @@ insert_cleanup: sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -127963,6 +131212,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pCol->zCnName); + testcase( zMsg==0 && db->mallocFailed==0 ); sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, iReg); sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); @@ -129826,9 +133076,9 @@ struct sqlite3_api_routines { const char *(*filename_journal)(const char*); const char *(*filename_wal)(const char*); /* Version 3.32.0 and later */ - char *(*create_filename)(const char*,const char*,const char*, + const char *(*create_filename)(const char*,const char*,const char*, int,const char**); - void (*free_filename)(char*); + void (*free_filename)(const char*); sqlite3_file *(*database_file_object)(const char*); /* Version 3.34.0 and later */ int (*txn_state)(sqlite3*,const char*); @@ -129852,6 +133102,10 @@ struct sqlite3_api_routines { unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, unsigned int); const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); + /* Version 3.41.0 and later */ + int (*is_interrupted)(sqlite3*); }; /* @@ -130176,6 +133430,10 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_serialize sqlite3_api->serialize #endif #define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding +/* Version 3.41.0 and later */ +#define sqlite3_is_interrupted sqlite3_api->is_interrupted #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -130688,7 +133946,11 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif - sqlite3_db_name + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_encoding, + /* Version 3.41.0 and later */ + sqlite3_is_interrupted }; /* True if x is the directory separator character @@ -130761,7 +134023,11 @@ static int sqlite3LoadExtension( /* tag-20210611-1. Some dlopen() implementations will segfault if given ** an oversize filename. Most filesystems have a pathname limit of 4K, ** so limit the extension filename length to about twice that. - ** https://sqlite.org/forum/forumpost/08a0d6d9bf */ + ** https://sqlite.org/forum/forumpost/08a0d6d9bf + ** + ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. + ** See https://sqlite.org/forum/forumpost/24083b579d. + */ if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; handle = sqlite3OsDlOpen(pVfs, zFile); @@ -130769,7 +134035,9 @@ static int sqlite3LoadExtension( for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){ char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]); if( zAltFile==0 ) return SQLITE_NOMEM_BKPT; - handle = sqlite3OsDlOpen(pVfs, zAltFile); + if( nMsg+strlen(azEndings[ii])+1<=SQLITE_MAX_PATHLEN ){ + handle = sqlite3OsDlOpen(pVfs, zAltFile); + } sqlite3_free(zAltFile); } #endif @@ -132705,6 +135973,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_TEMP_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_temp_directory); }else{ @@ -132714,6 +135983,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132731,6 +136001,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } @@ -132749,6 +136020,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_DATA_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_data_directory); }else{ @@ -132758,6 +136030,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132769,6 +136042,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } #endif @@ -133258,7 +136532,7 @@ SQLITE_PRIVATE void sqlite3Pragma( zDb = db->aDb[iDb].zDbSName; sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3TouchRegister(pParse, pTab->nCol+regRow); sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); assert( IsOrdinaryTable(pTab) ); @@ -133299,7 +136573,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** regRow..regRow+n. If any of the child key values are NULL, this ** row cannot cause an FK violation. Jump directly to addrOk in ** this case. */ - if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol; + sqlite3TouchRegister(pParse, regRow + pFK->nCol); for(j=0; j<pFK->nCol; j++){ int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); @@ -133428,6 +136702,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); + pParse->okConstFactor = 0; /* tag-20230327-1 */ /* Do an integrity check of the B-Tree ** @@ -133463,7 +136738,7 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3TouchRegister(pParse, 8+mxIdx); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ @@ -133482,15 +136757,24 @@ SQLITE_PRIVATE void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; - Index *pPrior = 0; + Index *pPrior = 0; /* Previous index */ int loopTop; int iDataCur, iIdxCur; int r1 = -1; - int bStrict; + int bStrict; /* True for a STRICT table */ + int r2; /* Previous key for WITHOUT ROWID tables */ + int mxCol; /* Maximum non-virtual column number */ if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; - pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + if( isQuick || HasRowid(pTab) ){ + pPk = 0; + r2 = 0; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol); + sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -133504,52 +136788,180 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + + /* Fetch the right-most column from the table. This will cause + ** the entire record header to be parsed and sanity checked. It + ** will also prepopulate the cursor column cache that is used + ** by the OP_IsType code, so it is a required step. + */ + assert( !IsVirtual(pTab) ); + if( HasRowid(pTab) ){ + mxCol = -1; + for(j=0; j<pTab->nCol; j++){ + if( (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)==0 ) mxCol++; + } + if( mxCol==pTab->iPKey ) mxCol--; + }else{ + /* COLFLAG_VIRTUAL columns are not included in the WITHOUT ROWID + ** PK index column-count, so there is no need to account for them + ** in this case. */ + mxCol = sqlite3PrimaryKeyIndex(pTab)->nColumn-1; + } + if( mxCol>=0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, mxCol, 3); + sqlite3VdbeTypeofColumn(v, 3); + } + if( !isQuick ){ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "(right-most column)")); + if( pPk ){ + /* Verify WITHOUT ROWID keys are in ascending order */ + int a1; + char *zErr; + a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, + "row not in PRIMARY KEY order for %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + sqlite3VdbeJumpHere(v, a1+1); + for(j=0; j<pPk->nKeyCol; j++){ + sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j); + } + } } - /* Verify that all NOT NULL columns really are NOT NULL. At the - ** same time verify the type of the content of STRICT tables */ + /* Verify datatypes for all columns: + ** + ** (1) NOT NULL columns may not contain a NULL + ** (2) Datatype must be exact for non-ANY columns in STRICT tables + ** (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. + ** (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be losslessly converted to numeric. + */ bStrict = (pTab->tabFlags & TF_Strict)!=0; for(j=0; j<pTab->nCol; j++){ char *zErr; - Column *pCol = pTab->aCol + j; - int doError, jmp2; + Column *pCol = pTab->aCol + j; /* The column to be checked */ + int labelError; /* Jump here to report an error */ + int labelOk; /* Jump here if all looks ok */ + int p1, p3, p4; /* Operands to the OP_IsType opcode */ + int doTypeCheck; /* Check datatypes (besides NOT NULL) */ + if( j==pTab->iPKey ) continue; - if( pCol->notNull==0 && !bStrict ) continue; - doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + if( bStrict ){ + doTypeCheck = pCol->eCType>COLTYPE_ANY; + }else{ + doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB; } + if( pCol->notNull==0 && !doTypeCheck ) continue; + + /* Compute the operands that will be needed for OP_IsType */ + p4 = SQLITE_NULL; + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + p1 = -1; + p3 = 3; + }else{ + if( pCol->iDflt ){ + sqlite3_value *pDfltValue = 0; + sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), + pCol->affinity, &pDfltValue); + if( pDfltValue ){ + p4 = sqlite3_value_type(pDfltValue); + sqlite3ValueFree(pDfltValue); + } + } + p1 = iDataCur; + if( !HasRowid(pTab) ){ + testcase( j!=sqlite3TableColumnToStorage(pTab, j) ); + p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j); + }else{ + p3 = sqlite3TableColumnToStorage(pTab,j); + testcase( p3!=j); + } + } + + labelError = sqlite3VdbeMakeLabel(pParse); + labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + /* (1) NOT NULL columns may not contain a NULL */ + int jmp3; + int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + VdbeCoverage(v); + if( p1<0 ){ + sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */ + jmp3 = jmp2; + }else{ + sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */ + /* OP_IsType does not detect NaN values in the database file + ** which should be treated as a NULL. So if the header type + ** is REAL, we have to load the actual data using OP_Column + ** to reliably determine if the value is a NULL. */ + sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); + VdbeCoverage(v); + } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - if( bStrict && pCol->eCType!=COLTYPE_ANY ){ - sqlite3VdbeGoto(v, doError); + if( doTypeCheck ){ + sqlite3VdbeGoto(v, labelError); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); }else{ - integrityCheckResultRow(v); + /* VDBE byte code will fall thru */ } - sqlite3VdbeJumpHere(v, jmp2); } - if( (pTab->tabFlags & TF_Strict)!=0 - && pCol->eCType!=COLTYPE_ANY - ){ - jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, - sqlite3StdTypeMap[pCol->eCType-1]); + if( bStrict && doTypeCheck ){ + /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/ + static unsigned char aStdTypeMask[] = { + 0x1f, /* ANY */ + 0x18, /* BLOB */ + 0x11, /* INT */ + 0x11, /* INTEGER */ + 0x13, /* REAL */ + 0x14 /* TEXT */ + }; + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) ); + sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]); VdbeCoverage(v); zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", sqlite3StdType[pCol->eCType-1], pTab->zName, pTab->aCol[j].zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeResolveLabel(v, doError); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){ + /* (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){ + /* (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be converted to numeric. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */ + VdbeCoverage(v); + if( p1>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + } + sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC); + sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "TEXT value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); } + sqlite3VdbeResolveLabel(v, labelError); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, labelOk); } /* Verify CHECK constraints */ if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ @@ -133578,7 +136990,8 @@ SQLITE_PRIVATE void sqlite3Pragma( if( !isQuick ){ /* Omit the remaining tests for quick_check */ /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2, jmp3, jmp4, jmp5; + int jmp2, jmp3, jmp4, jmp5, label6; + int kk; int ckUniq = sqlite3VdbeMakeLabel(pParse); if( pPk==pIdx ) continue; r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, @@ -133596,13 +137009,49 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp4 = integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, jmp2); + + /* The OP_IdxRowid opcode is an optimized version of OP_Column + ** that extracts the rowid off the end of the index record. + ** But it only works correctly if index record does not have + ** any extra bytes at the end. Verify that this is the case. */ + if( HasRowid(pTab) ){ + int jmp7; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); + jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 3, + "rowid not at end-of-record for row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " of index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp7); + } + + /* Any indexed columns with non-BINARY collations must still hold + ** the exact same text value as the table. */ + label6 = 0; + for(kk=0; kk<pIdx->nKeyCol; kk++){ + if( pIdx->azColl[kk]==sqlite3StrBINARY ) continue; + if( label6==0 ) label6 = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+j, kk, 3); + sqlite3VdbeAddOp3(v, OP_Ne, 3, label6, r1+kk); VdbeCoverage(v); + } + if( label6 ){ + int jmp6 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeResolveLabel(v, label6); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " values differ from index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp6); + } + /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL ** or (2) the next entry has a different key */ if( IsUniqueIndex(pIdx) ){ int uniqOk = sqlite3VdbeMakeLabel(pParse); int jmp6; - int kk; for(kk=0; kk<pIdx->nKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; assert( iCol!=XN_ROWID && iCol<pTab->nCol ); @@ -133637,6 +137086,9 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); } + if( pPk ){ + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); + } } } } @@ -133787,6 +137239,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aOp[1].p2 = iCookie; aOp[1].p3 = sqlite3Atoi(zRight); aOp[1].p5 = 1; + if( iCookie==BTREE_SCHEMA_VERSION && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow the use of PRAGMA schema_version=VALUE in defensive + ** mode. Change the OP_SetCookie opcode into a no-op. */ + aOp[1].opcode = OP_Noop; + } }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -134767,7 +138224,14 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl #else encoding = SQLITE_UTF8; #endif - sqlite3SetTextEncoding(db, encoding); + if( db->nVdbeActive>0 && encoding!=ENC(db) + && (db->mDbFlags & DBFLAG_Vacuum)==0 + ){ + rc = SQLITE_LOCKED; + goto initone_error_out; + }else{ + sqlite3SetTextEncoding(db, encoding); + } }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ @@ -134981,8 +138445,8 @@ static void schemaIsValid(Parse *pParse){ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; sqlite3ResetOneSchema(db, iDb); - pParse->rc = SQLITE_SCHEMA; } /* Close the transaction, if one was opened. */ @@ -135035,15 +138499,15 @@ SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ assert( db->pParse==pParse ); assert( pParse->nested==0 ); #ifndef SQLITE_OMIT_SHARED_CACHE - sqlite3DbFree(db, pParse->aTableLock); + if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock); #endif while( pParse->pCleanup ){ ParseCleanup *pCleanup = pParse->pCleanup; pParse->pCleanup = pCleanup->pNext; pCleanup->xCleanup(db, pCleanup->pPtr); - sqlite3DbFreeNN(db, pCleanup); + sqlite3DbNNFreeNN(db, pCleanup); } - sqlite3DbFree(db, pParse->aLabel); + if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel); if( pParse->pConstExpr ){ sqlite3ExprListDelete(db, pParse->pConstExpr); } @@ -135156,7 +138620,11 @@ static int sqlite3Prepare( sParse.db = db; sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - if( db->mallocFailed ) sqlite3ErrorMsg(&sParse, "out of memory"); + if( db->mallocFailed ){ + sqlite3ErrorMsg(&sParse, "out of memory"); + db->errCode = rc = SQLITE_NOMEM; + goto end_prepare; + } assert( sqlite3_mutex_held(db->mutex) ); /* For a long-term use prepared statement avoid the use of @@ -135166,7 +138634,7 @@ static int sqlite3Prepare( sParse.disableLookaside++; DisableLookaside; } - sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + sParse.prepFlags = prepFlags & 0xff; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that @@ -135207,7 +138675,9 @@ static int sqlite3Prepare( } } - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( db->pDisconnect ) sqlite3VtabUnlockList(db); +#endif if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; @@ -135591,6 +139061,10 @@ struct SortCtx { } aDefer[4]; #endif struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrPush; /* First instruction to push data into sorter */ + int addrPushEnd; /* Last instruction that pushes data into sorter */ +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -135602,6 +139076,7 @@ struct SortCtx { ** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ + assert( db!=0 ); while( p ){ Select *pPrior = p->pPrior; sqlite3ExprListDelete(db, p->pEList); @@ -135621,7 +139096,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3WindowUnlinkFromSelect(p->pWin); } #endif - if( bFree ) sqlite3DbFreeNN(db, p); + if( bFree ) sqlite3DbNNFreeNN(db, p); p = pPrior; bFree = 1; } @@ -136238,7 +139713,7 @@ static void pushOntoSorter( ** (2) All output columns are included in the sort record. In that ** case regData==regOrigData. ** (3) Some output columns are omitted from the sort record due to - ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the ** SQLITE_ECEL_OMITREF optimization, or due to the ** SortCtx.pDeferredRowLoad optimiation. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy @@ -136246,6 +139721,10 @@ static void pushOntoSorter( */ assert( nData==1 || regData==regOrigData || regOrigData==0 ); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPush = sqlite3VdbeCurrentAddr(v); +#endif + if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nPrefixReg; @@ -136346,6 +139825,9 @@ static void pushOntoSorter( sqlite3VdbeChangeP2(v, iSkip, pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushEnd = sqlite3VdbeCurrentAddr(v)-1; +#endif } /* @@ -137027,9 +140509,10 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ */ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ + assert( p->db!=0 ); assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p); } } @@ -137168,6 +140651,16 @@ static void generateSortTail( int bSeq; /* True if sorter record includes seq. no. */ int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif + + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") + ); + sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); + assert( addrBreak<0 ); if( pSort->labelBkOut ){ @@ -137214,7 +140707,7 @@ static void generateSortTail( if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -137222,6 +140715,9 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){ #ifdef SQLITE_ENABLE_SORTER_REFERENCES @@ -137277,6 +140773,7 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zEName)); } } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -137338,6 +140835,7 @@ static void generateSortTail( }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + sqlite3VdbeScanStatusRange(v, addrExplain, sqlite3VdbeCurrentAddr(v)-1, -1); if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); } @@ -137346,9 +140844,6 @@ static void generateSortTail( ** Return a pointer to a string containing the 'declaration type' of the ** expression pExpr. The string may be treated as static by the caller. ** -** Also try to estimate the size of the returned value and return that -** result in *pEstWidth. -** ** The declaration type is the exact datatype definition extracted from the ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression @@ -137612,7 +141107,7 @@ SQLITE_PRIVATE void sqlite3GenerateColumnNames( if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; - SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); pTabList = pSelect->pSrc; pEList = pSelect->pEList; assert( v!=0 ); @@ -137712,7 +141207,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; i<nCol && !db->mallocFailed; i++, pCol++){ + for(i=0, pCol=aCol; i<nCol && !pParse->nErr; i++, pCol++){ struct ExprList_item *pX = &pEList->a[i]; struct ExprList_item *pCollide; /* Get an appropriate name for the column @@ -137762,7 +141257,10 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + sqlite3ProgressCheck(pParse); + if( cnt>3 ){ + sqlite3_randomness(sizeof(cnt), &cnt); + } } pCol->zCnName = zName; pCol->hName = sqlite3StrIHash(zName); @@ -137775,71 +141273,104 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( } } sqlite3HashClear(&ht); - if( db->mallocFailed ){ + if( pParse->nErr ){ for(j=0; j<i; j++){ sqlite3DbFree(db, aCol[j].zCnName); } sqlite3DbFree(db, aCol); *paCol = 0; *pnCol = 0; - return SQLITE_NOMEM_BKPT; + return pParse->rc; } return SQLITE_OK; } /* -** Add type and collation information to a column list based on -** a SELECT statement. -** -** The column list presumably came from selectColumnNamesFromExprList(). -** The column list has only names, not types or collations. This -** routine goes through and adds the types and collations. +** pTab is a transient Table object that represents a subquery of some +** kind (maybe a parenthesized subquery in the FROM clause of a larger +** query, or a VIEW, or a CTE). This routine computes type information +** for that Table object based on the Select object that implements the +** subquery. For the purposes of this routine, "type infomation" means: ** -** This routine requires that all identifiers in the SELECT -** statement be resolved. +** * The datatype name, as it might appear in a CREATE TABLE statement +** * Which collating sequence to use for the column +** * The affinity of the column */ -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( - Parse *pParse, /* Parsing contexts */ - Table *pTab, /* Add column type information to this table */ - Select *pSelect, /* SELECT used to determine types and collations */ - char aff /* Default affinity for columns */ +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( + Parse *pParse, /* Parsing contexts */ + Table *pTab, /* Add column type information to this table */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity. */ ){ sqlite3 *db = pParse->db; - NameContext sNC; Column *pCol; CollSeq *pColl; - int i; + int i,j; Expr *p; struct ExprList_item *a; + NameContext sNC; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); - if( db->mallocFailed ) return; + assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); + assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); + if( db->mallocFailed || IN_RENAME_OBJECT ) return; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + a = pSelect->pEList->a; memset(&sNC, 0, sizeof(sNC)); sNC.pSrcList = pSelect->pSrc; - a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){ const char *zType; - i64 n, m; + i64 n; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; - zType = columnType(&sNC, p, 0, 0, 0); /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + if( pCol->affinity<=SQLITE_AFF_NONE ){ + pCol->affinity = aff; + } + if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ + int m = 0; + Select *pS2; + for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + }else + if( pCol->affinity>=SQLITE_AFF_NUMERIC && (m&0x02)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ + pCol->affinity = SQLITE_AFF_FLEXNUM; + } + } + zType = columnType(&sNC, p, 0, 0, 0); + if( zType==0 || pCol->affinity!=sqlite3AffinityType(zType, 0) ){ + if( pCol->affinity==SQLITE_AFF_NUMERIC + || pCol->affinity==SQLITE_AFF_FLEXNUM + ){ + zType = "NUM"; + }else{ + zType = 0; + for(j=1; j<SQLITE_N_STDTYPE; j++){ + if( sqlite3StdTypeAffinity[j]==pCol->affinity ){ + zType = sqlite3StdType[j]; + break; + } + } + } + } if( zType ){ - m = sqlite3Strlen30(zType); + i64 m = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ memcpy(&pCol->zCnName[n+1], zType, m+1); pCol->colFlags |= COLFLAG_HASTYPE; - }else{ - testcase( pCol->colFlags & COLFLAG_HASTYPE ); - pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); } } - if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl ){ assert( pTab->pIndex==0 ); @@ -137873,7 +141404,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, c pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); + sqlite3SubqueryColumnTypes(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -138398,7 +141929,7 @@ static int multiSelect( pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); rc = sqlite3Select(pParse, pPrior, &dest); pPrior->pLimit = 0; if( rc ){ @@ -138416,7 +141947,7 @@ static int multiSelect( } } ExplainQueryPlan((pParse, 1, "UNION ALL")); - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -138469,7 +142000,7 @@ static int multiSelect( */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; @@ -138489,7 +142020,7 @@ static int multiSelect( uniondest.eDest = op; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); @@ -138550,7 +142081,7 @@ static int multiSelect( /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; @@ -138567,7 +142098,7 @@ static int multiSelect( intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -139214,10 +142745,11 @@ static int multiSelectOrderBy( */ sqlite3VdbeResolveLabel(v, labelEnd); - /* Reassembly the compound query so that it will be freed correctly - ** by the calling function */ + /* Make arrangements to free the 2nd and subsequent arms of the compound + ** after the parse has finished */ if( pSplit->pPrior ){ - sqlite3SelectDelete(db, pSplit->pPrior); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior); } pSplit->pPrior = pPrior; pPrior->pNext = pSplit; @@ -139247,7 +142779,7 @@ static int multiSelectOrderBy( ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. ** -** Suppose the original expression integer constant. Even though the table +** Suppose the original expression is an integer constant. Even though the table ** has the nullRow flag set, because the expression is an integer constant, ** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode ** that checks to see if the nullRow flag is set on the table. If the nullRow @@ -139273,6 +142805,7 @@ typedef struct SubstContext { int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; /* Forward Declarations */ @@ -139314,19 +142847,23 @@ static Expr *substExpr( #endif { Expr *pNew; - Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + int iColumn = pExpr->iColumn; + Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; Expr ifNullRow; - assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr ); + assert( pSubst->pEList!=0 && iColumn<pSubst->pEList->nExpr ); assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ sqlite3 *db = pSubst->pParse->db; - if( pSubst->isOuterJoin && pCopy->op!=TK_COLUMN ){ + if( pSubst->isOuterJoin + && (pCopy->op!=TK_COLUMN || pCopy->iTable!=pSubst->iNewTable) + ){ memset(&ifNullRow, 0, sizeof(ifNullRow)); ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.iColumn = -99; ifNullRow.flags = EP_IfNullRow; pCopy = &ifNullRow; } @@ -139353,11 +142890,16 @@ static Expr *substExpr( /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ - CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, - (pColl ? pColl->zName : "BINARY") + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } } ExprClearProperty(pExpr, EP_Collate); } @@ -139550,6 +143092,46 @@ static void renumberCursors( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; ii<pList->nExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -139594,7 +143176,8 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (3c) the outer query may not be an aggregate. +** (**) Was: "The outer query may not have a GROUP BY." This case +** is now managed correctly ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -139651,6 +143234,8 @@ static void renumberCursors( ** (17g) either the subquery is the first element of the outer ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) +** (17h) The corresponding result set expressions in all arms of the +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -139702,19 +143287,13 @@ static void renumberCursors( ** See also (3) for restrictions on LEFT JOIN. ** ** (27) The subquery may not contain a FULL or RIGHT JOIN unless it -** is the first element of the parent query. This must be the -** the case if: -** (27a) the subquery is not compound query, and +** is the first element of the parent query. Two subcases: +** (27a) the subquery is not a compound query. ** (27b) the subquery is a compound query and the RIGHT JOIN occurs ** in any arm of the compound query. (See also (17g).) ** ** (28) The subquery is not a MATERIALIZED CTE. ** -** (29) Either the subquery is not the right-hand operand of a join with an -** ON or USING clause nor the right-hand operand of a NATURAL JOIN, or -** the right-most table within the FROM clause of the subquery -** is not part of an outer join. -** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -139806,16 +143385,10 @@ static int flattenSubquery( ** ** which is not at all the same thing. ** - ** If the subquery is the right operand of a LEFT JOIN, then the outer - ** query cannot be an aggregate. (3c) This is an artifact of the way - ** aggregates are processed - there is no mechanism to determine if - ** the LEFT JOIN table should be all-NULL. - ** ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || isAgg /* (3c) */ || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ @@ -139824,15 +143397,6 @@ static int flattenSubquery( } isOuterJoin = 1; } -#ifdef SQLITE_EXTRA_IFNULLROW - else if( iFrom>0 && !isAgg ){ - /* Setting isOuterJoin to -1 causes OP_IfNullRow opcodes to be generated for - ** every reference to any result column from subquery in a join, even - ** though they are not necessary. This will stress-test the OP_IfNullRow - ** opcode. */ - isOuterJoin = -1; - } -#endif assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ @@ -139842,41 +143406,13 @@ static int flattenSubquery( return 0; /* (28) */ } - /* Restriction (29): - ** - ** We do not want two constraints on the same term of the flattened - ** query where one constraint has EP_InnerON and the other is EP_OuterON. - ** To prevent this, one or the other of the following conditions must be - ** false: - ** - ** (29a) The right-most entry in the FROM clause of the subquery - ** must not be part of an outer join. - ** - ** (29b) The subquery itself must not be the right operand of a - ** NATURAL join or a join that as an ON or USING clause. - ** - ** These conditions are sufficient to keep an EP_OuterON from being - ** flattened into an EP_InnerON. Restrictions (3a) and (27a) prevent - ** an EP_InnerON from being flattened into an EP_OuterON. - */ - if( pSubSrc->nSrc>=2 - && (pSubSrc->a[pSubSrc->nSrc-1].fg.jointype & JT_OUTER)!=0 - ){ - if( (pSubitem->fg.jointype & JT_NATURAL)!=0 - || pSubitem->fg.isUsing - || NEVER(pSubitem->u3.pOn!=0) /* ON clause already shifted into WHERE */ - || pSubitem->fg.isOn - ){ - return 0; - } - } - /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries ** that make up the compound SELECT are allowed to be aggregate or distinct ** queries. */ if( pSub->pPrior ){ + int ii; if( pSub->pOrderBy ){ return 0; /* Restriction (20) */ } @@ -139909,7 +143445,6 @@ static int flattenSubquery( /* Restriction (18). */ if( p->pOrderBy ){ - int ii; for(ii=0; ii<p->pOrderBy->nExpr; ii++){ if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } @@ -139918,6 +143453,9 @@ static int flattenSubquery( /* Restriction (23) */ if( (p->selFlags & SF_Recursive) ) return 0; + /* Restriction (17h) */ + if( compoundHasDifferentAffinities(pSub) ) return 0; + if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; @@ -139927,7 +143465,7 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", pSub->selId, pSub, iFrom)); /* Authorize the subquery */ @@ -140006,7 +143544,7 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p,("compound-subquery flattener" + TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } assert( pSubitem->pSelect==0 ); @@ -140151,6 +143689,7 @@ static int flattenSubquery( x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } @@ -140170,7 +143709,7 @@ static int flattenSubquery( pSub->pLimit = 0; } - /* Recompute the SrcList_item.colUsed masks for the flattened + /* Recompute the SrcItem.colUsed masks for the flattened ** tables. */ for(i=0; i<nSubSrc; i++){ recomputeColumnsUsed(pParent, &pSrc->a[i+iFrom]); @@ -140185,8 +143724,8 @@ static int flattenSubquery( sqlite3SelectDelete(db, pSub1); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140560,6 +144099,29 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. +** +** (9) All three of the following are true: +** +** (9a) The WHERE clause expression originates in the ON or USING clause +** of a join (either an INNER or an OUTER join), and +** +** (9b) The subquery is to the right of the ON/USING clause +** +** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING +** clause and the subquery. +** +** Without this restriction, the push-down optimization might move +** the ON/USING filter expression from the left side of a RIGHT JOIN +** over to the right side, which leads to incorrect answers. See +** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** +** (10) The inner query is not the right-hand table of a RIGHT JOIN. +** +** (11) The subquery is not a VALUES clause +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -140567,24 +144129,56 @@ static int pushDownWhereTerms( Parse *pParse, /* Parse context (for malloc() and error reporting) */ Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ Expr *pWhere, /* The WHERE clause of the outer query */ - SrcItem *pSrc /* The subquery term of the outer FROM clause */ + SrcList *pSrcList, /* The complete from clause of the outer query */ + int iSrc /* Which FROM clause term to try to push into */ ){ Expr *pNew; + SrcItem *pSrc; /* The subquery FROM term into which WHERE is pushed */ int nChng = 0; + pSrc = &pSrcList->a[iSrc]; if( pWhere==0 ) return 0; - if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; - if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ){ + return 0; /* restrictions (2) and (11) */ + } + if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ){ + return 0; /* restrictions (10) */ + } -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + u8 op = pSel->op; + assert( op==TK_ALL || op==TK_SELECT + || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif + } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; ii<pList->nExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make @@ -140603,11 +144197,28 @@ static int pushDownWhereTerms( return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrcList, iSrc); pWhere = pWhere->pLeft; } -#if 0 /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */ +#if 0 /* These checks now done by sqlite3ExprIsSingleTableConstraint() */ + if( ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) /* (9a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (9c) */ + ){ + int jj; + for(jj=0; jj<iSrc; jj++){ + if( pWhere->w.iJoin==pSrcList->a[jj].iCursor ){ + /* If we reach this point, both (9a) and (9b) are satisfied. + ** The following loop checks (9c): + */ + for(jj++; jj<iSrc; jj++){ + if( (pSrcList->a[jj].fg.jointype & JT_RIGHT)!=0 ){ + return 0; /* restriction (9) */ + } + } + } + } + } if( isLeftJoin && (ExprHasProperty(pWhere,EP_OuterON)==0 || pWhere->w.iJoin!=iCursor) @@ -140621,7 +144232,7 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){ + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -140633,6 +144244,7 @@ static int pushDownWhereTerms( x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ @@ -140655,6 +144267,78 @@ static int pushDownWhereTerms( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ + + assert( pItem!=0 ); + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ + return 0; + } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pSub->selFlags & SF_Distinct ); + testcase( pSub->selFlags & SF_Aggregate ); + return 0; + } + for(pX=pSub; pX; pX=pX->pPrior){ + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } +#endif + } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; j<pList->nExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + nCol = pTab->nCol; + for(j=0; j<nCol; j++){ + Bitmask m = j<BMS-1 ? MASKBIT(j) : TOPBIT; + if( (m & colUsed)!=0 ) continue; + for(pX=pSub; pX; pX=pX->pPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + ExprClearProperty(pY, EP_Skip|EP_Unlikely); + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + +/* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. ** @@ -140736,6 +144420,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } @@ -141044,9 +144729,6 @@ static int resolveFromTermToCte( pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; - if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){ - pCteUse->eM10d = M10d_Yes; - } /* Check if this is a recursive CTE. */ pRecTerm = pSel = pFrom->pSelect; @@ -141156,9 +144838,9 @@ SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ #endif /* -** The SrcList_item structure passed as the second argument represents a +** The SrcItem structure passed as the second argument represents a ** sub-query in the FROM clause of a SELECT statement. This function -** allocates and populates the SrcList_item.pTab object. If successful, +** allocates and populates the SrcItem.pTab object. If successful, ** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, ** SQLITE_NOMEM. */ @@ -141586,8 +145268,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } } #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n")); + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -141638,14 +145320,14 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ ** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() ** interface. ** -** For each FROM-clause subquery, add Column.zType and Column.zColl -** information to the Table structure that represents the result set -** of that subquery. +** For each FROM-clause subquery, add Column.zType, Column.zColl, and +** Column.affinity information to the Table structure that represents +** the result set of that subquery. ** ** The Table structure that represents the result set was constructed -** by selectExpander() but the type and collation information was omitted -** at that point because identifiers had not yet been resolved. This -** routine is called after identifier resolution. +** by selectExpander() but the type and collation and affinity information +** was omitted at that point because identifiers had not yet been resolved. +** This routine is called after identifier resolution. */ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; @@ -141665,9 +145347,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, - SQLITE_AFF_NONE); + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -141722,6 +145402,178 @@ SQLITE_PRIVATE void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; ii<pAggInfo->nColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; ii<pAggInfo->nFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->iFirstReg+pAggInfo->nColumn+ii); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; i<pAggInfo->nFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ + assert( pAggInfo->iFirstReg==0 ); + assert( pSelect!=0 ); + assert( pSelect->pGroupBy!=0 ); + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ + int mx = pSelect->pGroupBy->nExpr - 1; + int j, k; + for(j=0; j<pAggInfo->nColumn; j++){ + k = pAggInfo->aCol[j].iSorterColumn; + if( k>mx ) mx = k; + } + pAggInfo->nSortingColumn = mx+1; + } + analyzeAggFuncArgs(pAggInfo, pNC); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + IndexedExpr *pIEpr; + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); +#endif +} + +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue; + assert( pExpr->iAgg>=0 ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + ExprClearProperty(pExpr, EP_Skip|EP_Collate); + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; i<pAggInfo->nFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; +} + /* ** Reset the aggregate accumulator. ** @@ -141735,24 +145587,13 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; if( pParse->nErr ) return; -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); - for(i=0; i<pAggInfo->nColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); - } - for(i=0; i<pAggInfo->nFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; i<pAggInfo->nFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -141784,15 +145625,16 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator @@ -141812,6 +145654,8 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){ int nArg; @@ -141872,7 +145716,7 @@ static void updateAccumulator( if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -141887,7 +145731,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -141983,26 +145827,31 @@ static void havingToWhere(Parse *pParse, Select *p){ sqlite3WalkExpr(&sWalker, p->pHaving); #if TREETRACE_ENABLED if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ - SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif } /* -** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, -** then return 0. +** Check to see if the pThis entry of pTabList is a self-join of another view. +** Search FROM-clause entries in the range of iFirst..iEnd, including iFirst +** but stopping before iEnd. +** +** If pThis is a self-join, then return the SrcItem for the first other +** instance of that view found. If pThis is not a self-join then return 0. */ static SrcItem *isSelfJoinView( SrcList *pTabList, /* Search for self-joins in this FROM clause */ - SrcItem *pThis /* Search for prior reference to this subquery */ + SrcItem *pThis, /* Search for prior reference to this subquery */ + int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; assert( pThis->pSelect!=0 ); if( pThis->pSelect->selFlags & SF_PushDown ) return 0; - for(pItem = pTabList->a; pItem<pThis; pItem++){ + while( iFirst<iEnd ){ Select *pS1; + pItem = &pTabList->a[iFirst++]; if( pItem->pSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; @@ -142035,7 +145884,6 @@ static void agginfoFree(sqlite3 *db, AggInfo *p){ sqlite3DbFreeNN(db, p); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION /* ** Attempt to transform a query of the form ** @@ -142063,7 +145911,9 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; + if( p->pHaving ) return 0; if( p->pGroupBy ) return 0; + if( p->pOrderBy ) return 0; pExpr = p->pEList->a[0].pExpr; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ assert( ExprUseUToken(pExpr) ); @@ -142071,15 +145921,18 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ assert( ExprUseXList(pExpr) ); if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ + if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ pSub = p->pSrc->a[0].pSelect; if( pSub==0 ) return 0; /* The FROM is a subquery */ - if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */ + if( pSub->pPrior==0 ) return 0; /* Must be a compound */ + if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ - pSub = pSub->pPrior; /* Repeat over compound */ + assert( pSub->pHaving==0 ); /* Due to the previous */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -142115,14 +145968,13 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ p->selFlags &= ~SF_Aggregate; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif return 1; } -#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ /* ** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same @@ -142148,6 +146000,68 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ } /* +** Return TRUE (non-zero) if the i-th entry in the pTabList SrcList can +** be implemented as a co-routine. The i-th entry is guaranteed to be +** a subquery. +** +** The subquery is implemented as a co-routine if all of the following are +** true: +** +** (1) The subquery will likely be implemented in the outer loop of +** the query. This will be the case if any one of the following +** conditions hold: +** (a) The subquery is the only term in the FROM clause +** (b) The subquery is the left-most term and a CROSS JOIN or similar +** requires it to be the outer loop +** (c) All of the following are true: +** (i) The subquery is the left-most subquery in the FROM clause +** (ii) There is nothing that would prevent the subquery from +** being used as the outer loop if the sqlite3WhereBegin() +** routine nominates it to that position. +** (iii) The query is not a UPDATE ... FROM +** (2) The subquery is not a CTE that should be materialized because +** (a) the AS MATERIALIZED keyword is used, or +** (b) the CTE is used multiple times and does not have the +** NOT MATERIALIZED keyword +** (3) The subquery is not part of a left operand for a RIGHT JOIN +** (4) The SQLITE_Coroutine optimization disable flag is not set +** (5) The subquery is not self-joined +*/ +static int fromClauseTermCanBeCoroutine( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* FROM clause */ + int i, /* Which term of the FROM clause holds the subquery */ + int selFlags /* Flags on the SELECT statement */ +){ + SrcItem *pItem = &pTabList->a[i]; + if( pItem->fg.isCte ){ + const CteUse *pCteUse = pItem->u2.pCteUse; + if( pCteUse->eM10d==M10d_Yes ) return 0; /* (2a) */ + if( pCteUse->nUse>=2 && pCteUse->eM10d!=M10d_No ) return 0; /* (2b) */ + } + if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0; /* (3) */ + if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4) */ + if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){ + return 0; /* (5) */ + } + if( i==0 ){ + if( pTabList->nSrc==1 ) return 1; /* (1a) */ + if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1; /* (1b) */ + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + return 1; + } + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + while( 1 /*exit-by-break*/ ){ + if( pItem->fg.jointype & (JT_OUTER|JT_CROSS) ) return 0; /* (1c-ii) */ + if( i==0 ) break; + i--; + pItem--; + if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + } + return 1; +} + +/* ** Generate code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. @@ -142192,8 +146106,8 @@ SQLITE_PRIVATE int sqlite3Select( assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); - if( sqlite3TreeTrace & 0x10100 ){ + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", __FILE__, __LINE__); @@ -142213,8 +146127,8 @@ SQLITE_PRIVATE int sqlite3Select( /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); - if( sqlite3TreeTrace & 0x100 ){ + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif @@ -142234,8 +146148,8 @@ SQLITE_PRIVATE int sqlite3Select( assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x104 ){ - SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -142276,8 +146190,8 @@ SQLITE_PRIVATE int sqlite3Select( goto select_end; } #if TREETRACE_ENABLED - if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){ - SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -142308,7 +146222,7 @@ SQLITE_PRIVATE int sqlite3Select( && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x1000,pParse,p, ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); assert( pItem->iCursor>=0 ); @@ -142316,7 +146230,7 @@ SQLITE_PRIVATE int sqlite3Select( pTabList->a[0].fg.jointype & JT_LTORJ); } - /* No futher action if this term of the FROM clause is no a subquery */ + /* No futher action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -142364,7 +146278,7 @@ SQLITE_PRIVATE int sqlite3Select( && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); sqlite3ParserAddCleanup(pParse, (void(*)(sqlite3*,void*))sqlite3ExprListDelete, @@ -142419,8 +146333,8 @@ SQLITE_PRIVATE int sqlite3Select( if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -142440,24 +146354,21 @@ SQLITE_PRIVATE int sqlite3Select( && propagateConstants(pParse, p) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif }else{ - SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ if( db->mallocFailed ) goto select_end; - pEList = p->pEList; pTabList = p->pSrc; } -#endif /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables @@ -142516,39 +146427,42 @@ SQLITE_PRIVATE int sqlite3Select( if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) - && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + } + + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif } zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if all of the following are - ** true: - ** - ** (1) the subquery is guaranteed to be the outer loop (so that - ** it does not need to be computed more than once), and - ** (2) the subquery is not a CTE that should be materialized - ** (3) the subquery is not part of a left operand for a RIGHT JOIN */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) /* (1) */ - && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ - && (pTabList->a[0].fg.jointype & JT_LTORJ)==0 /* (3) */ - ){ + if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ @@ -142579,7 +146493,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "%!S", pItem)); } pSub->nSelectRow = pCteUse->nRowEst; - }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){ + }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in ** this same FROM clause. Reuse it. */ if( pPrior->addrFillSub ){ @@ -142593,6 +146507,9 @@ SQLITE_PRIVATE int sqlite3Select( ** the same view can reuse the materialization. */ int topAddr; int onceAddr = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); @@ -142608,12 +146525,14 @@ SQLITE_PRIVATE int sqlite3Select( VdbeNoopComment((v, "materialize %!S", pItem)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + + ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ @@ -142639,8 +146558,8 @@ SQLITE_PRIVATE int sqlite3Select( sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -142676,8 +146595,8 @@ SQLITE_PRIVATE int sqlite3Select( sDistinct.isTnct = 2; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -142729,7 +146648,7 @@ SQLITE_PRIVATE int sqlite3Select( if( (p->selFlags & SF_FixedLimit)==0 ){ p->nSelectRow = 320; /* 4 billion rows */ } - computeLimitRegisters(pParse, p, iEnd); + if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; @@ -142763,7 +146682,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Begin the database scan. */ - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; @@ -142780,7 +146699,7 @@ SQLITE_PRIVATE int sqlite3Select( sSort.pOrderBy = 0; } } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -142819,7 +146738,7 @@ SQLITE_PRIVATE int sqlite3Select( /* End the database scan loop. */ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ @@ -142900,12 +146819,14 @@ SQLITE_PRIVATE int sqlite3Select( goto select_end; } pAggInfo->selId = p->selId; +#ifdef SQLITE_DEBUG + pAggInfo->pSelect = p; +#endif memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -142926,40 +146847,17 @@ SQLITE_PRIVATE int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; i<pAggInfo->nFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } - pAggInfo->mxReg = pParse->nMem; + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - int ii; - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } - for(ii=0; ii<pAggInfo->nColumn; ii++){ - sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", - ii, pAggInfo->aCol[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; ii<pAggInfo->nFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } + printAggInfo(pAggInfo); } #endif @@ -143028,17 +146926,21 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); + } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be @@ -143073,21 +146975,21 @@ SQLITE_PRIVATE int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; + pAggInfo->directMode = 1; for(i=0; i<pAggInfo->nColumn; i++){ struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ - int r1 = j + regBase; - sqlite3ExprCodeGetColumnOfTable(v, - pCol->pTab, pCol->iTable, pCol->iColumn, r1); + sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase); j++; } } + pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); @@ -143097,6 +146999,23 @@ SQLITE_PRIVATE int sqlite3Select( pAggInfo->useSortingIdx = 1; } + /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } + /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. @@ -143165,7 +147084,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } @@ -143275,7 +147194,8 @@ SQLITE_PRIVATE int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + assignAggregateRegisters(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ @@ -143311,6 +147231,7 @@ SQLITE_PRIVATE int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row @@ -143327,13 +147248,13 @@ SQLITE_PRIVATE int sqlite3Select( assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, - pDistinct, 0, minMaxFlag|distFlag, 0); + pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, regAcc, pAggInfo, eDist); if( eDist!=WHERE_DISTINCT_NOOP ){ @@ -143347,7 +147268,7 @@ SQLITE_PRIVATE int sqlite3Select( if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } @@ -143369,8 +147290,6 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -143394,7 +147313,7 @@ select_end: if( pAggInfo && !db->mallocFailed ){ for(i=0; i<pAggInfo->nColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } @@ -143408,8 +147327,8 @@ select_end: #endif #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -143683,7 +147602,7 @@ SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; @@ -143824,6 +147743,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( }else{ assert( !db->init.busy ); sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) } goto trigger_cleanup; } @@ -143973,6 +147893,23 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Vdbe *v; char *z; + /* If this is a new CREATE TABLE statement, and if shadow tables + ** are read-only, and the trigger makes a change to a shadow table, + ** then raise an error - do not allow the trigger to be created. */ + if( sqlite3ReadOnlyShadowTables(db) ){ + TriggerStep *pStep; + for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) + ){ + sqlite3ErrorMsg(pParse, + "trigger \"%s\" may not write to shadow table \"%s\"", + pTrig->zName, pStep->zTarget); + goto triggerfinish_cleanup; + } + } + } + /* Make an entry in the sqlite_schema table */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto triggerfinish_cleanup; @@ -144588,7 +148525,7 @@ static void codeReturningTrigger( } sqlite3ExprListDelete(db, sSelect.pEList); pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); - if( !db->mallocFailed ){ + if( pParse->nErr==0 ){ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); if( pReturning->nRetCol==0 ){ @@ -144796,7 +148733,7 @@ static TriggerPrg *codeRowTrigger( sSubParse.zAuthContext = pTrigger->zName; sSubParse.eTriggerOp = pTrigger->op; sSubParse.nQueryLoop = pParse->nQueryLoop; - sSubParse.disableVtab = pParse->disableVtab; + sSubParse.prepFlags = pParse->prepFlags; v = sqlite3GetVdbe(&sSubParse); if( v ){ @@ -145057,6 +148994,9 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ if( p->op==op && (tr_tm&p->tr_tm) @@ -145142,11 +149082,14 @@ static void updateVirtualTable( ** it has been converted into REAL. */ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + Column *pCol; assert( pTab!=0 ); - if( !IsView(pTab) ){ + assert( pTab->nCol>i ); + pCol = &pTab->aCol[i]; + if( pCol->iDflt ){ sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; + assert( !IsView(pTab) ); VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); assert( i<pTab->nCol ); sqlite3ValueFromExpr(sqlite3VdbeDb(v), @@ -145157,7 +149100,7 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ } } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif @@ -145343,7 +149286,8 @@ static void updateFromSelect( } } pSelect = sqlite3SelectNew(pParse, pList, - pSrc, pWhere2, pGrp, 0, pOrderBy2, SF_UFSrcCheck|SF_IncludeHidden, pLimit2 + pSrc, pWhere2, pGrp, 0, pOrderBy2, + SF_UFSrcCheck|SF_IncludeHidden|SF_UpdateFrom, pLimit2 ); if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; sqlite3SelectDestInit(&dest, eDest, iEph); @@ -145487,7 +149431,7 @@ SQLITE_PRIVATE void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } @@ -145806,12 +149750,22 @@ SQLITE_PRIVATE void sqlite3Update( /* Begin the database scan. ** ** Do not consider a single-pass strategy for a multi-row update if - ** there are any triggers or foreign keys to process, or rows may - ** be deleted as a result of REPLACE conflict handling. Any of these - ** things might disturb a cursor being used to scan through the table - ** or index, causing a single-pass approach to malfunction. */ + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ flags = WHERE_ONEPASS_DESIRED; - if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (sNC.ncFlags & NC_Subquery)==0 + ){ flags |= WHERE_ONEPASS_MULTIROW; } pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); @@ -146597,6 +150551,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( if( pIdx->aiColumn[ii]==XN_EXPR ){ assert( pIdx->aColExpr!=0 ); assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); pExpr = pIdx->aColExpr->a[ii].pExpr; if( pExpr->op!=TK_COLLATE ){ sCol[0].pLeft = pExpr; @@ -146910,6 +150865,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ + u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -146981,12 +150937,17 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( goto end_of_vacuum; } db->mDbFlags |= DBFLAG_VacuumInto; + + /* For a VACUUM INTO, the pager-flags are set to the same values as + ** they are for the database being vacuumed, except that PAGER_CACHESPILL + ** is always set. */ + pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); } nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); - sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, @@ -147370,10 +151331,10 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ pVTab->nRef--; if( pVTab->nRef==0 ){ sqlite3_vtab *p = pVTab->pVtab; - sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); if( p ){ p->pModule->xDisconnect(p); } + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); sqlite3DbFree(db, pVTab); } } @@ -147499,7 +151460,8 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ */ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ assert( IsVirtual(p) ); - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + assert( db!=0 ); + if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->u.vtab.azArg ){ int i; for(i=0; i<p->u.vtab.nArg; i++){ @@ -147768,7 +151730,9 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); @@ -148258,7 +152222,10 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ + u64 savedFlags = (db->flags & SQLITE_Defensive); + db->flags &= ~(u64)SQLITE_Defensive; rc = xMethod(pVTab->pVtab, iSavepoint); + db->flags |= savedFlags; } sqlite3VtabUnlock(pVTab); } @@ -148299,7 +152266,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; assert( ExprUseYTab(pExpr) ); pTab = pExpr->y.pTab; - if( pTab==0 ) return pDef; + if( NEVER(pTab==0) ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); @@ -148487,6 +152454,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; break; } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } default: { rc = SQLITE_MISUSE_BKPT; break; @@ -148906,7 +152877,7 @@ struct WhereAndInfo { ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** ** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** SrcItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping @@ -148978,20 +152949,6 @@ struct WhereLoopBuilder { #endif /* -** Each instance of this object records a change to a single node -** in an expression tree to cause that node to point to a column -** of an index rather than an expression or a virtual column. All -** such transformations need to be undone at the end of WHERE clause -** processing. -*/ -typedef struct WhereExprMod WhereExprMod; -struct WhereExprMod { - WhereExprMod *pNext; /* Next translation on a list of them all */ - Expr *pExpr; /* The Expr node that was transformed */ - Expr orig; /* Original value of the Expr node */ -}; - -/* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second ** half does the tail of the WHERE loop. An instance of @@ -149006,10 +152963,10 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ +#if WHERETRACE_ENABLED Expr *pWhere; /* The complete WHERE clause */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - Select *pLimit; /* Used to access LIMIT expr/registers for vtabs */ #endif + Select *pSelect; /* The entire SELECT statement containing WHERE */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ @@ -149028,7 +152985,6 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - WhereExprMod *pExprMods; /* Expression modifications */ WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ @@ -149176,6 +153132,8 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ +#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ @@ -149273,9 +153231,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -149287,8 +153245,8 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( sqlite3ParseToplevel(pParse)->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { SrcItem *pItem = &pTabList->a[pLevel->iFrom]; @@ -149432,6 +153390,8 @@ SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( zMsg = sqlite3StrAccumFinish(&str); ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + + sqlite3VdbeScanStatus(v, sqlite3VdbeCurrentAddr(v)-1, 0, 0, 0, 0); return ret; } #endif /* SQLITE_OMIT_EXPLAIN */ @@ -149452,16 +153412,31 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } + } } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); } #endif @@ -149521,7 +153496,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ pTerm->wtFlags |= TERM_CODED; } #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("DISABLE-"); sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); } @@ -149636,68 +153611,75 @@ static Expr *removeUnindexableInClauseTerms( Expr *pX /* The IN expression to be reduced */ ){ sqlite3 *db = pParse->db; + Select *pSelect; /* Pointer to the SELECT on the RHS */ Expr *pNew; pNew = sqlite3ExprDup(db, pX, 0); if( db->mallocFailed==0 ){ - ExprList *pOrigRhs; /* Original unmodified RHS */ - ExprList *pOrigLhs; /* Original unmodified LHS */ - ExprList *pRhs = 0; /* New RHS after modifications */ - ExprList *pLhs = 0; /* New LHS after mods */ - int i; /* Loop counter */ - Select *pSelect; /* Pointer to the SELECT on the RHS */ - - assert( ExprUseXSelect(pNew) ); - pOrigRhs = pNew->x.pSelect->pEList; - assert( pNew->pLeft!=0 ); - assert( ExprUseXList(pNew->pLeft) ); - pOrigLhs = pNew->pLeft->x.pList; - for(i=iEq; i<pLoop->nLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iField; - assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); - iField = pLoop->aLTerm[i]->u.x.iField - 1; - if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ - pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); - pOrigRhs->a[iField].pExpr = 0; - assert( pOrigLhs->a[iField].pExpr!=0 ); - pLhs = sqlite3ExprListAppend(pParse, pLhs, pOrigLhs->a[iField].pExpr); - pOrigLhs->a[iField].pExpr = 0; - } - } - sqlite3ExprListDelete(db, pOrigRhs); - sqlite3ExprListDelete(db, pOrigLhs); - pNew->pLeft->x.pList = pLhs; - pNew->x.pSelect->pEList = pRhs; - if( pLhs && pLhs->nExpr==1 ){ - /* Take care here not to generate a TK_VECTOR containing only a - ** single value. Since the parser never creates such a vector, some - ** of the subroutines do not handle this case. */ - Expr *p = pLhs->a[0].pExpr; - pLhs->a[0].pExpr = 0; - sqlite3ExprDelete(db, pNew->pLeft); - pNew->pLeft = p; - } - pSelect = pNew->x.pSelect; - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; i<pOrderBy->nExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; + for(pSelect=pNew->x.pSelect; pSelect; pSelect=pSelect->pPrior){ + ExprList *pOrigRhs; /* Original unmodified RHS */ + ExprList *pOrigLhs = 0; /* Original unmodified LHS */ + ExprList *pRhs = 0; /* New RHS after modifications */ + ExprList *pLhs = 0; /* New LHS after mods */ + int i; /* Loop counter */ + + assert( ExprUseXSelect(pNew) ); + pOrigRhs = pSelect->pEList; + assert( pNew->pLeft!=0 ); + assert( ExprUseXList(pNew->pLeft) ); + if( pSelect==pNew->x.pSelect ){ + pOrigLhs = pNew->pLeft->x.pList; + } + for(i=iEq; i<pLoop->nLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField; + assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); + iField = pLoop->aLTerm[i]->u.x.iField - 1; + if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); + pOrigRhs->a[iField].pExpr = 0; + if( pOrigLhs ){ + assert( pOrigLhs->a[iField].pExpr!=0 ); + pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); + pOrigLhs->a[iField].pExpr = 0; + } + } + } + sqlite3ExprListDelete(db, pOrigRhs); + if( pOrigLhs ){ + sqlite3ExprListDelete(db, pOrigLhs); + pNew->pLeft->x.pList = pLhs; + } + pSelect->pEList = pRhs; + if( pLhs && pLhs->nExpr==1 ){ + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + Expr *p = pLhs->a[0].pExpr; + pLhs->a[0].pExpr = 0; + sqlite3ExprDelete(db, pNew->pLeft); + pNew->pLeft = p; + } + if( pSelect->pOrderBy ){ + /* If the SELECT statement has an ORDER BY clause, zero the + ** iOrderByCol variables. These are set to non-zero when an + ** ORDER BY term exactly matches one of the terms of the + ** result-set. Since the result-set of the SELECT statement may + ** have been modified or reordered, these variables are no longer + ** set correctly. Since setting them is just an optimization, + ** it's easiest just to zero them here. */ + ExprList *pOrderBy = pSelect->pOrderBy; + for(i=0; i<pOrderBy->nExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } } - } #if 0 - printf("For indexing, change the IN expr:\n"); - sqlite3TreeViewExpr(0, pX, 0); - printf("Into:\n"); - sqlite3TreeViewExpr(0, pNew, 0); + printf("For indexing, change the IN expr:\n"); + sqlite3TreeViewExpr(0, pX, 0); + printf("Into:\n"); + sqlite3TreeViewExpr(0, pNew, 0); #endif + } } return pNew; } @@ -149784,7 +153766,8 @@ static int codeEqualityTerm( } sqlite3ExprDelete(db, pX); }else{ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); } pX = pExpr; @@ -150054,7 +154037,7 @@ static void whereLikeOptimizationStringFixup( if( pTerm->wtFlags & TERM_LIKEOPT ){ VdbeOp *pOp; assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); + pOp = sqlite3VdbeGetLastOp(v); assert( pOp!=0 ); assert( pOp->opcode==OP_String8 || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); @@ -150148,11 +154131,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCode(pWalker->pParse, pExpr, reg); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -150160,15 +154144,15 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pExpr->op==TK_TRUEFALSE ){ + /* Do not walk disabled expressions. tag-20230504-1 */ + return WRC_Prune; } return rc; } @@ -150270,7 +154254,7 @@ static void codeCursorHint( } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); @@ -150378,143 +154362,6 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ } } -/* An instance of the IdxExprTrans object carries information about a -** mapping from an expression on table columns into a column in an index -** down through the Walker. -*/ -typedef struct IdxExprTrans { - Expr *pIdxExpr; /* The index expression */ - int iTabCur; /* The cursor of the corresponding table */ - int iIdxCur; /* The cursor for the index */ - int iIdxCol; /* The column for the index */ - int iTabCol; /* The column for the table */ - WhereInfo *pWInfo; /* Complete WHERE clause information */ - sqlite3 *db; /* Database connection (for malloc()) */ -} IdxExprTrans; - -/* -** Preserve pExpr on the WhereETrans list of the WhereInfo. -*/ -static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ - WhereExprMod *pNew; - pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); - if( pNew==0 ) return; - pNew->pNext = pTrans->pWInfo->pExprMods; - pTrans->pWInfo->pExprMods = pNew; - pNew->pExpr = pExpr; - memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); -} - -/* The walker node callback used to transform matching expressions into -** a reference to an index column for an index on an expression. -** -** If pExpr matches, then transform it into a reference to the index column -** that contains the value of pExpr. -*/ -static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ - pExpr = sqlite3ExprSkipCollate(pExpr); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3ExprAffinity(pExpr); - pExpr->op = TK_COLUMN; - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - testcase( ExprHasProperty(pExpr, EP_Unlikely) ); - ExprClearProperty(pExpr, EP_Skip|EP_Unlikely|EP_WinFunc|EP_Subrtn); - pExpr->y.pTab = 0; - return WRC_Prune; - }else{ - return WRC_Continue; - } -} - -#ifndef SQLITE_OMIT_GENERATED_COLUMNS -/* A walker node callback that translates a column reference to a table -** into a corresponding column reference of an index. -*/ -static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ - if( pExpr->op==TK_COLUMN ){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ - assert( ExprUseYTab(pExpr) && pExpr->y.pTab!=0 ); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - pExpr->y.pTab = 0; - } - } - return WRC_Continue; -} -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - -/* -** For an indexes on expression X, locate every instance of expression X -** in pExpr and change that subexpression into a reference to the appropriate -** column of the index. -** -** 2019-10-24: Updated to also translate references to a VIRTUAL column in -** the table into references to the corresponding (stored) column of the -** index. -*/ -static void whereIndexExprTrans( - Index *pIdx, /* The Index */ - int iTabCur, /* Cursor of the table that is being indexed */ - int iIdxCur, /* Cursor of the index itself */ - WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ -){ - int iIdxCol; /* Column number of the index */ - ExprList *aColExpr; /* Expressions that are indexed */ - Table *pTab; - Walker w; - IdxExprTrans x; - aColExpr = pIdx->aColExpr; - if( aColExpr==0 && !pIdx->bHasVCol ){ - /* The index does not reference any expressions or virtual columns - ** so no translations are needed. */ - return; - } - pTab = pIdx->pTable; - memset(&w, 0, sizeof(w)); - w.u.pIdxTrans = &x; - x.iTabCur = iTabCur; - x.iIdxCur = iIdxCur; - x.pWInfo = pWInfo; - x.db = pWInfo->pParse->db; - for(iIdxCol=0; iIdxCol<pIdx->nColumn; iIdxCol++){ - i16 iRef = pIdx->aiColumn[iIdxCol]; - if( iRef==XN_EXPR ){ - assert( aColExpr!=0 && aColExpr->a[iIdxCol].pExpr!=0 ); - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; - if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; - w.xExprCallback = whereIndexExprTransNode; -#ifndef SQLITE_OMIT_GENERATED_COLUMNS - }else if( iRef>=0 - && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 - && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0 - || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]), - sqlite3StrBINARY)==0) - ){ - /* Check to see if there are direct references to generated columns - ** that are contained in the index. Pulling the generated column - ** out of the index is an optimization only - the main table is always - ** available if the index cannot be used. To avoid unnecessary - ** complication, omit this optimization if the collating sequence for - ** the column is non-standard */ - x.iTabCol = iRef; - w.xExprCallback = whereIndexExprTransColumn; -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - }else{ - continue; - } - x.iIdxCol = iIdxCol; - sqlite3WalkExpr(&w, pWInfo->pWhere); - sqlite3WalkExprList(&w, pWInfo->pOrderBy); - sqlite3WalkExprList(&w, pWInfo->pResultSet); - } -} - /* ** The pTruth expression is always true because it is the WHERE clause ** a partial index that is driving a query loop. Look through all of the @@ -150583,6 +154430,8 @@ static SQLITE_NOINLINE void filterPullDown( testcase( pTerm->wtFlags & TERM_VIRTUAL ); regRowid = sqlite3GetTempReg(pParse); regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt); + VdbeCoverage(pParse->pVdbe); sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, addrNxt, regRowid, 1); VdbeCoverage(pParse->pVdbe); @@ -150642,13 +154491,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x800 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); - sqlite3WhereLoopPrint(pLoop, pWC); + if( sqlite3WhereTrace & 0x1000 ){ + sqlite3WhereLoopPrint(pLoop, pWC); + } } - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ if( iLevel==0 ){ sqlite3DebugPrintf("WHERE clause being coded:\n"); sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); @@ -150734,9 +154585,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( && pLoop->u.vtab.bOmitOffset ){ assert( pTerm->eOperator==WO_AUX ); - assert( pWInfo->pLimit!=0 ); - assert( pWInfo->pLimit->iOffset>0 ); - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pLimit->iOffset); + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->iOffset>0 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset); VdbeComment((v,"Zero OFFSET counter")); } } @@ -150844,6 +154695,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; if( pLevel->regFilter ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + VdbeCoverage(v); sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, iRowidReg, 1); VdbeCoverage(v); @@ -151195,6 +155048,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); + if( pRangeStart || pRangeEnd ){ + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); + addrSeekScan = 0; + } VdbeCoverage(v); } sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -151231,16 +155089,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - if( addrSeekScan ){ - /* For a seek-scan that has a range on the lowest term of the index, - ** we have to make the top of the loop be code that sets the end - ** condition of the range. Otherwise, the OP_SeekScan might jump - ** over that initialization, leaving the range-end value set to the - ** range-start value, resulting in a wrong answer. - ** See ticket 5981a8c041a3c2f3 (2021-11-02). - */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - } + assert( addrSeekScan==0 ); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -151270,11 +155119,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } nConstraint++; } - sqlite3DbFree(db, zStartAff); - sqlite3DbFree(db, zEndAff); + if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff); + if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ - if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ @@ -151333,27 +155182,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } if( pLevel->iLeftJoin==0 ){ - /* If pIdx is an index on one or more expressions, then look through - ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. Also attempt to translate references - ** to virtual columns in the table into references to (stored) columns - ** of the index. - ** - ** Do not do this for the RHS of a LEFT JOIN. This is because the - ** expression may be evaluated after OP_NullRow has been executed on - ** the cursor. In this case it is important to do the full evaluation, - ** as the result of the expression may not be NULL, even if all table - ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a - ** - ** Also, do not do this when processing one index an a multi-index - ** OR clause, since the transformation will become invalid once we - ** move forward to the next index. - ** https://sqlite.org/src/info/4e8e4857d32d401f - */ - if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ){ - whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); - } - /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of ** the partial index. @@ -151466,7 +155294,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, + pOrTab = sqlite3DbMallocRawNN(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); @@ -151586,7 +155414,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + WHERETRACE(0xffffffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, 0, WHERE_OR_SUBCLAUSE, iCovCur); assert( pSubWInfo || pParse->nErr ); @@ -151719,7 +155547,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->op==OP_Return ); pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } + if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -151823,12 +155651,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } #endif } -#ifdef WHERETRACE_ENABLED /* 0xffff */ +#ifdef WHERETRACE_ENABLED /* 0xffffffff */ if( sqlite3WhereTrace ){ VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", pWC->nTerm-j, pTerm, iLoop)); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("Coding auxiliary constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } @@ -151857,8 +155685,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pTerm->leftCursor!=iCur ) continue; if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ) continue; pE = pTerm->pExpr; -#ifdef WHERETRACE_ENABLED /* 0x800 */ - if( sqlite3WhereTrace & 0x800 ){ +#ifdef WHERETRACE_ENABLED /* 0x4001 */ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("Coding transitive constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } @@ -151973,13 +155801,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } } -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x20000 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", iLevel); sqlite3WhereClausePrint(pWC); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", iLevel, (u64)pLevel->notReady); } @@ -152347,7 +156175,7 @@ static int isLikeOrGlob( if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT || (ALWAYS( ExprUseYTab(pLeft) ) - && pLeft->y.pTab + && ALWAYS(pLeft->y.pTab) && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ ){ int isNum; @@ -152464,8 +156292,7 @@ static int isAuxiliaryVtabOperator( ** MATCH(expression,vtab_column) */ pCol = pList->a[1].pExpr; - assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ for(i=0; i<ArraySize(aOp); i++){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); @@ -152490,7 +156317,7 @@ static int isAuxiliaryVtabOperator( */ pCol = pList->a[0].pExpr; assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ sqlite3_vtab *pVtab; sqlite3_module *pMod; @@ -152515,13 +156342,12 @@ static int isAuxiliaryVtabOperator( int res = 0; Expr *pLeft = pExpr->pLeft; Expr *pRight = pExpr->pRight; - assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); - testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) ); if( ExprIsVtab(pLeft) ){ res++; } - assert( pRight==0 || pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); - testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + assert( pRight==0 || pRight->op!=TK_COLUMN + || (ExprUseYTab(pRight) && pRight->y.pTab!=0) ); if( pRight && ExprIsVtab(pRight) ){ res++; SWAP(Expr*, pLeft, pRight); @@ -153057,35 +156883,40 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ */ static SQLITE_NOINLINE int exprMightBeIndexed2( SrcList *pFrom, /* The FROM clause */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ int *aiCurCol, /* Write the referenced table cursor and column here */ - Expr *pExpr /* An operand of a comparison operator */ + Expr *pExpr, /* An operand of a comparison operator */ + int j /* Start looking with the j-th pFrom entry */ ){ Index *pIdx; int i; int iCur; - for(i=0; mPrereq>1; i++, mPrereq>>=1){} - iCur = pFrom->a[i].iCursor; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->aColExpr==0 ) continue; - for(i=0; i<pIdx->nKeyCol; i++){ - if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ - aiCurCol[0] = iCur; - aiCurCol[1] = XN_EXPR; - return 1; + do{ + iCur = pFrom->a[j].iCursor; + for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; i<pIdx->nKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); + if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 + && pExpr->op!=TK_STRING + ){ + aiCurCol[0] = iCur; + aiCurCol[1] = XN_EXPR; + return 1; + } } } - } + }while( ++j < pFrom->nSrc ); return 0; } static int exprMightBeIndexed( SrcList *pFrom, /* The FROM clause */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ int *aiCurCol, /* Write the referenced table cursor & column here */ Expr *pExpr, /* An operand of a comparison operator */ int op /* The specific comparison operator */ ){ + int i; + /* If this expression is a vector to the left or right of a ** inequality constraint (>, <, >= or <=), perform the processing ** on the first element of the vector. */ @@ -153095,7 +156926,6 @@ static int exprMightBeIndexed( if( pExpr->op==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ assert( ExprUseXList(pExpr) ); pExpr = pExpr->x.pList->a[0].pExpr; - } if( pExpr->op==TK_COLUMN ){ @@ -153103,9 +156933,16 @@ static int exprMightBeIndexed( aiCurCol[1] = pExpr->iColumn; return 1; } - if( mPrereq==0 ) return 0; /* No table references */ - if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ - return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr); + + for(i=0; i<pFrom->nSrc; i++){ + Index *pIdx; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr ){ + return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); + } + } + } + return 0; } @@ -153231,7 +157068,7 @@ static void exprAnalyze( pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr; } - if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){ + if( exprMightBeIndexed(pSrc, aiCurCol, pLeft, op) ){ pTerm->leftCursor = aiCurCol[0]; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); pTerm->u.x.leftColumn = aiCurCol[1]; @@ -153239,7 +157076,7 @@ static void exprAnalyze( } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; if( pRight - && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op) + && exprMightBeIndexed(pSrc, aiCurCol, pRight, op) && !ExprHasProperty(pRight, EP_FixedCol) ){ WhereTerm *pNew; @@ -153283,7 +157120,7 @@ static void exprAnalyze( && 0==sqlite3ExprCanBeNull(pLeft) ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->op = TK_TRUEFALSE; + pExpr->op = TK_TRUEFALSE; /* See tag-20230504-1 */ pExpr->u.zToken = "false"; ExprSetProperty(pExpr, EP_IsFalse); pTerm->prereqAll = 0; @@ -153450,7 +157287,6 @@ static void exprAnalyze( transferJoinMarkings(pNewExpr1, pExpr); idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), @@ -153458,6 +157294,7 @@ static void exprAnalyze( transferJoinMarkings(pNewExpr2, pExpr); idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew1); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ @@ -153514,7 +157351,7 @@ static void exprAnalyze( && pTerm->u.x.iField==0 && pExpr->pLeft->op==TK_VECTOR && ALWAYS( ExprUseXSelect(pExpr) ) - && pExpr->x.pSelect->pPrior==0 + && (pExpr->x.pSelect->pPrior==0 || (pExpr->x.pSelect->selFlags & SF_Values)) #ifndef SQLITE_OMIT_WINDOWFUNC && pExpr->x.pSelect->pWin==0 #endif @@ -153683,9 +157520,9 @@ static void whereAddLimitExpr( ** exist only so that they may be passed to the xBestIndex method of the ** single virtual table in the FROM clause of the SELECT. */ -SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ - assert( p==0 || (p->pGroupBy==0 && (p->selFlags & SF_Aggregate)==0) ); - if( (p && p->pLimit) /* 1 */ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ + assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ + if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ ){ @@ -153702,6 +157539,13 @@ SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ assert( pWC->a[ii].eOperator==WO_ROWVAL ); continue; } + if( pWC->a[ii].nChild ){ + /* If this term has child terms, then they are also part of the + ** pWC->a[] array. So this term can be ignored, as a LIMIT clause + ** will only be added if each of the child terms passes the + ** (leftCursor==iCsr) test below. */ + continue; + } if( pWC->a[ii].leftCursor!=iCsr ) return; } @@ -153921,9 +157765,12 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ) ){ + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ joinType = EP_OuterON; }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ joinType = EP_InnerON; } sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); @@ -154002,7 +157849,7 @@ SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** block sorting is required. */ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; } /* @@ -154640,7 +158487,7 @@ static void translateColumnToCopy( #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; i<p->nConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -154660,7 +158507,7 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ } static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; i<p->nConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -154678,6 +158525,43 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ #define whereTraceIndexInfoOutputs(A) #endif +/* +** We know that pSrc is an operand of an outer join. Return true if +** pTerm is a constraint that is compatible with that join. +** +** pTerm must be EP_OuterON if pSrc is the right operand of an +** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc +** is the left operand of a RIGHT join. +** +** See https://sqlite.org/forum/forumpost/206d99a16dd9212f +** for an example of a WHERE clause constraints that may not be used on +** the right table of a RIGHT JOIN because the constraint implies a +** not-NULL condition on the left table of the RIGHT JOIN. +*/ +static int constraintCompatibleWithOuterJoin( + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc /* Table we are trying to access */ +){ + assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); + testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) + testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) + || pTerm->pExpr->w.iJoin != pSrc->iCursor + ){ + return 0; + } + if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + ){ + return 0; + } + return 1; +} + + + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -154693,16 +158577,10 @@ static int termCanDriveIndex( if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - return 0; /* See tag-20191211-001 */ - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); @@ -154716,6 +158594,57 @@ static int termCanDriveIndex( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + sqlite3_str *pStr = sqlite3_str_new(pParse->db); + sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + sqlite3_str_appendf(pStr, "%s%s", zSep, zName); + zSep = ", "; + } + zText = sqlite3_str_finish(pStr); + if( zText==0 ){ + sqlite3OomFault(pParse->db); + }else{ + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "") + ); + sqlite3_free(zText); + } + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator @@ -154723,8 +158652,7 @@ static int termCanDriveIndex( */ static SQLITE_NOINLINE void constructAutomaticIndex( Parse *pParse, /* The parsing context */ - const WhereClause *pWC, /* The WHERE clause */ - const SrcItem *pSrc, /* The FROM clause term to get the next index */ + WhereClause *pWC, /* The WHERE clause */ const Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ @@ -154745,12 +158673,17 @@ static SQLITE_NOINLINE void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ - SrcItem *pTabItem; /* FROM clause term being indexed */ + SrcList *pTabList; /* The complete FROM clause */ + SrcItem *pSrc; /* The FROM clause term to get the next index */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -154761,6 +158694,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ nKeyCol = 0; + pTabList = pWC->pWInfo->pTabList; + pSrc = &pTabList->a[pLevel->iFrom]; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; @@ -154771,7 +158706,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsTableConstraint(pExpr, pSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -154812,7 +158747,11 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -154848,6 +158787,16 @@ static SQLITE_NOINLINE void constructAutomaticIndex( assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -154874,25 +158823,27 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); - if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); pLevel->regFilter = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); } /* Fill the automatic index with content */ - pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; + assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); + if( pSrc->fg.viaCoroutine ){ + int regYield = pSrc->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -154909,17 +158860,18 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, regBase, pLoop->u.btree.nEq); } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->fg.viaCoroutine ){ + if( pSrc->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pTabItem->regResult, pLevel->iIdxCur); + pSrc->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); - pTabItem->fg.viaCoroutine = 0; + pSrc->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); @@ -154929,6 +158881,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); @@ -154970,6 +158923,10 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( Vdbe *v = pParse->pVdbe; /* VDBE under construction */ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + pParse->pIdxEpr = 0; assert( pLoop!=0 ); assert( v!=0 ); @@ -154977,9 +158934,11 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); do{ + const SrcList *pTabList; const SrcItem *pItem; const Table *pTab; u64 sz; + int iSrc; sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); addrCont = sqlite3VdbeMakeLabel(pParse); iCur = pLevel->iTabCur; @@ -154993,7 +158952,9 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( ** testing complicated. By basing the blob size on the value in the ** sqlite_stat1 table, testing is much easier. */ - pItem = &pWInfo->pTabList->a[pLevel->iFrom]; + pTabList = pWInfo->pTabList; + iSrc = pLevel->iFrom; + pItem = &pTabList->a[iSrc]; assert( pItem!=0 ); pTab = pItem->pTab; assert( pTab!=0 ); @@ -155010,7 +158971,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){ Expr *pExpr = pTerm->pExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsTableConstraint(pExpr, pItem) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -155026,9 +158987,8 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jj<n; jj++){ - int iCol = pIdx->aiColumn[jj]; assert( pIdx->pTable==pItem->pTab ); - sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); sqlite3ReleaseTempRange(pParse, r1, n); @@ -155059,6 +159019,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( } }while( iLevel < pWInfo->nLevel ); sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; } @@ -155114,22 +159075,10 @@ static sqlite3_index_info *allocateIndexInfo( assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); assert( pTerm->u.x.leftColumn>=XN_ROWID ); assert( pTerm->u.x.leftColumn<pTab->nCol ); - - /* tag-20191211-002: WHERE-clause constraints are not useful to the - ** right-hand table of a LEFT JOIN nor to the either table of a - ** RIGHT JOIN. See tag-20191211-001 for the - ** equivalent restriction for ordinary tables. */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ); - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } nTerm++; pTerm->wtFlags |= TERM_OK; @@ -155326,6 +159275,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; return rc; @@ -155370,6 +159322,7 @@ static int whereKeyStats( assert( pIdx->nSample>0 ); assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search ** is simply the aSample[] array. If the samples in aSample[] contain more @@ -155414,7 +159367,12 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = MIN(pRec->nField, pIdx->nSample); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -155480,12 +159438,12 @@ static int whereKeyStats( if( iCol>0 ){ pRec->nField = iCol; assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } if( i>0 ){ pRec->nField = nField; assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } } } @@ -155502,7 +159460,7 @@ static int whereKeyStats( ** is larger than all samples in the array. */ tRowcnt iUpper, iGap; if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = pIdx->nRowEst0; }else{ iUpper = aSample[i].anLt[iCol]; } @@ -155658,7 +159616,7 @@ static int whereRangeSkipScanEst( int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); pLoop->nOut -= nAdjust; *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", nLower, nUpper, nAdjust*-1, pLoop->nOut)); } @@ -155836,7 +159794,7 @@ static int whereRangeScanEst( if( nNew<nOut ){ nOut = nNew; } - WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n", + WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n", (u32)iLower, (u32)iUpper, nOut)); } }else{ @@ -155850,7 +159808,7 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); assert( pLower || pUpper ); #endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); @@ -155869,7 +159827,7 @@ static int whereRangeScanEst( if( nNew<nOut ) nOut = nNew; #if defined(WHERETRACE_ENABLED) if( pLoop->nOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", pLoop->nOut, nOut)); } #endif @@ -155934,7 +159892,7 @@ static int whereEqualScanEst( pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; @@ -155982,9 +159940,9 @@ static int whereInScanEst( } if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; + if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; @@ -156093,7 +160051,7 @@ SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; i<p->nLTerm; i++){ sqlite3WhereTermPrint(p->aLTerm[i], i); @@ -156131,12 +160089,18 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ } /* -** Deallocate internal memory used by a WhereLoop object +** Deallocate internal memory used by a WhereLoop object. Leave the +** object in an initialized state, as if it had been newly allocated. */ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ){ + sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = p->aLTermSpace; + p->nLSlot = ArraySize(p->aLTermSpace); + } whereLoopClearUnion(db, p); - whereLoopInit(p); + p->nLTerm = 0; + p->wsFlags = 0; } /* @@ -156160,7 +160124,9 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ */ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + if( pFrom->nLTerm > pTo->nLSlot + && whereLoopResize(db, pTo, pFrom->nLTerm) + ){ memset(pTo, 0, WHERE_LOOP_XFER_SZ); return SQLITE_NOMEM_BKPT; } @@ -156178,8 +160144,9 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + assert( db!=0 ); whereLoopClear(db, p); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -156187,30 +160154,19 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ assert( pWInfo!=0 ); + assert( db!=0 ); sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); } - assert( pWInfo->pExprMods==0 ); while( pWInfo->pMemToFree ){ WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; - sqlite3DbFreeNN(db, pWInfo->pMemToFree); + sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); pWInfo->pMemToFree = pNext; } - sqlite3DbFreeNN(db, pWInfo); -} - -/* Undo all Expr node modifications -*/ -static void whereUndoExprMods(WhereInfo *pWInfo){ - while( pWInfo->pExprMods ){ - WhereExprMod *p = pWInfo->pExprMods; - pWInfo->pExprMods = p->pNext; - memcpy(p->pExpr, &p->orig, sizeof(p->orig)); - sqlite3DbFree(pWInfo->pParse->db, p); - } + sqlite3DbNNFreeNN(db, pWInfo); } /* @@ -156559,6 +160515,7 @@ static void whereLoopOutputAdjust( if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ + sqlite3ProgressCheck(pWC->pWInfo->pParse); if( pLoop->maskSelf==pTerm->prereqAll ){ /* If there are extra terms in the WHERE clause not used by an index ** that depend only on the table being scanned, and that will tend to @@ -156726,7 +160683,10 @@ static int whereLoopAddBtreeIndex( WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + assert( db->mallocFailed==0 || pParse->nErr>0 ); + if( pParse->nErr ){ + return pParse->rc; + } WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", pProbe->pTable->zName,pProbe->zName, pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); @@ -156777,32 +160737,11 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - /* tag-20191211-001: Do not allow constraints from the WHERE clause to - ** be used by the right table of a LEFT JOIN nor by the left table of a - ** RIGHT JOIN. Only constraints in the ON clause are allowed. - ** See tag-20191211-002 for the vtab equivalent. - ** - ** 2022-06-06: See https://sqlite.org/forum/forumpost/206d99a16dd9212f - ** for an example of a WHERE clause constraints that may not be used on - ** the right table of a RIGHT JOIN because the constraint implies a - ** not-NULL condition on the left table of the RIGHT JOIN. - ** - ** 2022-06-10: The same condition applies to termCanDriveIndex() above. - ** https://sqlite.org/forum/forumpost/51e6959f61 - */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } - if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; }else{ @@ -156813,7 +160752,11 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nBtm = saved_nBtm; pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + if( pNew->nLTerm>=pNew->nLSlot + && whereLoopResize(db, pNew, pNew->nLTerm+1) + ){ + break; /* OOM while trying to enlarge the pNew->aLTerm array */ + } pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; @@ -156906,38 +160849,39 @@ static int whereLoopAddBtreeIndex( if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->u.btree.nBtm = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range constraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - pNew->u.btree.nTop = 1; - } - }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->u.btree.nTop = whereRangeVectorLen( + }else{ + int nVecLen = whereRangeVectorLen( pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm ); - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; + if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = nVecLen; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = nVecLen; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } } /* At this point pNew->nOut is set to the number of rows expected to @@ -156989,7 +160933,7 @@ static int whereLoopAddBtreeIndex( && pNew->nOut+10 > pProbe->aiRowLogEst[0] ){ #if WHERETRACE_ENABLED /* 0x01 */ - if( sqlite3WhereTrace & 0x01 ){ + if( sqlite3WhereTrace & 0x20 ){ sqlite3DebugPrintf( "STAT4 determines term has low selectivity:\n"); sqlite3WhereTermPrint(pTerm, 999); @@ -157026,9 +160970,17 @@ static int whereLoopAddBtreeIndex( ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ assert( pSrc->pTab->szTabRow>0 ); - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* The pProbe->szIdxRow is low for an IPK table since the interior + ** pages are small. Thuse szIdxRow gives a good estimate of seek cost. + ** But the leaf pages are full-size, so pProbe->szIdxRow would badly + ** under-estimate the scanning cost. */ + rCostIdx = pNew->nOut + 16; + }else{ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + } pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -157050,6 +161002,9 @@ static int whereLoopAddBtreeIndex( && (pNew->u.btree.nEq<pProbe->nKeyCol || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ + if( pNew->u.btree.nEq>3 ){ + sqlite3ProgressCheck(pParse); + } whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; @@ -157182,6 +161137,149 @@ static int whereUsablePartialIndex( } /* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; i<pIdx->nColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + +/* +** Structure passed to the whereIsCoveringIndex Walker callback. +*/ +typedef struct CoveringIndexCheck CoveringIndexCheck; +struct CoveringIndexCheck { + Index *pIdx; /* The index */ + int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ +}; + +/* +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. +** +** If the Expr node references the table with cursor pCk->iTabCur, then +** make sure that column is covered by the index pCk->pIdx. We know that +** all columns less than 63 (really BMS-1) are covered, so we don't need +** to check them. But we do need to check any column at 63 or greater. +** +** If the index does not cover the column, then set pWalk->eCode to +** non-zero and return WRC_Abort to stop the search. +** +** If this node does not disprove that the index can be a covering index, +** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. +*/ +static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; i<nColumn; i++){ + if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue; + } + pCk->bUnidx = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; + return WRC_Prune; + } + return WRC_Continue; +} + + +/* +** pIdx is an index that covers all of the low-number columns used by +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. +** +** The return value is one of these: +** +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. +*/ +static SQLITE_NOINLINE u32 whereIsCoveringIndex( + WhereInfo *pWInfo, /* The WHERE clause context */ + Index *pIdx, /* Index that is being tested */ + int iTabCur /* Cursor for the table being indexed */ +){ + int i, rc; + struct CoveringIndexCheck ck; + Walker w; + if( pWInfo->pSelect==0 ){ + /* We don't have access to the full query, so we cannot check to see + ** if pIdx is covering. Assume it is not. */ + return 0; + } + if( pIdx->bHasExpr==0 ){ + for(i=0; i<pIdx->nColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 0; + } + } + ck.pIdx = pIdx; + ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIsCoveringIndexWalkCallback; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pCovIdxCk = &ck; + sqlite3WalkSelect(&w, pWInfo->pSelect); + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; +} + +/* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. @@ -157263,7 +161361,7 @@ static int whereLoopAddBtree( sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; + sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; @@ -157314,7 +161412,8 @@ static int whereLoopAddBtree( if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ pNew->rSetup += 28; }else{ - pNew->rSetup -= 10; + pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes + ** on ephemeral materializations of views */ } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); if( pNew->rSetup<0 ) pNew->rSetup = 0; @@ -157383,6 +161482,9 @@ static int whereLoopAddBtree( #else pNew->rRun = rSize + 16; #endif + if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){ + pNew->wsFlags |= WHERE_VIEWSCAN; + } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); @@ -157391,11 +161493,38 @@ static int whereLoopAddBtree( }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0x200, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0x200, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0x200, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 ){ + WHERETRACE(0x200, + ("-> %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + } } /* Full scan via index */ @@ -157568,7 +161697,7 @@ static int whereLoopAddVirtualOne( ** that the particular combination of parameters provided is unusable. ** Make no entries in the loop table. */ - WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); return SQLITE_OK; } return rc; @@ -157679,7 +161808,7 @@ static int whereLoopAddVirtualOne( sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", *pbIn, (sqlite3_uint64)mPrereq, (sqlite3_uint64)(pNew->prereq & ~mPrereq))); @@ -157780,32 +161909,27 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ return pHidden->eDistinct; } -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** Cause the prepared statement that is associated with a call to -** xBestIndex to potentiall use all schemas. If the statement being +** xBestIndex to potentially use all schemas. If the statement being ** prepared is read-only, then just start read transactions on all ** schemas. But if this is a write operation, start writes on all ** schemas. ** ** This is used by the (built-in) sqlite_dbpage virtual table. */ -SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){ - HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; - Parse *pParse = pHidden->pParse; +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse *pParse){ int nDb = pParse->db->nDb; int i; for(i=0; i<nDb; i++){ sqlite3CodeVerifySchema(pParse, i); } - if( pParse->writeMask ){ + if( DbMaskNonZero(pParse->writeMask) ){ for(i=0; i<nDb; i++){ sqlite3BeginWriteOperation(pParse, 0, i); } } } -#endif /* ** Add all WhereLoop objects for a table of the join identified by @@ -157871,7 +161995,7 @@ static int whereLoopAddVirtual( /* First call xBestIndex() with all constraints usable. */ WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); - WHERETRACE(0x40, (" VirtualOne: all usable\n")); + WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry ); @@ -157896,7 +162020,7 @@ static int whereLoopAddVirtual( /* If the plan produced by the earlier call uses an IN(...) term, call ** xBestIndex again, this time with IN(...) terms disabled. */ if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); assert( bIn==0 ); @@ -157922,7 +162046,7 @@ static int whereLoopAddVirtual( mPrev = mNext; if( mNext==ALLBITS ) break; if( mNext==mBest || mNext==mBestNoIn ) continue; - WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", + WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0); @@ -157936,7 +162060,7 @@ static int whereLoopAddVirtual( ** that requires no source tables at all (i.e. one guaranteed to be ** usable), make a call here with all source tables disabled */ if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); if( bIn==0 ) seenZeroNoIN = 1; @@ -157946,7 +162070,7 @@ static int whereLoopAddVirtual( ** that requires no source tables at all and does not use an IN(...) ** operator, make a final call to obtain one here. */ if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); } @@ -158002,7 +162126,7 @@ static int whereLoopAddOr( sSubBuild = *pBuilder; sSubBuild.pOrSet = &sCur; - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){ if( (pOrTerm->eOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -158019,9 +162143,9 @@ static int whereLoopAddOr( } sCur.n = 0; #ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ + if( sqlite3WhereTrace & 0x20000 ){ sqlite3WhereClausePrint(sSubBuild.pWC); } #endif @@ -158036,8 +162160,6 @@ static int whereLoopAddOr( if( rc==SQLITE_OK ){ rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } - assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 - || rc==SQLITE_NOMEM ); testcase( rc==SQLITE_NOMEM && sCur.n>0 ); testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ @@ -158083,7 +162205,7 @@ static int whereLoopAddOr( pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); } } return rc; @@ -158109,7 +162231,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; - whereLoopInit(pNew); + + /* Verify that pNew has already been initialized */ + assert( pNew->nLTerm==0 ); + assert( pNew->wsFlags==0 ); + assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm!=0 ); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){ Bitmask mUnusable = 0; @@ -158425,8 +162553,8 @@ static i8 wherePathSatisfiesOrderBy( if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; - if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } @@ -158558,37 +162686,56 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** order. */ static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted + WhereInfo *pWInfo, /* Query planning context */ + LogEst nRow, /* Estimated number of rows to sort */ + int nOrderBy, /* Number of ORDER BY clause terms */ + int nSorted /* Number of initial ORDER BY terms naturally in order */ ){ - /* TUNING: Estimated cost of a full external sort, where N is + /* Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** - ** cost = (3.0 * N * log(N)). + ** cost = (K * N * log(N)). ** ** Or, if the order-by clause has X terms but only the last Y ** terms are out of order, then block-sorting will reduce the ** sorting cost to: ** - ** cost = (3.0 * N * log(N)) * (Y/X) + ** cost = (K * N * log(N)) * (Y/X) + ** + ** The constant K is at least 2.0 but will be larger if there are a + ** large number of columns to be sorted, as the sorting time is + ** proportional to the amount of content to be sorted. The algorithm + ** does not currently distinguish between fat columns (BLOBs and TEXTs) + ** and skinny columns (INTs). It just uses the number of columns as + ** an approximation for the row width. ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. + ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort + ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; + LogEst rSortCost, nCol; + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->pEList!=0 ); + /* TUNING: sorting cost proportional to the number of output columns: */ + nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); + rSortCost = nRow + nCol; + if( nSorted>0 ){ + /* Scale the result by (Y/X) */ + rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + } /* Multiple by log(M) where M is the number of output rows. ** Use the LIMIT for M if it is smaller. Or if this sort is for ** a DISTINCT operator, M will be the number of distinct output ** rows, so fudge it downwards a bit. */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){ - nRow = pWInfo->iLimit; + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ + if( nSorted!=0 ){ + rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ + } + if( pWInfo->iLimit<nRow ){ + nRow = pWInfo->iLimit; + } }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT ** reduces the number of output rows by a factor of 2 */ @@ -158614,7 +162761,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ Parse *pParse; /* Parsing context */ - sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ @@ -158633,7 +162779,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; - db = pParse->db; nLoop = pWInfo->nLevel; /* TUNING: For simple queries, only the best path is tracked. ** For 2-way joins, the 5 best paths are followed. @@ -158656,7 +162801,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); + pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -158706,9 +162851,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + Bitmask revMask; /* Mask of rev-order loops for (..) */ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; @@ -158727,7 +162872,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; + isOrdered = pFrom->isOrdered; if( isOrdered<0 ){ + revMask = 0; isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); @@ -158740,11 +162887,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - /* TUNING: Add a small extra penalty (5) to sorting as an + /* TUNING: Add a small extra penalty (3) to sorting as an ** extra encouragment to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", @@ -158755,6 +162902,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } + /* TUNING: A full-scan of a VIEW or subquery in the outer loop + ** is not so bad. */ + if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ + rCost += -10; + nOut += -30; + } + /* Check to see if pWLoop should be added to the set of ** mxChoice best-so-far paths. ** @@ -158905,7 +163059,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nFrom==0 ){ sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_ERROR; } @@ -158941,6 +163095,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + if( pWInfo->pSelect->pOrderBy + && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ + pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; + } }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -158987,7 +163145,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } @@ -159085,7 +163243,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->cId = '0'; #endif #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); } #endif @@ -159152,6 +163310,13 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** at most a single row. ** 4) The table must not be referenced by any part of the query apart ** from its own USING or ON clause. +** 5) The table must not have an inner-join ON or USING clause if there is +** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause +** might move from the right side to the left side of the RIGHT JOIN. +** Note: Due to (2), this condition can only arise if the table is +** the right-most table of a subquery that was flattened into the +** main query and that subquery was the right-hand operand of an +** inner join that held an ON or USING clause. ** ** For example, given: ** @@ -159177,6 +163342,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( ){ int i; Bitmask tabUsed; + int hasRightJoin; /* Preconditions checked by the caller */ assert( pWInfo->nLevel>=2 ); @@ -159191,6 +163357,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( if( pWInfo->pOrderBy ){ tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); } + hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; for(i=pWInfo->nLevel-1; i>=1; i--){ WhereTerm *pTerm, *pEnd; SrcItem *pItem; @@ -159213,9 +163380,15 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( break; } } + if( hasRightJoin + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + && pTerm->pExpr->w.iJoin==pItem->iCursor + ){ + break; /* restriction (5) */ + } } if( pTerm<pEnd ) continue; - WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId)); + WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId)); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ @@ -159254,28 +163427,27 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( const WhereInfo *pWInfo ){ int i; - LogEst nSearch; + LogEst nSearch = 0; assert( pWInfo->nLevel>=2 ); assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) ); - nSearch = pWInfo->a[0].pWLoop->nOut; - for(i=1; i<pWInfo->nLevel; i++){ + for(i=0; i<pWInfo->nLevel; i++){ WhereLoop *pLoop = pWInfo->a[i].pWLoop; const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); - if( (pLoop->wsFlags & reqFlags)==reqFlags + SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; + Table *pTab = pItem->pTab; + if( (pTab->tabFlags & TF_HasStat1)==0 ) break; + pTab->tabFlags |= TF_StatsUsed; + if( i>=1 + && (pLoop->wsFlags & reqFlags)==reqFlags /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0) ){ - SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; - Table *pTab = pItem->pTab; - pTab->tabFlags |= TF_StatsUsed; - if( nSearch > pTab->nRowLogEst - && (pTab->tabFlags & TF_HasStat1)!=0 - ){ + if( nSearch > pTab->nRowLogEst ){ testcase( pItem->fg.jointype & JT_LEFT ); pLoop->wsFlags |= WHERE_BLOOMFILTER; pLoop->wsFlags &= ~WHERE_IDX_ONLY; - WHERETRACE(0xffff, ( + WHERETRACE(0xffffffff, ( "-> use Bloom-filter on loop %c because there are ~%.1e " "lookups into %s which has only ~%.1e rows\n", pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, @@ -159287,6 +163459,86 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } /* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxEpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + Parse *pParse = (Parse*)pObject; + while( pParse->pIdxEpr!=0 ){ + IndexedExpr *p = pParse->pIdxEpr; + pParse->pIdxEpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + SrcItem *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + Table *pTab; + assert( pIdx->bHasExpr ); + pTab = pIdx->pTable; + for(i=0; i<pIdx->nColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + int bMaybeNullRow; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + testcase( pTabItem->fg.jointype & JT_LEFT ); + testcase( pTabItem->fg.jointype & JT_RIGHT ); + testcase( pTabItem->fg.jointype & JT_LTORJ ); + bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; + }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ + pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); + bMaybeNullRow = 0; + }else{ + continue; + } + if( sqlite3ExprIsConstant(pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxEpr; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr); + } +#endif + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = bMaybeNullRow; + if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ + p->aff = pIdx->zColAff[i]; + } +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxEpr = p; + if( p->pIENext==0 ){ + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); + } + } +} + +/* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains ** information needed to terminate the loop. Later, the calling routine @@ -159380,7 +163632,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ - Select *pLimit, /* Use this LIMIT/OFFSET clause, if any */ + Select *pSelect, /* The entire SELECT statement */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -159449,7 +163701,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; +#if WHERETRACE_ENABLED pWInfo->pWhere = pWhere; +#endif pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; @@ -159457,9 +163711,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; -#ifndef SQLITE_OMIT_VIRTUALTABLE - pWInfo->pLimit = pLimit; -#endif + pWInfo->pSelect = pSelect; memset(&pWInfo->nOBSat, 0, offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); @@ -159528,25 +163780,50 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Analyze all of the subexpressions. */ sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); - sqlite3WhereAddLimit(&pWInfo->sWC, pLimit); + if( pSelect && pSelect->pLimit ){ + sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); + } if( pParse->nErr ) goto whereBeginError; - /* Special case: WHERE terms that do not refer to any tables in the join - ** (constant expressions). Evaluate each such term, and jump over all the - ** generated code if the result is not true. + /* The False-WHERE-Term-Bypass optimization: + ** + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. ** - ** Do not do this if the expression contains non-deterministic functions - ** that are not within a sub-select. This is not strictly required, but - ** preserves SQLite's legacy behaviour in the following two cases: + ** Conditions: ** - ** FROM ... WHERE random()>0; -- eval random() once per row - ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. */ for(ii=0; ii<sWLB.pWC->nBase; ii++){ - WhereTerm *pT = &sWLB.pWC->a[ii]; + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ if( pT->wtFlags & TERM_VIRTUAL ) continue; - if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ - sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; } } @@ -159569,13 +163846,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ + if( sqlite3WhereTrace & 0xffffffff ){ sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); if( wctrlFlags & WHERE_USE_LIMIT ){ sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x8000 ){ Select sSelect; memset(&sSelect, 0, sizeof(sSelect)); sSelect.selFlags = SF_WhereBegin; @@ -159585,10 +163862,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sSelect.pEList = pResultSet; sqlite3TreeViewSelect(0, &sSelect, 0); } - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); - sqlite3WhereClausePrint(sWLB.pWC); + if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } } #endif @@ -159604,7 +163881,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** loops will be built using the revised truthProb values. */ if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); - WHERETRACE(0xffff, + WHERETRACE(0xffffffff, ("**** Redo all loop computations due to" " TERM_HIGHTRUTH changes ****\n")); while( pWInfo->pLoops ){ @@ -159690,11 +163967,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); sqlite3WhereClausePrint(sWLB.pWC); } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); #endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; @@ -159789,7 +164066,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif @@ -159831,6 +164108,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } } pLevel->iIdxCur = iIndexCur; assert( pIx!=0 ); @@ -159923,11 +164203,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeJumpHere(v, iOnce); } } + assert( pTabList == pWInfo->pTabList ); if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ #ifndef SQLITE_OMIT_AUTOMATIC_INDEX - constructAutomaticIndex(pParse, &pWInfo->sWC, - &pTabList->a[pLevel->iFrom], notReady, pLevel); + constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel); #endif }else{ sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); @@ -159953,8 +164233,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Jump here if malloc fails */ whereBeginError: if( pWInfo ){ - testcase( pWInfo->pExprMods!=0 ); - whereUndoExprMods(pWInfo); pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } @@ -160173,7 +164451,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } assert( pWInfo->nLevel<=pTabList->nSrc ); - if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo); for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){ int k, last; VdbeOp *pOp, *pLastOp; @@ -160227,10 +164504,28 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxEpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n", + p->iIdxCur, p->iIdxCol); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr); + } +#endif + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ - printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); } /* Proof that the "+1" on the k value above is safe */ pOp = sqlite3VdbeGetOp(v, k - 1); @@ -161105,6 +165400,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ } /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: case TK_AGG_FUNCTION: case TK_COLUMN: { int iCol = -1; @@ -161220,7 +165516,6 @@ static ExprList *exprListAppendList( for(i=0; i<pAppend->nExpr; i++){ sqlite3 *db = pParse->db; Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); - assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); break; @@ -161390,7 +165685,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ pSub = sqlite3SelectNew( pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 ); - SELECTTRACE(1,pParse,pSub, + TREETRACE(0x40,pParse,pSub, ("New window-function subquery in FROM clause of (%u/%p)\n", p->selId, p)); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); @@ -161400,6 +165695,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ if( p->pSrc ){ Table *pTab2; p->pSrc->a[0].pSelect = pSub; + p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); @@ -162491,10 +166787,9 @@ static void windowCodeRangeTest( /* This block runs if reg1 is not NULL, but reg2 is. */ sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); - if( op==OP_Gt || op==OP_Ge ){ - sqlite3VdbeChangeP2(v, -1, addrDone); - } + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, + (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); + VdbeCoverage(v); } /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). @@ -163266,8 +167561,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound <expr> */ VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ windowAggFinal(&s, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); windowReturnOneRow(&s); sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); @@ -163279,13 +167573,10 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( } if( pMWin->eStart!=TK_UNBOUNDED ){ - sqlite3VdbeAddOp2(v, OP_Rewind, s.start.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.start.csr); } - sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); - VdbeCoverageNeverTaken(v); - sqlite3VdbeAddOp2(v, OP_Rewind, s.end.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); + sqlite3VdbeAddOp1(v, OP_Rewind, s.end.csr); if( regPeer && pOrderBy ){ sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); @@ -163938,18 +168229,18 @@ typedef union { #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 576 -#define YYNRULE 405 -#define YYNRULE_WITH_ACTION 342 +#define YYNSTATE 575 +#define YYNRULE 403 +#define YYNRULE_WITH_ACTION 340 #define YYNTOKEN 185 -#define YY_MAX_SHIFT 575 -#define YY_MIN_SHIFTREDUCE 835 -#define YY_MAX_SHIFTREDUCE 1239 -#define YY_ERROR_ACTION 1240 -#define YY_ACCEPT_ACTION 1241 -#define YY_NO_ACTION 1242 -#define YY_MIN_REDUCE 1243 -#define YY_MAX_REDUCE 1647 +#define YY_MAX_SHIFT 574 +#define YY_MIN_SHIFTREDUCE 833 +#define YY_MAX_SHIFTREDUCE 1235 +#define YY_ERROR_ACTION 1236 +#define YY_ACCEPT_ACTION 1237 +#define YY_NO_ACTION 1238 +#define YY_MIN_REDUCE 1239 +#define YY_MAX_REDUCE 1641 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -164016,218 +168307,218 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2098) +#define YY_ACTTAB_COUNT (2096) static const YYACTIONTYPE yy_action[] = { /* 0 */ 568, 208, 568, 118, 115, 229, 568, 118, 115, 229, - /* 10 */ 568, 1314, 377, 1293, 408, 562, 562, 562, 568, 409, - /* 20 */ 378, 1314, 1276, 41, 41, 41, 41, 208, 1526, 71, - /* 30 */ 71, 971, 419, 41, 41, 491, 303, 279, 303, 972, - /* 40 */ 397, 71, 71, 125, 126, 80, 1217, 1217, 1050, 1053, - /* 50 */ 1040, 1040, 123, 123, 124, 124, 124, 124, 476, 409, - /* 60 */ 1241, 1, 1, 575, 2, 1245, 550, 118, 115, 229, - /* 70 */ 317, 480, 146, 480, 524, 118, 115, 229, 529, 1327, - /* 80 */ 417, 523, 142, 125, 126, 80, 1217, 1217, 1050, 1053, - /* 90 */ 1040, 1040, 123, 123, 124, 124, 124, 124, 118, 115, + /* 10 */ 568, 1310, 377, 1289, 408, 562, 562, 562, 568, 409, + /* 20 */ 378, 1310, 1272, 41, 41, 41, 41, 208, 1520, 71, + /* 30 */ 71, 969, 419, 41, 41, 491, 303, 279, 303, 970, + /* 40 */ 397, 71, 71, 125, 126, 80, 1212, 1212, 1047, 1050, + /* 50 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 476, 409, + /* 60 */ 1237, 1, 1, 574, 2, 1241, 550, 118, 115, 229, + /* 70 */ 317, 480, 146, 480, 524, 118, 115, 229, 529, 1323, + /* 80 */ 417, 523, 142, 125, 126, 80, 1212, 1212, 1047, 1050, + /* 90 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 118, 115, /* 100 */ 229, 327, 122, 122, 122, 122, 121, 121, 120, 120, /* 110 */ 120, 119, 116, 444, 284, 284, 284, 284, 442, 442, - /* 120 */ 442, 1567, 376, 1569, 1192, 375, 1163, 565, 1163, 565, - /* 130 */ 409, 1567, 537, 259, 226, 444, 101, 145, 449, 316, + /* 120 */ 442, 1561, 376, 1563, 1188, 375, 1159, 565, 1159, 565, + /* 130 */ 409, 1561, 537, 259, 226, 444, 101, 145, 449, 316, /* 140 */ 559, 240, 122, 122, 122, 122, 121, 121, 120, 120, - /* 150 */ 120, 119, 116, 444, 125, 126, 80, 1217, 1217, 1050, - /* 160 */ 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, 142, - /* 170 */ 294, 1192, 339, 448, 120, 120, 120, 119, 116, 444, - /* 180 */ 127, 1192, 1193, 1194, 148, 441, 440, 568, 119, 116, + /* 150 */ 120, 119, 116, 444, 125, 126, 80, 1212, 1212, 1047, + /* 160 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 142, + /* 170 */ 294, 1188, 339, 448, 120, 120, 120, 119, 116, 444, + /* 180 */ 127, 1188, 1189, 1188, 148, 441, 440, 568, 119, 116, /* 190 */ 444, 124, 124, 124, 124, 117, 122, 122, 122, 122, /* 200 */ 121, 121, 120, 120, 120, 119, 116, 444, 454, 113, /* 210 */ 13, 13, 546, 122, 122, 122, 122, 121, 121, 120, - /* 220 */ 120, 120, 119, 116, 444, 422, 316, 559, 1192, 1193, - /* 230 */ 1194, 149, 1224, 409, 1224, 124, 124, 124, 124, 122, + /* 220 */ 120, 120, 119, 116, 444, 422, 316, 559, 1188, 1189, + /* 230 */ 1188, 149, 1220, 409, 1220, 124, 124, 124, 124, 122, /* 240 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 250 */ 444, 465, 342, 1037, 1037, 1051, 1054, 125, 126, 80, - /* 260 */ 1217, 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, - /* 270 */ 124, 124, 1279, 522, 222, 1192, 568, 409, 224, 514, + /* 250 */ 444, 465, 342, 1034, 1034, 1048, 1051, 125, 126, 80, + /* 260 */ 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, + /* 270 */ 124, 124, 1275, 522, 222, 1188, 568, 409, 224, 514, /* 280 */ 175, 82, 83, 122, 122, 122, 122, 121, 121, 120, - /* 290 */ 120, 120, 119, 116, 444, 1007, 16, 16, 1192, 133, - /* 300 */ 133, 125, 126, 80, 1217, 1217, 1050, 1053, 1040, 1040, + /* 290 */ 120, 120, 119, 116, 444, 1005, 16, 16, 1188, 133, + /* 300 */ 133, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, /* 310 */ 123, 123, 124, 124, 124, 124, 122, 122, 122, 122, - /* 320 */ 121, 121, 120, 120, 120, 119, 116, 444, 1041, 546, - /* 330 */ 1192, 373, 1192, 1193, 1194, 252, 1434, 399, 504, 501, - /* 340 */ 500, 111, 560, 566, 4, 926, 926, 433, 499, 340, - /* 350 */ 460, 328, 360, 394, 1237, 1192, 1193, 1194, 563, 568, + /* 320 */ 121, 121, 120, 120, 120, 119, 116, 444, 1038, 546, + /* 330 */ 1188, 373, 1188, 1189, 1188, 252, 1429, 399, 504, 501, + /* 340 */ 500, 111, 560, 566, 4, 924, 924, 433, 499, 340, + /* 350 */ 460, 328, 360, 394, 1233, 1188, 1189, 1188, 563, 568, /* 360 */ 122, 122, 122, 122, 121, 121, 120, 120, 120, 119, - /* 370 */ 116, 444, 284, 284, 369, 1580, 1607, 441, 440, 154, - /* 380 */ 409, 445, 71, 71, 1286, 565, 1221, 1192, 1193, 1194, - /* 390 */ 85, 1223, 271, 557, 543, 515, 1561, 568, 98, 1222, - /* 400 */ 6, 1278, 472, 142, 125, 126, 80, 1217, 1217, 1050, - /* 410 */ 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, 550, - /* 420 */ 13, 13, 1027, 507, 1224, 1192, 1224, 549, 109, 109, - /* 430 */ 222, 568, 1238, 175, 568, 427, 110, 197, 445, 570, - /* 440 */ 569, 430, 1552, 1017, 325, 551, 1192, 270, 287, 368, + /* 370 */ 116, 444, 284, 284, 369, 1574, 1600, 441, 440, 154, + /* 380 */ 409, 445, 71, 71, 1282, 565, 1217, 1188, 1189, 1188, + /* 390 */ 85, 1219, 271, 557, 543, 515, 1555, 568, 98, 1218, + /* 400 */ 6, 1274, 472, 142, 125, 126, 80, 1212, 1212, 1047, + /* 410 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 550, + /* 420 */ 13, 13, 1024, 507, 1220, 1188, 1220, 549, 109, 109, + /* 430 */ 222, 568, 1234, 175, 568, 427, 110, 197, 445, 569, + /* 440 */ 445, 430, 1546, 1014, 325, 551, 1188, 270, 287, 368, /* 450 */ 510, 363, 509, 257, 71, 71, 543, 71, 71, 359, - /* 460 */ 316, 559, 1613, 122, 122, 122, 122, 121, 121, 120, - /* 470 */ 120, 120, 119, 116, 444, 1017, 1017, 1019, 1020, 27, - /* 480 */ 284, 284, 1192, 1193, 1194, 1158, 568, 1612, 409, 901, - /* 490 */ 190, 550, 356, 565, 550, 937, 533, 517, 1158, 516, - /* 500 */ 413, 1158, 552, 1192, 1193, 1194, 568, 544, 1554, 51, - /* 510 */ 51, 214, 125, 126, 80, 1217, 1217, 1050, 1053, 1040, - /* 520 */ 1040, 123, 123, 124, 124, 124, 124, 1192, 474, 135, - /* 530 */ 135, 409, 284, 284, 1490, 505, 121, 121, 120, 120, - /* 540 */ 120, 119, 116, 444, 1007, 565, 518, 217, 541, 1561, - /* 550 */ 316, 559, 142, 6, 532, 125, 126, 80, 1217, 1217, - /* 560 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 570 */ 1555, 122, 122, 122, 122, 121, 121, 120, 120, 120, - /* 580 */ 119, 116, 444, 485, 1192, 1193, 1194, 482, 281, 1267, - /* 590 */ 957, 252, 1192, 373, 504, 501, 500, 1192, 340, 571, - /* 600 */ 1192, 571, 409, 292, 499, 957, 876, 191, 480, 316, + /* 460 */ 316, 559, 1606, 122, 122, 122, 122, 121, 121, 120, + /* 470 */ 120, 120, 119, 116, 444, 1014, 1014, 1016, 1017, 27, + /* 480 */ 284, 284, 1188, 1189, 1188, 1154, 568, 1605, 409, 899, + /* 490 */ 190, 550, 356, 565, 550, 935, 533, 517, 1154, 516, + /* 500 */ 413, 1154, 552, 1188, 1189, 1188, 568, 544, 1548, 51, + /* 510 */ 51, 214, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, + /* 520 */ 1037, 123, 123, 124, 124, 124, 124, 1188, 474, 135, + /* 530 */ 135, 409, 284, 284, 1484, 505, 121, 121, 120, 120, + /* 540 */ 120, 119, 116, 444, 1005, 565, 518, 217, 541, 1555, + /* 550 */ 316, 559, 142, 6, 532, 125, 126, 80, 1212, 1212, + /* 560 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 570 */ 1549, 122, 122, 122, 122, 121, 121, 120, 120, 120, + /* 580 */ 119, 116, 444, 485, 1188, 1189, 1188, 482, 281, 1263, + /* 590 */ 955, 252, 1188, 373, 504, 501, 500, 1188, 340, 570, + /* 600 */ 1188, 570, 409, 292, 499, 955, 874, 191, 480, 316, /* 610 */ 559, 384, 290, 380, 122, 122, 122, 122, 121, 121, - /* 620 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 630 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 640 */ 124, 409, 394, 1136, 1192, 869, 100, 284, 284, 1192, - /* 650 */ 1193, 1194, 373, 1093, 1192, 1193, 1194, 1192, 1193, 1194, - /* 660 */ 565, 455, 32, 373, 233, 125, 126, 80, 1217, 1217, - /* 670 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 680 */ 1433, 959, 568, 228, 958, 122, 122, 122, 122, 121, - /* 690 */ 121, 120, 120, 120, 119, 116, 444, 1158, 228, 1192, - /* 700 */ 157, 1192, 1193, 1194, 1553, 13, 13, 301, 957, 1232, - /* 710 */ 1158, 153, 409, 1158, 373, 1583, 1176, 5, 369, 1580, - /* 720 */ 429, 1238, 3, 957, 122, 122, 122, 122, 121, 121, - /* 730 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 740 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 750 */ 124, 409, 208, 567, 1192, 1028, 1192, 1193, 1194, 1192, - /* 760 */ 388, 852, 155, 1552, 286, 402, 1098, 1098, 488, 568, - /* 770 */ 465, 342, 1319, 1319, 1552, 125, 126, 80, 1217, 1217, - /* 780 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, + /* 620 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 630 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 640 */ 124, 409, 394, 1132, 1188, 867, 100, 284, 284, 1188, + /* 650 */ 1189, 1188, 373, 1089, 1188, 1189, 1188, 1188, 1189, 1188, + /* 660 */ 565, 455, 32, 373, 233, 125, 126, 80, 1212, 1212, + /* 670 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 680 */ 1428, 957, 568, 228, 956, 122, 122, 122, 122, 121, + /* 690 */ 121, 120, 120, 120, 119, 116, 444, 1154, 228, 1188, + /* 700 */ 157, 1188, 1189, 1188, 1547, 13, 13, 301, 955, 1228, + /* 710 */ 1154, 153, 409, 1154, 373, 1577, 1172, 5, 369, 1574, + /* 720 */ 429, 1234, 3, 955, 122, 122, 122, 122, 121, 121, + /* 730 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 740 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 750 */ 124, 409, 208, 567, 1188, 1025, 1188, 1189, 1188, 1188, + /* 760 */ 388, 850, 155, 1546, 286, 402, 1094, 1094, 488, 568, + /* 770 */ 465, 342, 1315, 1315, 1546, 125, 126, 80, 1212, 1212, + /* 780 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, /* 790 */ 129, 568, 13, 13, 374, 122, 122, 122, 122, 121, /* 800 */ 121, 120, 120, 120, 119, 116, 444, 302, 568, 453, - /* 810 */ 528, 1192, 1193, 1194, 13, 13, 1192, 1193, 1194, 1297, - /* 820 */ 463, 1267, 409, 1317, 1317, 1552, 1012, 453, 452, 200, - /* 830 */ 299, 71, 71, 1265, 122, 122, 122, 122, 121, 121, - /* 840 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 850 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 860 */ 124, 409, 227, 1073, 1158, 284, 284, 419, 312, 278, - /* 870 */ 278, 285, 285, 1419, 406, 405, 382, 1158, 565, 568, - /* 880 */ 1158, 1196, 565, 1600, 565, 125, 126, 80, 1217, 1217, - /* 890 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 900 */ 453, 1482, 13, 13, 1536, 122, 122, 122, 122, 121, + /* 810 */ 528, 1188, 1189, 1188, 13, 13, 1188, 1189, 1188, 1293, + /* 820 */ 463, 1263, 409, 1313, 1313, 1546, 1010, 453, 452, 200, + /* 830 */ 299, 71, 71, 1261, 122, 122, 122, 122, 121, 121, + /* 840 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 850 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 860 */ 124, 409, 227, 1069, 1154, 284, 284, 419, 312, 278, + /* 870 */ 278, 285, 285, 1415, 406, 405, 382, 1154, 565, 568, + /* 880 */ 1154, 1191, 565, 1594, 565, 125, 126, 80, 1212, 1212, + /* 890 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 900 */ 453, 1476, 13, 13, 1530, 122, 122, 122, 122, 121, /* 910 */ 121, 120, 120, 120, 119, 116, 444, 201, 568, 354, - /* 920 */ 1586, 575, 2, 1245, 840, 841, 842, 1562, 317, 1212, - /* 930 */ 146, 6, 409, 255, 254, 253, 206, 1327, 9, 1196, + /* 920 */ 1580, 574, 2, 1241, 838, 839, 840, 1556, 317, 1207, + /* 930 */ 146, 6, 409, 255, 254, 253, 206, 1323, 9, 1191, /* 940 */ 262, 71, 71, 424, 122, 122, 122, 122, 121, 121, - /* 950 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 960 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 970 */ 124, 568, 284, 284, 568, 1213, 409, 574, 313, 1245, - /* 980 */ 349, 1296, 352, 419, 317, 565, 146, 491, 525, 1643, - /* 990 */ 395, 371, 491, 1327, 70, 70, 1295, 71, 71, 240, - /* 1000 */ 1325, 104, 80, 1217, 1217, 1050, 1053, 1040, 1040, 123, + /* 950 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 960 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 970 */ 124, 568, 284, 284, 568, 1208, 409, 573, 313, 1241, + /* 980 */ 349, 1292, 352, 419, 317, 565, 146, 491, 525, 1637, + /* 990 */ 395, 371, 491, 1323, 70, 70, 1291, 71, 71, 240, + /* 1000 */ 1321, 104, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, /* 1010 */ 123, 124, 124, 124, 124, 122, 122, 122, 122, 121, - /* 1020 */ 121, 120, 120, 120, 119, 116, 444, 1114, 284, 284, - /* 1030 */ 428, 448, 1525, 1213, 439, 284, 284, 1489, 1352, 311, - /* 1040 */ 474, 565, 1115, 971, 491, 491, 217, 1263, 565, 1538, - /* 1050 */ 568, 972, 207, 568, 1027, 240, 383, 1116, 519, 122, + /* 1020 */ 121, 120, 120, 120, 119, 116, 444, 1110, 284, 284, + /* 1030 */ 428, 448, 1519, 1208, 439, 284, 284, 1483, 1348, 311, + /* 1040 */ 474, 565, 1111, 969, 491, 491, 217, 1259, 565, 1532, + /* 1050 */ 568, 970, 207, 568, 1024, 240, 383, 1112, 519, 122, /* 1060 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 1070 */ 444, 1018, 107, 71, 71, 1017, 13, 13, 912, 568, - /* 1080 */ 1495, 568, 284, 284, 97, 526, 491, 448, 913, 1326, - /* 1090 */ 1322, 545, 409, 284, 284, 565, 151, 209, 1495, 1497, - /* 1100 */ 262, 450, 55, 55, 56, 56, 565, 1017, 1017, 1019, - /* 1110 */ 443, 332, 409, 527, 12, 295, 125, 126, 80, 1217, - /* 1120 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1130 */ 124, 347, 409, 864, 1534, 1213, 125, 126, 80, 1217, - /* 1140 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1150 */ 124, 1137, 1641, 474, 1641, 371, 125, 114, 80, 1217, - /* 1160 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1170 */ 124, 1495, 329, 474, 331, 122, 122, 122, 122, 121, - /* 1180 */ 121, 120, 120, 120, 119, 116, 444, 203, 1419, 568, - /* 1190 */ 1294, 864, 464, 1213, 436, 122, 122, 122, 122, 121, - /* 1200 */ 121, 120, 120, 120, 119, 116, 444, 553, 1137, 1642, - /* 1210 */ 539, 1642, 15, 15, 892, 122, 122, 122, 122, 121, + /* 1070 */ 444, 1015, 107, 71, 71, 1014, 13, 13, 910, 568, + /* 1080 */ 1489, 568, 284, 284, 97, 526, 491, 448, 911, 1322, + /* 1090 */ 1318, 545, 409, 284, 284, 565, 151, 209, 1489, 1491, + /* 1100 */ 262, 450, 55, 55, 56, 56, 565, 1014, 1014, 1016, + /* 1110 */ 443, 332, 409, 527, 12, 295, 125, 126, 80, 1212, + /* 1120 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1130 */ 124, 347, 409, 862, 1528, 1208, 125, 126, 80, 1212, + /* 1140 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1150 */ 124, 1133, 1635, 474, 1635, 371, 125, 114, 80, 1212, + /* 1160 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1170 */ 124, 1489, 329, 474, 331, 122, 122, 122, 122, 121, + /* 1180 */ 121, 120, 120, 120, 119, 116, 444, 203, 1415, 568, + /* 1190 */ 1290, 862, 464, 1208, 436, 122, 122, 122, 122, 121, + /* 1200 */ 121, 120, 120, 120, 119, 116, 444, 553, 1133, 1636, + /* 1210 */ 539, 1636, 15, 15, 890, 122, 122, 122, 122, 121, /* 1220 */ 121, 120, 120, 120, 119, 116, 444, 568, 298, 538, - /* 1230 */ 1135, 1419, 1559, 1560, 1331, 409, 6, 6, 1169, 1268, - /* 1240 */ 415, 320, 284, 284, 1419, 508, 565, 525, 300, 457, - /* 1250 */ 43, 43, 568, 893, 12, 565, 330, 478, 425, 407, - /* 1260 */ 126, 80, 1217, 1217, 1050, 1053, 1040, 1040, 123, 123, - /* 1270 */ 124, 124, 124, 124, 568, 57, 57, 288, 1192, 1419, - /* 1280 */ 496, 458, 392, 392, 391, 273, 389, 1135, 1558, 849, - /* 1290 */ 1169, 407, 6, 568, 321, 1158, 470, 44, 44, 1557, - /* 1300 */ 1114, 426, 234, 6, 323, 256, 540, 256, 1158, 431, - /* 1310 */ 568, 1158, 322, 17, 487, 1115, 58, 58, 122, 122, + /* 1230 */ 1131, 1415, 1553, 1554, 1327, 409, 6, 6, 1165, 1264, + /* 1240 */ 415, 320, 284, 284, 1415, 508, 565, 525, 300, 457, + /* 1250 */ 43, 43, 568, 891, 12, 565, 330, 478, 425, 407, + /* 1260 */ 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, + /* 1270 */ 124, 124, 124, 124, 568, 57, 57, 288, 1188, 1415, + /* 1280 */ 496, 458, 392, 392, 391, 273, 389, 1131, 1552, 847, + /* 1290 */ 1165, 407, 6, 568, 321, 1154, 470, 44, 44, 1551, + /* 1300 */ 1110, 426, 234, 6, 323, 256, 540, 256, 1154, 431, + /* 1310 */ 568, 1154, 322, 17, 487, 1111, 58, 58, 122, 122, /* 1320 */ 122, 122, 121, 121, 120, 120, 120, 119, 116, 444, - /* 1330 */ 1116, 216, 481, 59, 59, 1192, 1193, 1194, 111, 560, + /* 1330 */ 1112, 216, 481, 59, 59, 1188, 1189, 1188, 111, 560, /* 1340 */ 324, 4, 236, 456, 526, 568, 237, 456, 568, 437, - /* 1350 */ 168, 556, 420, 141, 479, 563, 568, 293, 568, 1095, - /* 1360 */ 568, 293, 568, 1095, 531, 568, 872, 8, 60, 60, + /* 1350 */ 168, 556, 420, 141, 479, 563, 568, 293, 568, 1091, + /* 1360 */ 568, 293, 568, 1091, 531, 568, 870, 8, 60, 60, /* 1370 */ 235, 61, 61, 568, 414, 568, 414, 568, 445, 62, /* 1380 */ 62, 45, 45, 46, 46, 47, 47, 199, 49, 49, /* 1390 */ 557, 568, 359, 568, 100, 486, 50, 50, 63, 63, - /* 1400 */ 64, 64, 561, 415, 535, 410, 568, 1027, 568, 534, - /* 1410 */ 316, 559, 316, 559, 65, 65, 14, 14, 568, 1027, - /* 1420 */ 568, 512, 932, 872, 1018, 109, 109, 931, 1017, 66, - /* 1430 */ 66, 131, 131, 110, 451, 445, 570, 569, 416, 177, - /* 1440 */ 1017, 132, 132, 67, 67, 568, 467, 568, 932, 471, - /* 1450 */ 1364, 283, 226, 931, 315, 1363, 407, 568, 459, 407, - /* 1460 */ 1017, 1017, 1019, 239, 407, 86, 213, 1350, 52, 52, - /* 1470 */ 68, 68, 1017, 1017, 1019, 1020, 27, 1585, 1180, 447, - /* 1480 */ 69, 69, 288, 97, 108, 1541, 106, 392, 392, 391, - /* 1490 */ 273, 389, 568, 879, 849, 883, 568, 111, 560, 466, - /* 1500 */ 4, 568, 152, 30, 38, 568, 1132, 234, 396, 323, + /* 1400 */ 64, 64, 561, 415, 535, 410, 568, 1024, 568, 534, + /* 1410 */ 316, 559, 316, 559, 65, 65, 14, 14, 568, 1024, + /* 1420 */ 568, 512, 930, 870, 1015, 109, 109, 929, 1014, 66, + /* 1430 */ 66, 131, 131, 110, 451, 445, 569, 445, 416, 177, + /* 1440 */ 1014, 132, 132, 67, 67, 568, 467, 568, 930, 471, + /* 1450 */ 1360, 283, 226, 929, 315, 1359, 407, 568, 459, 407, + /* 1460 */ 1014, 1014, 1016, 239, 407, 86, 213, 1346, 52, 52, + /* 1470 */ 68, 68, 1014, 1014, 1016, 1017, 27, 1579, 1176, 447, + /* 1480 */ 69, 69, 288, 97, 108, 1535, 106, 392, 392, 391, + /* 1490 */ 273, 389, 568, 877, 847, 881, 568, 111, 560, 466, + /* 1500 */ 4, 568, 152, 30, 38, 568, 1128, 234, 396, 323, /* 1510 */ 111, 560, 527, 4, 563, 53, 53, 322, 568, 163, /* 1520 */ 163, 568, 337, 468, 164, 164, 333, 563, 76, 76, - /* 1530 */ 568, 289, 1514, 568, 31, 1513, 568, 445, 338, 483, - /* 1540 */ 100, 54, 54, 344, 72, 72, 296, 236, 1080, 557, - /* 1550 */ 445, 879, 1360, 134, 134, 168, 73, 73, 141, 161, - /* 1560 */ 161, 1574, 557, 535, 568, 319, 568, 348, 536, 1009, - /* 1570 */ 473, 261, 261, 891, 890, 235, 535, 568, 1027, 568, + /* 1530 */ 568, 289, 1508, 568, 31, 1507, 568, 445, 338, 483, + /* 1540 */ 100, 54, 54, 344, 72, 72, 296, 236, 1076, 557, + /* 1550 */ 445, 877, 1356, 134, 134, 168, 73, 73, 141, 161, + /* 1560 */ 161, 1568, 557, 535, 568, 319, 568, 348, 536, 1007, + /* 1570 */ 473, 261, 261, 889, 888, 235, 535, 568, 1024, 568, /* 1580 */ 475, 534, 261, 367, 109, 109, 521, 136, 136, 130, - /* 1590 */ 130, 1027, 110, 366, 445, 570, 569, 109, 109, 1017, - /* 1600 */ 162, 162, 156, 156, 568, 110, 1080, 445, 570, 569, - /* 1610 */ 410, 351, 1017, 568, 353, 316, 559, 568, 343, 568, - /* 1620 */ 100, 497, 357, 258, 100, 898, 899, 140, 140, 355, - /* 1630 */ 1310, 1017, 1017, 1019, 1020, 27, 139, 139, 362, 451, - /* 1640 */ 137, 137, 138, 138, 1017, 1017, 1019, 1020, 27, 1180, - /* 1650 */ 447, 568, 372, 288, 111, 560, 1021, 4, 392, 392, - /* 1660 */ 391, 273, 389, 568, 1141, 849, 568, 1076, 568, 258, - /* 1670 */ 492, 563, 568, 211, 75, 75, 555, 962, 234, 261, - /* 1680 */ 323, 111, 560, 929, 4, 113, 77, 77, 322, 74, - /* 1690 */ 74, 42, 42, 1373, 445, 48, 48, 1418, 563, 974, - /* 1700 */ 975, 1092, 1091, 1092, 1091, 862, 557, 150, 930, 1346, - /* 1710 */ 113, 1358, 554, 1424, 1021, 1275, 1266, 1254, 236, 1253, - /* 1720 */ 1255, 445, 1593, 1343, 308, 276, 168, 309, 11, 141, - /* 1730 */ 393, 310, 232, 557, 1405, 1027, 335, 291, 1400, 219, - /* 1740 */ 336, 109, 109, 936, 297, 1410, 235, 341, 477, 110, - /* 1750 */ 502, 445, 570, 569, 1393, 1409, 1017, 400, 1293, 365, - /* 1760 */ 223, 1486, 1027, 1485, 1355, 1356, 1354, 1353, 109, 109, - /* 1770 */ 204, 1596, 1232, 558, 265, 218, 110, 205, 445, 570, - /* 1780 */ 569, 410, 387, 1017, 1533, 179, 316, 559, 1017, 1017, - /* 1790 */ 1019, 1020, 27, 230, 1531, 1229, 79, 560, 85, 4, - /* 1800 */ 418, 215, 548, 81, 84, 188, 1406, 173, 181, 461, - /* 1810 */ 451, 35, 462, 563, 183, 1017, 1017, 1019, 1020, 27, - /* 1820 */ 184, 1491, 185, 186, 495, 242, 98, 398, 1412, 36, - /* 1830 */ 1411, 484, 91, 469, 401, 1414, 445, 192, 1480, 246, - /* 1840 */ 1502, 490, 346, 277, 248, 196, 493, 511, 557, 350, - /* 1850 */ 1256, 249, 250, 403, 1313, 1312, 111, 560, 432, 4, - /* 1860 */ 1311, 1304, 93, 1611, 883, 1610, 224, 404, 434, 520, - /* 1870 */ 263, 435, 1579, 563, 1283, 1282, 364, 1027, 306, 1281, - /* 1880 */ 264, 1609, 1565, 109, 109, 370, 1303, 307, 1564, 438, - /* 1890 */ 128, 110, 1378, 445, 570, 569, 445, 546, 1017, 10, - /* 1900 */ 1466, 105, 381, 1377, 34, 572, 99, 1336, 557, 314, - /* 1910 */ 1186, 530, 272, 274, 379, 210, 1335, 547, 385, 386, - /* 1920 */ 275, 573, 1251, 1246, 411, 412, 1518, 165, 178, 1519, - /* 1930 */ 1017, 1017, 1019, 1020, 27, 1517, 1516, 1027, 78, 147, - /* 1940 */ 166, 220, 221, 109, 109, 836, 304, 167, 446, 212, - /* 1950 */ 318, 110, 231, 445, 570, 569, 144, 1090, 1017, 1088, - /* 1960 */ 326, 180, 169, 1212, 182, 334, 238, 915, 241, 1104, + /* 1590 */ 130, 1024, 110, 366, 445, 569, 445, 109, 109, 1014, + /* 1600 */ 162, 162, 156, 156, 568, 110, 1076, 445, 569, 445, + /* 1610 */ 410, 351, 1014, 568, 353, 316, 559, 568, 343, 568, + /* 1620 */ 100, 497, 357, 258, 100, 896, 897, 140, 140, 355, + /* 1630 */ 1306, 1014, 1014, 1016, 1017, 27, 139, 139, 362, 451, + /* 1640 */ 137, 137, 138, 138, 1014, 1014, 1016, 1017, 27, 1176, + /* 1650 */ 447, 568, 372, 288, 111, 560, 1018, 4, 392, 392, + /* 1660 */ 391, 273, 389, 568, 1137, 847, 568, 1072, 568, 258, + /* 1670 */ 492, 563, 568, 211, 75, 75, 555, 960, 234, 261, + /* 1680 */ 323, 111, 560, 927, 4, 113, 77, 77, 322, 74, + /* 1690 */ 74, 42, 42, 1369, 445, 48, 48, 1414, 563, 972, + /* 1700 */ 973, 1088, 1087, 1088, 1087, 860, 557, 150, 928, 1342, + /* 1710 */ 113, 1354, 554, 1419, 1018, 1271, 1262, 1250, 236, 1249, + /* 1720 */ 1251, 445, 1587, 1339, 308, 276, 168, 309, 11, 141, + /* 1730 */ 393, 310, 232, 557, 1401, 1024, 335, 291, 1396, 219, + /* 1740 */ 336, 109, 109, 934, 297, 1406, 235, 341, 477, 110, + /* 1750 */ 502, 445, 569, 445, 1389, 1405, 1014, 400, 1289, 365, + /* 1760 */ 223, 1480, 1024, 1479, 1351, 1352, 1350, 1349, 109, 109, + /* 1770 */ 204, 1590, 1228, 558, 265, 218, 110, 205, 445, 569, + /* 1780 */ 445, 410, 387, 1014, 1527, 179, 316, 559, 1014, 1014, + /* 1790 */ 1016, 1017, 27, 230, 1525, 1225, 79, 560, 85, 4, + /* 1800 */ 418, 215, 548, 81, 84, 188, 1402, 173, 181, 461, + /* 1810 */ 451, 35, 462, 563, 183, 1014, 1014, 1016, 1017, 27, + /* 1820 */ 184, 1485, 185, 186, 495, 242, 98, 398, 1408, 36, + /* 1830 */ 1407, 484, 91, 469, 401, 1410, 445, 192, 1474, 246, + /* 1840 */ 1496, 490, 346, 277, 248, 196, 493, 511, 557, 350, + /* 1850 */ 1252, 249, 250, 403, 1309, 1308, 111, 560, 432, 4, + /* 1860 */ 1307, 1300, 93, 1604, 881, 1603, 224, 404, 434, 520, + /* 1870 */ 263, 435, 1573, 563, 1279, 1278, 364, 1024, 306, 1277, + /* 1880 */ 264, 1602, 1559, 109, 109, 370, 1299, 307, 1558, 438, + /* 1890 */ 128, 110, 1374, 445, 569, 445, 445, 546, 1014, 10, + /* 1900 */ 1461, 105, 381, 1373, 34, 571, 99, 1332, 557, 314, + /* 1910 */ 1182, 530, 272, 274, 379, 210, 1331, 547, 385, 386, + /* 1920 */ 275, 572, 1247, 1242, 411, 412, 1512, 165, 178, 1513, + /* 1930 */ 1014, 1014, 1016, 1017, 27, 1511, 1510, 1024, 78, 147, + /* 1940 */ 166, 220, 221, 109, 109, 834, 304, 167, 446, 212, + /* 1950 */ 318, 110, 231, 445, 569, 445, 144, 1086, 1014, 1084, + /* 1960 */ 326, 180, 169, 1207, 182, 334, 238, 913, 241, 1100, /* 1970 */ 187, 170, 171, 421, 87, 88, 423, 189, 89, 90, - /* 1980 */ 172, 1107, 243, 1103, 244, 158, 18, 245, 345, 247, - /* 1990 */ 1017, 1017, 1019, 1020, 27, 261, 1096, 193, 1226, 489, - /* 2000 */ 194, 37, 366, 851, 494, 251, 195, 506, 92, 19, - /* 2010 */ 498, 358, 20, 503, 881, 361, 94, 894, 305, 159, - /* 2020 */ 513, 39, 95, 1174, 160, 1056, 966, 1143, 96, 174, - /* 2030 */ 1142, 225, 280, 282, 198, 960, 113, 1164, 1160, 260, - /* 2040 */ 21, 22, 23, 1162, 1168, 1167, 1148, 24, 33, 25, - /* 2050 */ 202, 542, 26, 100, 1071, 102, 1057, 103, 7, 1055, - /* 2060 */ 1059, 1113, 1060, 1112, 266, 267, 28, 40, 390, 1022, - /* 2070 */ 863, 112, 29, 564, 1182, 1181, 268, 176, 143, 925, - /* 2080 */ 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, - /* 2090 */ 1242, 1242, 1242, 1242, 269, 1602, 1242, 1601, + /* 1980 */ 172, 1103, 243, 1099, 244, 158, 18, 245, 345, 247, + /* 1990 */ 1014, 1014, 1016, 1017, 27, 261, 1092, 193, 1222, 489, + /* 2000 */ 194, 37, 366, 849, 494, 251, 195, 506, 92, 19, + /* 2010 */ 498, 358, 20, 503, 879, 361, 94, 892, 305, 159, + /* 2020 */ 513, 39, 95, 1170, 160, 1053, 964, 1139, 96, 174, + /* 2030 */ 1138, 225, 280, 282, 198, 958, 113, 1160, 1156, 260, + /* 2040 */ 21, 22, 23, 1158, 1164, 1163, 1144, 24, 33, 25, + /* 2050 */ 202, 542, 26, 100, 1067, 102, 1054, 103, 7, 1052, + /* 2060 */ 1056, 1109, 1057, 1108, 266, 267, 28, 40, 390, 1019, + /* 2070 */ 861, 112, 29, 564, 1178, 1177, 268, 176, 143, 923, + /* 2080 */ 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, + /* 2090 */ 1238, 1238, 1238, 1238, 269, 1595, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, @@ -164439,7 +168730,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 2060 */ 23, 23, 11, 23, 25, 22, 22, 22, 15, 23, /* 2070 */ 23, 22, 22, 25, 1, 1, 141, 25, 23, 135, /* 2080 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2090 */ 319, 319, 319, 319, 141, 141, 319, 141, 319, 319, + /* 2090 */ 319, 319, 319, 319, 141, 141, 319, 319, 319, 319, /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, @@ -164458,9 +168749,9 @@ static const YYCODETYPE yy_lookahead[] = { /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, 319, 319, + /* 2280 */ 319, }; -#define YY_SHIFT_COUNT (575) +#define YY_SHIFT_COUNT (574) #define YY_SHIFT_MIN (0) #define YY_SHIFT_MAX (2074) static const unsigned short int yy_shift_ofst[] = { @@ -164480,12 +168771,12 @@ static const unsigned short int yy_shift_ofst[] = { /* 130 */ 137, 181, 181, 181, 181, 181, 181, 181, 94, 430, /* 140 */ 66, 65, 112, 366, 533, 533, 740, 1261, 533, 533, /* 150 */ 79, 79, 533, 412, 412, 412, 77, 412, 123, 113, - /* 160 */ 113, 22, 22, 2098, 2098, 328, 328, 328, 239, 468, + /* 160 */ 113, 22, 22, 2096, 2096, 328, 328, 328, 239, 468, /* 170 */ 468, 468, 468, 1015, 1015, 409, 366, 1129, 1186, 533, /* 180 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 969, /* 200 */ 621, 621, 533, 642, 788, 788, 1228, 1228, 822, 822, - /* 210 */ 67, 1274, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 1307, + /* 210 */ 67, 1274, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 1307, /* 220 */ 954, 954, 585, 472, 640, 387, 695, 538, 541, 700, /* 230 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, /* 240 */ 222, 533, 533, 533, 533, 533, 533, 533, 533, 533, @@ -164503,8 +168794,8 @@ static const unsigned short int yy_shift_ofst[] = { /* 360 */ 1840, 1840, 1823, 1732, 1738, 1732, 1794, 1732, 1732, 1701, /* 370 */ 1844, 1758, 1758, 1823, 1633, 1789, 1789, 1807, 1807, 1742, /* 380 */ 1752, 1877, 1633, 1743, 1742, 1759, 1765, 1677, 1879, 1897, - /* 390 */ 1897, 1914, 1914, 1914, 2098, 2098, 2098, 2098, 2098, 2098, - /* 400 */ 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 207, + /* 390 */ 1897, 1914, 1914, 1914, 2096, 2096, 2096, 2096, 2096, 2096, + /* 400 */ 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 207, /* 410 */ 1095, 331, 620, 903, 806, 1074, 1483, 1432, 1481, 1322, /* 420 */ 1370, 1394, 1515, 1291, 1546, 1547, 1557, 1595, 1598, 1599, /* 430 */ 1434, 1453, 1618, 1462, 1567, 1489, 1644, 1654, 1616, 1660, @@ -164521,7 +168812,7 @@ static const unsigned short int yy_shift_ofst[] = { /* 540 */ 2015, 2023, 2026, 2027, 2025, 2028, 2018, 1913, 1915, 2031, /* 550 */ 2011, 2033, 2036, 2037, 2038, 2039, 2040, 2043, 2051, 2044, /* 560 */ 2045, 2046, 2047, 2049, 2050, 2048, 1944, 1935, 1953, 1954, - /* 570 */ 1956, 2052, 2055, 2053, 2073, 2074, + /* 570 */ 2052, 2055, 2053, 2073, 2074, }; #define YY_REDUCE_COUNT (408) #define YY_REDUCE_MIN (-271) @@ -164570,64 +168861,64 @@ static const short yy_reduce_ofst[] = { /* 400 */ 1722, 1723, 1733, 1717, 1724, 1727, 1728, 1725, 1740, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1647, 1647, 1647, 1475, 1240, 1351, 1240, 1240, 1240, 1475, - /* 10 */ 1475, 1475, 1240, 1381, 1381, 1528, 1273, 1240, 1240, 1240, - /* 20 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1474, 1240, 1240, - /* 30 */ 1240, 1240, 1563, 1563, 1240, 1240, 1240, 1240, 1240, 1240, - /* 40 */ 1240, 1240, 1390, 1240, 1397, 1240, 1240, 1240, 1240, 1240, - /* 50 */ 1476, 1477, 1240, 1240, 1240, 1527, 1529, 1492, 1404, 1403, - /* 60 */ 1402, 1401, 1510, 1369, 1395, 1388, 1392, 1470, 1471, 1469, - /* 70 */ 1473, 1477, 1476, 1240, 1391, 1438, 1454, 1437, 1240, 1240, - /* 80 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 90 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 100 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 110 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 120 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 130 */ 1446, 1453, 1452, 1451, 1460, 1450, 1447, 1440, 1439, 1441, - /* 140 */ 1442, 1240, 1240, 1264, 1240, 1240, 1261, 1315, 1240, 1240, - /* 150 */ 1240, 1240, 1240, 1547, 1546, 1240, 1443, 1240, 1273, 1432, - /* 160 */ 1431, 1457, 1444, 1456, 1455, 1535, 1599, 1598, 1493, 1240, - /* 170 */ 1240, 1240, 1240, 1240, 1240, 1563, 1240, 1240, 1240, 1240, - /* 180 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 190 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1371, - /* 200 */ 1563, 1563, 1240, 1273, 1563, 1563, 1372, 1372, 1269, 1269, - /* 210 */ 1375, 1240, 1542, 1342, 1342, 1342, 1342, 1351, 1342, 1240, - /* 220 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 230 */ 1240, 1240, 1240, 1240, 1532, 1530, 1240, 1240, 1240, 1240, - /* 240 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 250 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 260 */ 1240, 1240, 1240, 1347, 1240, 1240, 1240, 1240, 1240, 1240, - /* 270 */ 1240, 1240, 1240, 1240, 1240, 1592, 1240, 1505, 1329, 1347, - /* 280 */ 1347, 1347, 1347, 1349, 1330, 1328, 1341, 1274, 1247, 1639, - /* 290 */ 1407, 1396, 1348, 1396, 1636, 1394, 1407, 1407, 1394, 1407, - /* 300 */ 1348, 1636, 1290, 1615, 1285, 1381, 1381, 1381, 1371, 1371, - /* 310 */ 1371, 1371, 1375, 1375, 1472, 1348, 1341, 1240, 1639, 1639, - /* 320 */ 1357, 1357, 1638, 1638, 1357, 1493, 1623, 1416, 1318, 1324, - /* 330 */ 1324, 1324, 1324, 1357, 1258, 1394, 1623, 1623, 1394, 1416, - /* 340 */ 1318, 1394, 1318, 1394, 1357, 1258, 1509, 1633, 1357, 1258, - /* 350 */ 1483, 1357, 1258, 1357, 1258, 1483, 1316, 1316, 1316, 1305, - /* 360 */ 1240, 1240, 1483, 1316, 1290, 1316, 1305, 1316, 1316, 1581, - /* 370 */ 1240, 1487, 1487, 1483, 1357, 1573, 1573, 1384, 1384, 1389, - /* 380 */ 1375, 1478, 1357, 1240, 1389, 1387, 1385, 1394, 1308, 1595, - /* 390 */ 1595, 1591, 1591, 1591, 1644, 1644, 1542, 1608, 1273, 1273, - /* 400 */ 1273, 1273, 1608, 1292, 1292, 1274, 1274, 1273, 1608, 1240, - /* 410 */ 1240, 1240, 1240, 1240, 1240, 1603, 1240, 1537, 1494, 1361, - /* 420 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 430 */ 1240, 1240, 1240, 1240, 1548, 1240, 1240, 1240, 1240, 1240, - /* 440 */ 1240, 1240, 1240, 1240, 1240, 1421, 1240, 1243, 1539, 1240, - /* 450 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1398, 1399, 1362, - /* 460 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1413, 1240, 1240, - /* 470 */ 1240, 1408, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 480 */ 1635, 1240, 1240, 1240, 1240, 1240, 1240, 1508, 1507, 1240, - /* 490 */ 1240, 1359, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 500 */ 1240, 1240, 1240, 1240, 1240, 1288, 1240, 1240, 1240, 1240, - /* 510 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 520 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1386, - /* 530 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 540 */ 1240, 1240, 1240, 1240, 1578, 1376, 1240, 1240, 1240, 1240, - /* 550 */ 1626, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 560 */ 1240, 1240, 1240, 1240, 1240, 1619, 1332, 1423, 1240, 1422, - /* 570 */ 1426, 1262, 1240, 1252, 1240, 1240, + /* 0 */ 1641, 1641, 1641, 1469, 1236, 1347, 1236, 1236, 1236, 1469, + /* 10 */ 1469, 1469, 1236, 1377, 1377, 1522, 1269, 1236, 1236, 1236, + /* 20 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1468, 1236, 1236, + /* 30 */ 1236, 1236, 1557, 1557, 1236, 1236, 1236, 1236, 1236, 1236, + /* 40 */ 1236, 1236, 1386, 1236, 1393, 1236, 1236, 1236, 1236, 1236, + /* 50 */ 1470, 1471, 1236, 1236, 1236, 1521, 1523, 1486, 1400, 1399, + /* 60 */ 1398, 1397, 1504, 1365, 1391, 1384, 1388, 1465, 1466, 1464, + /* 70 */ 1619, 1471, 1470, 1236, 1387, 1433, 1449, 1432, 1236, 1236, + /* 80 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 90 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 100 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 110 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 120 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 130 */ 1441, 1448, 1447, 1446, 1455, 1445, 1442, 1435, 1434, 1436, + /* 140 */ 1437, 1236, 1236, 1260, 1236, 1236, 1257, 1311, 1236, 1236, + /* 150 */ 1236, 1236, 1236, 1541, 1540, 1236, 1438, 1236, 1269, 1427, + /* 160 */ 1426, 1452, 1439, 1451, 1450, 1529, 1593, 1592, 1487, 1236, + /* 170 */ 1236, 1236, 1236, 1236, 1236, 1557, 1236, 1236, 1236, 1236, + /* 180 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 190 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1367, + /* 200 */ 1557, 1557, 1236, 1269, 1557, 1557, 1368, 1368, 1265, 1265, + /* 210 */ 1371, 1236, 1536, 1338, 1338, 1338, 1338, 1347, 1338, 1236, + /* 220 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 230 */ 1236, 1236, 1236, 1236, 1526, 1524, 1236, 1236, 1236, 1236, + /* 240 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 250 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 260 */ 1236, 1236, 1236, 1343, 1236, 1236, 1236, 1236, 1236, 1236, + /* 270 */ 1236, 1236, 1236, 1236, 1236, 1586, 1236, 1499, 1325, 1343, + /* 280 */ 1343, 1343, 1343, 1345, 1326, 1324, 1337, 1270, 1243, 1633, + /* 290 */ 1403, 1392, 1344, 1392, 1630, 1390, 1403, 1403, 1390, 1403, + /* 300 */ 1344, 1630, 1286, 1608, 1281, 1377, 1377, 1377, 1367, 1367, + /* 310 */ 1367, 1367, 1371, 1371, 1467, 1344, 1337, 1236, 1633, 1633, + /* 320 */ 1353, 1353, 1632, 1632, 1353, 1487, 1616, 1412, 1314, 1320, + /* 330 */ 1320, 1320, 1320, 1353, 1254, 1390, 1616, 1616, 1390, 1412, + /* 340 */ 1314, 1390, 1314, 1390, 1353, 1254, 1503, 1627, 1353, 1254, + /* 350 */ 1477, 1353, 1254, 1353, 1254, 1477, 1312, 1312, 1312, 1301, + /* 360 */ 1236, 1236, 1477, 1312, 1286, 1312, 1301, 1312, 1312, 1575, + /* 370 */ 1236, 1481, 1481, 1477, 1353, 1567, 1567, 1380, 1380, 1385, + /* 380 */ 1371, 1472, 1353, 1236, 1385, 1383, 1381, 1390, 1304, 1589, + /* 390 */ 1589, 1585, 1585, 1585, 1638, 1638, 1536, 1601, 1269, 1269, + /* 400 */ 1269, 1269, 1601, 1288, 1288, 1270, 1270, 1269, 1601, 1236, + /* 410 */ 1236, 1236, 1236, 1236, 1236, 1596, 1236, 1531, 1488, 1357, + /* 420 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 430 */ 1236, 1236, 1236, 1236, 1542, 1236, 1236, 1236, 1236, 1236, + /* 440 */ 1236, 1236, 1236, 1236, 1236, 1417, 1236, 1239, 1533, 1236, + /* 450 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1394, 1395, 1358, + /* 460 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1409, 1236, 1236, + /* 470 */ 1236, 1404, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 480 */ 1629, 1236, 1236, 1236, 1236, 1236, 1236, 1502, 1501, 1236, + /* 490 */ 1236, 1355, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 500 */ 1236, 1236, 1236, 1236, 1236, 1284, 1236, 1236, 1236, 1236, + /* 510 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 520 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1382, + /* 530 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 540 */ 1236, 1236, 1236, 1236, 1572, 1372, 1236, 1236, 1236, 1236, + /* 550 */ 1620, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 560 */ 1236, 1236, 1236, 1236, 1236, 1612, 1328, 1418, 1236, 1421, + /* 570 */ 1258, 1236, 1248, 1236, 1236, }; /********** End of lemon-generated parsing tables *****************************/ @@ -165424,233 +169715,231 @@ static const char *const yyRuleName[] = { /* 175 */ "idlist ::= idlist COMMA nm", /* 176 */ "idlist ::= nm", /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED", - /* 179 */ "expr ::= JOIN_KW", - /* 180 */ "expr ::= nm DOT nm", - /* 181 */ "expr ::= nm DOT nm DOT nm", - /* 182 */ "term ::= NULL|FLOAT|BLOB", - /* 183 */ "term ::= STRING", - /* 184 */ "term ::= INTEGER", - /* 185 */ "expr ::= VARIABLE", - /* 186 */ "expr ::= expr COLLATE ID|STRING", - /* 187 */ "expr ::= CAST LP expr AS typetoken RP", - /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 189 */ "expr ::= ID|INDEXED LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED LP STAR RP filter_over", - /* 192 */ "term ::= CTIME_KW", - /* 193 */ "expr ::= LP nexprlist COMMA expr RP", - /* 194 */ "expr ::= expr AND expr", - /* 195 */ "expr ::= expr OR expr", - /* 196 */ "expr ::= expr LT|GT|GE|LE expr", - /* 197 */ "expr ::= expr EQ|NE expr", - /* 198 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 199 */ "expr ::= expr PLUS|MINUS expr", - /* 200 */ "expr ::= expr STAR|SLASH|REM expr", - /* 201 */ "expr ::= expr CONCAT expr", - /* 202 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 203 */ "expr ::= expr likeop expr", - /* 204 */ "expr ::= expr likeop expr ESCAPE expr", - /* 205 */ "expr ::= expr ISNULL|NOTNULL", - /* 206 */ "expr ::= expr NOT NULL", - /* 207 */ "expr ::= expr IS expr", - /* 208 */ "expr ::= expr IS NOT expr", - /* 209 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 210 */ "expr ::= expr IS DISTINCT FROM expr", - /* 211 */ "expr ::= NOT expr", - /* 212 */ "expr ::= BITNOT expr", - /* 213 */ "expr ::= PLUS|MINUS expr", - /* 214 */ "expr ::= expr PTR expr", - /* 215 */ "between_op ::= BETWEEN", - /* 216 */ "between_op ::= NOT BETWEEN", - /* 217 */ "expr ::= expr between_op expr AND expr", - /* 218 */ "in_op ::= IN", - /* 219 */ "in_op ::= NOT IN", - /* 220 */ "expr ::= expr in_op LP exprlist RP", - /* 221 */ "expr ::= LP select RP", - /* 222 */ "expr ::= expr in_op LP select RP", - /* 223 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 224 */ "expr ::= EXISTS LP select RP", - /* 225 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 226 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 227 */ "case_exprlist ::= WHEN expr THEN expr", - /* 228 */ "case_else ::= ELSE expr", - /* 229 */ "case_else ::=", - /* 230 */ "case_operand ::= expr", - /* 231 */ "case_operand ::=", - /* 232 */ "exprlist ::=", - /* 233 */ "nexprlist ::= nexprlist COMMA expr", - /* 234 */ "nexprlist ::= expr", - /* 235 */ "paren_exprlist ::=", - /* 236 */ "paren_exprlist ::= LP exprlist RP", - /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 238 */ "uniqueflag ::= UNIQUE", - /* 239 */ "uniqueflag ::=", - /* 240 */ "eidlist_opt ::=", - /* 241 */ "eidlist_opt ::= LP eidlist RP", - /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 243 */ "eidlist ::= nm collate sortorder", - /* 244 */ "collate ::=", - /* 245 */ "collate ::= COLLATE ID|STRING", - /* 246 */ "cmd ::= DROP INDEX ifexists fullname", - /* 247 */ "cmd ::= VACUUM vinto", - /* 248 */ "cmd ::= VACUUM nm vinto", - /* 249 */ "vinto ::= INTO expr", - /* 250 */ "vinto ::=", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE|AFTER", - /* 261 */ "trigger_time ::= INSTEAD OF", - /* 262 */ "trigger_time ::=", - /* 263 */ "trigger_event ::= DELETE|INSERT", - /* 264 */ "trigger_event ::= UPDATE", - /* 265 */ "trigger_event ::= UPDATE OF idlist", - /* 266 */ "when_clause ::=", - /* 267 */ "when_clause ::= WHEN expr", - /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 270 */ "trnm ::= nm DOT nm", - /* 271 */ "tridxby ::= INDEXED BY nm", - /* 272 */ "tridxby ::= NOT INDEXED", - /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt select scanpt", - /* 277 */ "expr ::= RAISE LP IGNORE RP", - /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 279 */ "raisetype ::= ROLLBACK", - /* 280 */ "raisetype ::= ABORT", - /* 281 */ "raisetype ::= FAIL", - /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 284 */ "cmd ::= DETACH database_kw_opt expr", - /* 285 */ "key_opt ::=", - /* 286 */ "key_opt ::= KEY expr", - /* 287 */ "cmd ::= REINDEX", - /* 288 */ "cmd ::= REINDEX nm dbnm", - /* 289 */ "cmd ::= ANALYZE", - /* 290 */ "cmd ::= ANALYZE nm dbnm", - /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 294 */ "add_column_fullname ::= fullname", - /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 296 */ "cmd ::= create_vtab", - /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 299 */ "vtabarg ::=", - /* 300 */ "vtabargtoken ::= ANY", - /* 301 */ "vtabargtoken ::= lp anylist RP", - /* 302 */ "lp ::= LP", - /* 303 */ "with ::= WITH wqlist", - /* 304 */ "with ::= WITH RECURSIVE wqlist", - /* 305 */ "wqas ::= AS", - /* 306 */ "wqas ::= AS MATERIALIZED", - /* 307 */ "wqas ::= AS NOT MATERIALIZED", - /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 309 */ "wqlist ::= wqitem", - /* 310 */ "wqlist ::= wqlist COMMA wqitem", - /* 311 */ "windowdefn_list ::= windowdefn", - /* 312 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 313 */ "windowdefn ::= nm AS LP window RP", - /* 314 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 315 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 316 */ "window ::= ORDER BY sortlist frame_opt", - /* 317 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 318 */ "window ::= frame_opt", - /* 319 */ "window ::= nm frame_opt", - /* 320 */ "frame_opt ::=", - /* 321 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 322 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 323 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 324 */ "frame_bound_s ::= frame_bound", - /* 325 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 326 */ "frame_bound_e ::= frame_bound", - /* 327 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 328 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 329 */ "frame_bound ::= CURRENT ROW", - /* 330 */ "frame_exclude_opt ::=", - /* 331 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 332 */ "frame_exclude ::= NO OTHERS", - /* 333 */ "frame_exclude ::= CURRENT ROW", - /* 334 */ "frame_exclude ::= GROUP|TIES", - /* 335 */ "window_clause ::= WINDOW windowdefn_list", - /* 336 */ "filter_over ::= filter_clause over_clause", - /* 337 */ "filter_over ::= over_clause", - /* 338 */ "filter_over ::= filter_clause", - /* 339 */ "over_clause ::= OVER LP window RP", - /* 340 */ "over_clause ::= OVER nm", - /* 341 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 342 */ "input ::= cmdlist", - /* 343 */ "cmdlist ::= cmdlist ecmd", - /* 344 */ "cmdlist ::= ecmd", - /* 345 */ "ecmd ::= SEMI", - /* 346 */ "ecmd ::= cmdx SEMI", - /* 347 */ "ecmd ::= explain cmdx SEMI", - /* 348 */ "trans_opt ::=", - /* 349 */ "trans_opt ::= TRANSACTION", - /* 350 */ "trans_opt ::= TRANSACTION nm", - /* 351 */ "savepoint_opt ::= SAVEPOINT", - /* 352 */ "savepoint_opt ::=", - /* 353 */ "cmd ::= create_table create_table_args", - /* 354 */ "table_option_set ::= table_option", - /* 355 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 356 */ "columnlist ::= columnname carglist", - /* 357 */ "nm ::= ID|INDEXED", - /* 358 */ "nm ::= STRING", - /* 359 */ "nm ::= JOIN_KW", - /* 360 */ "typetoken ::= typename", - /* 361 */ "typename ::= ID|STRING", - /* 362 */ "signed ::= plus_num", - /* 363 */ "signed ::= minus_num", - /* 364 */ "carglist ::= carglist ccons", - /* 365 */ "carglist ::=", - /* 366 */ "ccons ::= NULL onconf", - /* 367 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 368 */ "ccons ::= AS generated", - /* 369 */ "conslist_opt ::= COMMA conslist", - /* 370 */ "conslist ::= conslist tconscomma tcons", - /* 371 */ "conslist ::= tcons", - /* 372 */ "tconscomma ::=", - /* 373 */ "defer_subclause_opt ::= defer_subclause", - /* 374 */ "resolvetype ::= raisetype", - /* 375 */ "selectnowith ::= oneselect", - /* 376 */ "oneselect ::= values", - /* 377 */ "sclp ::= selcollist COMMA", - /* 378 */ "as ::= ID|STRING", - /* 379 */ "indexed_opt ::= indexed_by", - /* 380 */ "returning ::=", - /* 381 */ "expr ::= term", - /* 382 */ "likeop ::= LIKE_KW|MATCH", - /* 383 */ "exprlist ::= nexprlist", - /* 384 */ "nmnum ::= plus_num", - /* 385 */ "nmnum ::= nm", - /* 386 */ "nmnum ::= ON", - /* 387 */ "nmnum ::= DELETE", - /* 388 */ "nmnum ::= DEFAULT", - /* 389 */ "plus_num ::= INTEGER|FLOAT", - /* 390 */ "foreach_clause ::=", - /* 391 */ "foreach_clause ::= FOR EACH ROW", - /* 392 */ "trnm ::= nm", - /* 393 */ "tridxby ::=", - /* 394 */ "database_kw_opt ::= DATABASE", - /* 395 */ "database_kw_opt ::=", - /* 396 */ "kwcolumn_opt ::=", - /* 397 */ "kwcolumn_opt ::= COLUMNKW", - /* 398 */ "vtabarglist ::= vtabarg", - /* 399 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 400 */ "vtabarg ::= vtabarg vtabargtoken", - /* 401 */ "anylist ::=", - /* 402 */ "anylist ::= anylist LP anylist RP", - /* 403 */ "anylist ::= anylist ANY", - /* 404 */ "with ::=", + /* 178 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 179 */ "expr ::= nm DOT nm", + /* 180 */ "expr ::= nm DOT nm DOT nm", + /* 181 */ "term ::= NULL|FLOAT|BLOB", + /* 182 */ "term ::= STRING", + /* 183 */ "term ::= INTEGER", + /* 184 */ "expr ::= VARIABLE", + /* 185 */ "expr ::= expr COLLATE ID|STRING", + /* 186 */ "expr ::= CAST LP expr AS typetoken RP", + /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 191 */ "term ::= CTIME_KW", + /* 192 */ "expr ::= LP nexprlist COMMA expr RP", + /* 193 */ "expr ::= expr AND expr", + /* 194 */ "expr ::= expr OR expr", + /* 195 */ "expr ::= expr LT|GT|GE|LE expr", + /* 196 */ "expr ::= expr EQ|NE expr", + /* 197 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 198 */ "expr ::= expr PLUS|MINUS expr", + /* 199 */ "expr ::= expr STAR|SLASH|REM expr", + /* 200 */ "expr ::= expr CONCAT expr", + /* 201 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 202 */ "expr ::= expr likeop expr", + /* 203 */ "expr ::= expr likeop expr ESCAPE expr", + /* 204 */ "expr ::= expr ISNULL|NOTNULL", + /* 205 */ "expr ::= expr NOT NULL", + /* 206 */ "expr ::= expr IS expr", + /* 207 */ "expr ::= expr IS NOT expr", + /* 208 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 209 */ "expr ::= expr IS DISTINCT FROM expr", + /* 210 */ "expr ::= NOT expr", + /* 211 */ "expr ::= BITNOT expr", + /* 212 */ "expr ::= PLUS|MINUS expr", + /* 213 */ "expr ::= expr PTR expr", + /* 214 */ "between_op ::= BETWEEN", + /* 215 */ "between_op ::= NOT BETWEEN", + /* 216 */ "expr ::= expr between_op expr AND expr", + /* 217 */ "in_op ::= IN", + /* 218 */ "in_op ::= NOT IN", + /* 219 */ "expr ::= expr in_op LP exprlist RP", + /* 220 */ "expr ::= LP select RP", + /* 221 */ "expr ::= expr in_op LP select RP", + /* 222 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 223 */ "expr ::= EXISTS LP select RP", + /* 224 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 225 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 226 */ "case_exprlist ::= WHEN expr THEN expr", + /* 227 */ "case_else ::= ELSE expr", + /* 228 */ "case_else ::=", + /* 229 */ "case_operand ::=", + /* 230 */ "exprlist ::=", + /* 231 */ "nexprlist ::= nexprlist COMMA expr", + /* 232 */ "nexprlist ::= expr", + /* 233 */ "paren_exprlist ::=", + /* 234 */ "paren_exprlist ::= LP exprlist RP", + /* 235 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 236 */ "uniqueflag ::= UNIQUE", + /* 237 */ "uniqueflag ::=", + /* 238 */ "eidlist_opt ::=", + /* 239 */ "eidlist_opt ::= LP eidlist RP", + /* 240 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 241 */ "eidlist ::= nm collate sortorder", + /* 242 */ "collate ::=", + /* 243 */ "collate ::= COLLATE ID|STRING", + /* 244 */ "cmd ::= DROP INDEX ifexists fullname", + /* 245 */ "cmd ::= VACUUM vinto", + /* 246 */ "cmd ::= VACUUM nm vinto", + /* 247 */ "vinto ::= INTO expr", + /* 248 */ "vinto ::=", + /* 249 */ "cmd ::= PRAGMA nm dbnm", + /* 250 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 251 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 254 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 255 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 256 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 257 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 258 */ "trigger_time ::= BEFORE|AFTER", + /* 259 */ "trigger_time ::= INSTEAD OF", + /* 260 */ "trigger_time ::=", + /* 261 */ "trigger_event ::= DELETE|INSERT", + /* 262 */ "trigger_event ::= UPDATE", + /* 263 */ "trigger_event ::= UPDATE OF idlist", + /* 264 */ "when_clause ::=", + /* 265 */ "when_clause ::= WHEN expr", + /* 266 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 267 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 268 */ "trnm ::= nm DOT nm", + /* 269 */ "tridxby ::= INDEXED BY nm", + /* 270 */ "tridxby ::= NOT INDEXED", + /* 271 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 272 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 273 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 274 */ "trigger_cmd ::= scanpt select scanpt", + /* 275 */ "expr ::= RAISE LP IGNORE RP", + /* 276 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 277 */ "raisetype ::= ROLLBACK", + /* 278 */ "raisetype ::= ABORT", + /* 279 */ "raisetype ::= FAIL", + /* 280 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 281 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 282 */ "cmd ::= DETACH database_kw_opt expr", + /* 283 */ "key_opt ::=", + /* 284 */ "key_opt ::= KEY expr", + /* 285 */ "cmd ::= REINDEX", + /* 286 */ "cmd ::= REINDEX nm dbnm", + /* 287 */ "cmd ::= ANALYZE", + /* 288 */ "cmd ::= ANALYZE nm dbnm", + /* 289 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 290 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 291 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 292 */ "add_column_fullname ::= fullname", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 294 */ "cmd ::= create_vtab", + /* 295 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 296 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 297 */ "vtabarg ::=", + /* 298 */ "vtabargtoken ::= ANY", + /* 299 */ "vtabargtoken ::= lp anylist RP", + /* 300 */ "lp ::= LP", + /* 301 */ "with ::= WITH wqlist", + /* 302 */ "with ::= WITH RECURSIVE wqlist", + /* 303 */ "wqas ::= AS", + /* 304 */ "wqas ::= AS MATERIALIZED", + /* 305 */ "wqas ::= AS NOT MATERIALIZED", + /* 306 */ "wqitem ::= nm eidlist_opt wqas LP select RP", + /* 307 */ "wqlist ::= wqitem", + /* 308 */ "wqlist ::= wqlist COMMA wqitem", + /* 309 */ "windowdefn_list ::= windowdefn", + /* 310 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 311 */ "windowdefn ::= nm AS LP window RP", + /* 312 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 313 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 314 */ "window ::= ORDER BY sortlist frame_opt", + /* 315 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 316 */ "window ::= frame_opt", + /* 317 */ "window ::= nm frame_opt", + /* 318 */ "frame_opt ::=", + /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 322 */ "frame_bound_s ::= frame_bound", + /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 324 */ "frame_bound_e ::= frame_bound", + /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 327 */ "frame_bound ::= CURRENT ROW", + /* 328 */ "frame_exclude_opt ::=", + /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 330 */ "frame_exclude ::= NO OTHERS", + /* 331 */ "frame_exclude ::= CURRENT ROW", + /* 332 */ "frame_exclude ::= GROUP|TIES", + /* 333 */ "window_clause ::= WINDOW windowdefn_list", + /* 334 */ "filter_over ::= filter_clause over_clause", + /* 335 */ "filter_over ::= over_clause", + /* 336 */ "filter_over ::= filter_clause", + /* 337 */ "over_clause ::= OVER LP window RP", + /* 338 */ "over_clause ::= OVER nm", + /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 340 */ "input ::= cmdlist", + /* 341 */ "cmdlist ::= cmdlist ecmd", + /* 342 */ "cmdlist ::= ecmd", + /* 343 */ "ecmd ::= SEMI", + /* 344 */ "ecmd ::= cmdx SEMI", + /* 345 */ "ecmd ::= explain cmdx SEMI", + /* 346 */ "trans_opt ::=", + /* 347 */ "trans_opt ::= TRANSACTION", + /* 348 */ "trans_opt ::= TRANSACTION nm", + /* 349 */ "savepoint_opt ::= SAVEPOINT", + /* 350 */ "savepoint_opt ::=", + /* 351 */ "cmd ::= create_table create_table_args", + /* 352 */ "table_option_set ::= table_option", + /* 353 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 354 */ "columnlist ::= columnname carglist", + /* 355 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 356 */ "nm ::= STRING", + /* 357 */ "typetoken ::= typename", + /* 358 */ "typename ::= ID|STRING", + /* 359 */ "signed ::= plus_num", + /* 360 */ "signed ::= minus_num", + /* 361 */ "carglist ::= carglist ccons", + /* 362 */ "carglist ::=", + /* 363 */ "ccons ::= NULL onconf", + /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 365 */ "ccons ::= AS generated", + /* 366 */ "conslist_opt ::= COMMA conslist", + /* 367 */ "conslist ::= conslist tconscomma tcons", + /* 368 */ "conslist ::= tcons", + /* 369 */ "tconscomma ::=", + /* 370 */ "defer_subclause_opt ::= defer_subclause", + /* 371 */ "resolvetype ::= raisetype", + /* 372 */ "selectnowith ::= oneselect", + /* 373 */ "oneselect ::= values", + /* 374 */ "sclp ::= selcollist COMMA", + /* 375 */ "as ::= ID|STRING", + /* 376 */ "indexed_opt ::= indexed_by", + /* 377 */ "returning ::=", + /* 378 */ "expr ::= term", + /* 379 */ "likeop ::= LIKE_KW|MATCH", + /* 380 */ "case_operand ::= expr", + /* 381 */ "exprlist ::= nexprlist", + /* 382 */ "nmnum ::= plus_num", + /* 383 */ "nmnum ::= nm", + /* 384 */ "nmnum ::= ON", + /* 385 */ "nmnum ::= DELETE", + /* 386 */ "nmnum ::= DEFAULT", + /* 387 */ "plus_num ::= INTEGER|FLOAT", + /* 388 */ "foreach_clause ::=", + /* 389 */ "foreach_clause ::= FOR EACH ROW", + /* 390 */ "trnm ::= nm", + /* 391 */ "tridxby ::=", + /* 392 */ "database_kw_opt ::= DATABASE", + /* 393 */ "database_kw_opt ::=", + /* 394 */ "kwcolumn_opt ::=", + /* 395 */ "kwcolumn_opt ::= COLUMNKW", + /* 396 */ "vtabarglist ::= vtabarg", + /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 398 */ "vtabarg ::= vtabarg vtabargtoken", + /* 399 */ "anylist ::=", + /* 400 */ "anylist ::= anylist LP anylist RP", + /* 401 */ "anylist ::= anylist ANY", + /* 402 */ "with ::=", }; #endif /* NDEBUG */ @@ -166335,233 +170624,231 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 263, /* (175) idlist ::= idlist COMMA nm */ 263, /* (176) idlist ::= nm */ 217, /* (177) expr ::= LP expr RP */ - 217, /* (178) expr ::= ID|INDEXED */ - 217, /* (179) expr ::= JOIN_KW */ - 217, /* (180) expr ::= nm DOT nm */ - 217, /* (181) expr ::= nm DOT nm DOT nm */ - 216, /* (182) term ::= NULL|FLOAT|BLOB */ - 216, /* (183) term ::= STRING */ - 216, /* (184) term ::= INTEGER */ - 217, /* (185) expr ::= VARIABLE */ - 217, /* (186) expr ::= expr COLLATE ID|STRING */ - 217, /* (187) expr ::= CAST LP expr AS typetoken RP */ - 217, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - 217, /* (189) expr ::= ID|INDEXED LP STAR RP */ - 217, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - 217, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - 216, /* (192) term ::= CTIME_KW */ - 217, /* (193) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (194) expr ::= expr AND expr */ - 217, /* (195) expr ::= expr OR expr */ - 217, /* (196) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (197) expr ::= expr EQ|NE expr */ - 217, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (199) expr ::= expr PLUS|MINUS expr */ - 217, /* (200) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (201) expr ::= expr CONCAT expr */ - 274, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (203) expr ::= expr likeop expr */ - 217, /* (204) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (205) expr ::= expr ISNULL|NOTNULL */ - 217, /* (206) expr ::= expr NOT NULL */ - 217, /* (207) expr ::= expr IS expr */ - 217, /* (208) expr ::= expr IS NOT expr */ - 217, /* (209) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (210) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (211) expr ::= NOT expr */ - 217, /* (212) expr ::= BITNOT expr */ - 217, /* (213) expr ::= PLUS|MINUS expr */ - 217, /* (214) expr ::= expr PTR expr */ - 275, /* (215) between_op ::= BETWEEN */ - 275, /* (216) between_op ::= NOT BETWEEN */ - 217, /* (217) expr ::= expr between_op expr AND expr */ - 276, /* (218) in_op ::= IN */ - 276, /* (219) in_op ::= NOT IN */ - 217, /* (220) expr ::= expr in_op LP exprlist RP */ - 217, /* (221) expr ::= LP select RP */ - 217, /* (222) expr ::= expr in_op LP select RP */ - 217, /* (223) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (224) expr ::= EXISTS LP select RP */ - 217, /* (225) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (227) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (228) case_else ::= ELSE expr */ - 280, /* (229) case_else ::= */ - 278, /* (230) case_operand ::= expr */ - 278, /* (231) case_operand ::= */ - 261, /* (232) exprlist ::= */ - 253, /* (233) nexprlist ::= nexprlist COMMA expr */ - 253, /* (234) nexprlist ::= expr */ - 277, /* (235) paren_exprlist ::= */ - 277, /* (236) paren_exprlist ::= LP exprlist RP */ - 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (238) uniqueflag ::= UNIQUE */ - 281, /* (239) uniqueflag ::= */ - 221, /* (240) eidlist_opt ::= */ - 221, /* (241) eidlist_opt ::= LP eidlist RP */ - 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (243) eidlist ::= nm collate sortorder */ - 282, /* (244) collate ::= */ - 282, /* (245) collate ::= COLLATE ID|STRING */ - 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (247) cmd ::= VACUUM vinto */ - 190, /* (248) cmd ::= VACUUM nm vinto */ - 283, /* (249) vinto ::= INTO expr */ - 283, /* (250) vinto ::= */ - 190, /* (251) cmd ::= PRAGMA nm dbnm */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (260) trigger_time ::= BEFORE|AFTER */ - 287, /* (261) trigger_time ::= INSTEAD OF */ - 287, /* (262) trigger_time ::= */ - 288, /* (263) trigger_event ::= DELETE|INSERT */ - 288, /* (264) trigger_event ::= UPDATE */ - 288, /* (265) trigger_event ::= UPDATE OF idlist */ - 290, /* (266) when_clause ::= */ - 290, /* (267) when_clause ::= WHEN expr */ - 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (270) trnm ::= nm DOT nm */ - 293, /* (271) tridxby ::= INDEXED BY nm */ - 293, /* (272) tridxby ::= NOT INDEXED */ - 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (276) trigger_cmd ::= scanpt select scanpt */ - 217, /* (277) expr ::= RAISE LP IGNORE RP */ - 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (279) raisetype ::= ROLLBACK */ - 236, /* (280) raisetype ::= ABORT */ - 236, /* (281) raisetype ::= FAIL */ - 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (284) cmd ::= DETACH database_kw_opt expr */ - 295, /* (285) key_opt ::= */ - 295, /* (286) key_opt ::= KEY expr */ - 190, /* (287) cmd ::= REINDEX */ - 190, /* (288) cmd ::= REINDEX nm dbnm */ - 190, /* (289) cmd ::= ANALYZE */ - 190, /* (290) cmd ::= ANALYZE nm dbnm */ - 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (294) add_column_fullname ::= fullname */ - 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (296) cmd ::= create_vtab */ - 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (299) vtabarg ::= */ - 301, /* (300) vtabargtoken ::= ANY */ - 301, /* (301) vtabargtoken ::= lp anylist RP */ - 302, /* (302) lp ::= LP */ - 266, /* (303) with ::= WITH wqlist */ - 266, /* (304) with ::= WITH RECURSIVE wqlist */ - 305, /* (305) wqas ::= AS */ - 305, /* (306) wqas ::= AS MATERIALIZED */ - 305, /* (307) wqas ::= AS NOT MATERIALIZED */ - 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (309) wqlist ::= wqitem */ - 241, /* (310) wqlist ::= wqlist COMMA wqitem */ - 306, /* (311) windowdefn_list ::= windowdefn */ - 306, /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (313) windowdefn ::= nm AS LP window RP */ - 308, /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (316) window ::= ORDER BY sortlist frame_opt */ - 308, /* (317) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (318) window ::= frame_opt */ - 308, /* (319) window ::= nm frame_opt */ - 309, /* (320) frame_opt ::= */ - 309, /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 309, /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 313, /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */ - 315, /* (324) frame_bound_s ::= frame_bound */ - 315, /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */ - 316, /* (326) frame_bound_e ::= frame_bound */ - 316, /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 314, /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */ - 314, /* (329) frame_bound ::= CURRENT ROW */ - 317, /* (330) frame_exclude_opt ::= */ - 317, /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 318, /* (332) frame_exclude ::= NO OTHERS */ - 318, /* (333) frame_exclude ::= CURRENT ROW */ - 318, /* (334) frame_exclude ::= GROUP|TIES */ - 251, /* (335) window_clause ::= WINDOW windowdefn_list */ - 273, /* (336) filter_over ::= filter_clause over_clause */ - 273, /* (337) filter_over ::= over_clause */ - 273, /* (338) filter_over ::= filter_clause */ - 312, /* (339) over_clause ::= OVER LP window RP */ - 312, /* (340) over_clause ::= OVER nm */ - 311, /* (341) filter_clause ::= FILTER LP WHERE expr RP */ - 185, /* (342) input ::= cmdlist */ - 186, /* (343) cmdlist ::= cmdlist ecmd */ - 186, /* (344) cmdlist ::= ecmd */ - 187, /* (345) ecmd ::= SEMI */ - 187, /* (346) ecmd ::= cmdx SEMI */ - 187, /* (347) ecmd ::= explain cmdx SEMI */ - 192, /* (348) trans_opt ::= */ - 192, /* (349) trans_opt ::= TRANSACTION */ - 192, /* (350) trans_opt ::= TRANSACTION nm */ - 194, /* (351) savepoint_opt ::= SAVEPOINT */ - 194, /* (352) savepoint_opt ::= */ - 190, /* (353) cmd ::= create_table create_table_args */ - 203, /* (354) table_option_set ::= table_option */ - 201, /* (355) columnlist ::= columnlist COMMA columnname carglist */ - 201, /* (356) columnlist ::= columnname carglist */ - 193, /* (357) nm ::= ID|INDEXED */ - 193, /* (358) nm ::= STRING */ - 193, /* (359) nm ::= JOIN_KW */ - 208, /* (360) typetoken ::= typename */ - 209, /* (361) typename ::= ID|STRING */ - 210, /* (362) signed ::= plus_num */ - 210, /* (363) signed ::= minus_num */ - 207, /* (364) carglist ::= carglist ccons */ - 207, /* (365) carglist ::= */ - 215, /* (366) ccons ::= NULL onconf */ - 215, /* (367) ccons ::= GENERATED ALWAYS AS generated */ - 215, /* (368) ccons ::= AS generated */ - 202, /* (369) conslist_opt ::= COMMA conslist */ - 228, /* (370) conslist ::= conslist tconscomma tcons */ - 228, /* (371) conslist ::= tcons */ - 229, /* (372) tconscomma ::= */ - 233, /* (373) defer_subclause_opt ::= defer_subclause */ - 235, /* (374) resolvetype ::= raisetype */ - 239, /* (375) selectnowith ::= oneselect */ - 240, /* (376) oneselect ::= values */ - 254, /* (377) sclp ::= selcollist COMMA */ - 255, /* (378) as ::= ID|STRING */ - 264, /* (379) indexed_opt ::= indexed_by */ - 272, /* (380) returning ::= */ - 217, /* (381) expr ::= term */ - 274, /* (382) likeop ::= LIKE_KW|MATCH */ - 261, /* (383) exprlist ::= nexprlist */ - 284, /* (384) nmnum ::= plus_num */ - 284, /* (385) nmnum ::= nm */ - 284, /* (386) nmnum ::= ON */ - 284, /* (387) nmnum ::= DELETE */ - 284, /* (388) nmnum ::= DEFAULT */ - 211, /* (389) plus_num ::= INTEGER|FLOAT */ - 289, /* (390) foreach_clause ::= */ - 289, /* (391) foreach_clause ::= FOR EACH ROW */ - 292, /* (392) trnm ::= nm */ - 293, /* (393) tridxby ::= */ - 294, /* (394) database_kw_opt ::= DATABASE */ - 294, /* (395) database_kw_opt ::= */ - 297, /* (396) kwcolumn_opt ::= */ - 297, /* (397) kwcolumn_opt ::= COLUMNKW */ - 299, /* (398) vtabarglist ::= vtabarg */ - 299, /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ - 300, /* (400) vtabarg ::= vtabarg vtabargtoken */ - 303, /* (401) anylist ::= */ - 303, /* (402) anylist ::= anylist LP anylist RP */ - 303, /* (403) anylist ::= anylist ANY */ - 266, /* (404) with ::= */ + 217, /* (178) expr ::= ID|INDEXED|JOIN_KW */ + 217, /* (179) expr ::= nm DOT nm */ + 217, /* (180) expr ::= nm DOT nm DOT nm */ + 216, /* (181) term ::= NULL|FLOAT|BLOB */ + 216, /* (182) term ::= STRING */ + 216, /* (183) term ::= INTEGER */ + 217, /* (184) expr ::= VARIABLE */ + 217, /* (185) expr ::= expr COLLATE ID|STRING */ + 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ + 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 216, /* (191) term ::= CTIME_KW */ + 217, /* (192) expr ::= LP nexprlist COMMA expr RP */ + 217, /* (193) expr ::= expr AND expr */ + 217, /* (194) expr ::= expr OR expr */ + 217, /* (195) expr ::= expr LT|GT|GE|LE expr */ + 217, /* (196) expr ::= expr EQ|NE expr */ + 217, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 217, /* (198) expr ::= expr PLUS|MINUS expr */ + 217, /* (199) expr ::= expr STAR|SLASH|REM expr */ + 217, /* (200) expr ::= expr CONCAT expr */ + 274, /* (201) likeop ::= NOT LIKE_KW|MATCH */ + 217, /* (202) expr ::= expr likeop expr */ + 217, /* (203) expr ::= expr likeop expr ESCAPE expr */ + 217, /* (204) expr ::= expr ISNULL|NOTNULL */ + 217, /* (205) expr ::= expr NOT NULL */ + 217, /* (206) expr ::= expr IS expr */ + 217, /* (207) expr ::= expr IS NOT expr */ + 217, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ + 217, /* (209) expr ::= expr IS DISTINCT FROM expr */ + 217, /* (210) expr ::= NOT expr */ + 217, /* (211) expr ::= BITNOT expr */ + 217, /* (212) expr ::= PLUS|MINUS expr */ + 217, /* (213) expr ::= expr PTR expr */ + 275, /* (214) between_op ::= BETWEEN */ + 275, /* (215) between_op ::= NOT BETWEEN */ + 217, /* (216) expr ::= expr between_op expr AND expr */ + 276, /* (217) in_op ::= IN */ + 276, /* (218) in_op ::= NOT IN */ + 217, /* (219) expr ::= expr in_op LP exprlist RP */ + 217, /* (220) expr ::= LP select RP */ + 217, /* (221) expr ::= expr in_op LP select RP */ + 217, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ + 217, /* (223) expr ::= EXISTS LP select RP */ + 217, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ + 279, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 279, /* (226) case_exprlist ::= WHEN expr THEN expr */ + 280, /* (227) case_else ::= ELSE expr */ + 280, /* (228) case_else ::= */ + 278, /* (229) case_operand ::= */ + 261, /* (230) exprlist ::= */ + 253, /* (231) nexprlist ::= nexprlist COMMA expr */ + 253, /* (232) nexprlist ::= expr */ + 277, /* (233) paren_exprlist ::= */ + 277, /* (234) paren_exprlist ::= LP exprlist RP */ + 190, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 281, /* (236) uniqueflag ::= UNIQUE */ + 281, /* (237) uniqueflag ::= */ + 221, /* (238) eidlist_opt ::= */ + 221, /* (239) eidlist_opt ::= LP eidlist RP */ + 232, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ + 232, /* (241) eidlist ::= nm collate sortorder */ + 282, /* (242) collate ::= */ + 282, /* (243) collate ::= COLLATE ID|STRING */ + 190, /* (244) cmd ::= DROP INDEX ifexists fullname */ + 190, /* (245) cmd ::= VACUUM vinto */ + 190, /* (246) cmd ::= VACUUM nm vinto */ + 283, /* (247) vinto ::= INTO expr */ + 283, /* (248) vinto ::= */ + 190, /* (249) cmd ::= PRAGMA nm dbnm */ + 190, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 190, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 190, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 190, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 211, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ + 212, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ + 190, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 285, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 287, /* (258) trigger_time ::= BEFORE|AFTER */ + 287, /* (259) trigger_time ::= INSTEAD OF */ + 287, /* (260) trigger_time ::= */ + 288, /* (261) trigger_event ::= DELETE|INSERT */ + 288, /* (262) trigger_event ::= UPDATE */ + 288, /* (263) trigger_event ::= UPDATE OF idlist */ + 290, /* (264) when_clause ::= */ + 290, /* (265) when_clause ::= WHEN expr */ + 286, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 286, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ + 292, /* (268) trnm ::= nm DOT nm */ + 293, /* (269) tridxby ::= INDEXED BY nm */ + 293, /* (270) tridxby ::= NOT INDEXED */ + 291, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 291, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 291, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 291, /* (274) trigger_cmd ::= scanpt select scanpt */ + 217, /* (275) expr ::= RAISE LP IGNORE RP */ + 217, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ + 236, /* (277) raisetype ::= ROLLBACK */ + 236, /* (278) raisetype ::= ABORT */ + 236, /* (279) raisetype ::= FAIL */ + 190, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ + 190, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 190, /* (282) cmd ::= DETACH database_kw_opt expr */ + 295, /* (283) key_opt ::= */ + 295, /* (284) key_opt ::= KEY expr */ + 190, /* (285) cmd ::= REINDEX */ + 190, /* (286) cmd ::= REINDEX nm dbnm */ + 190, /* (287) cmd ::= ANALYZE */ + 190, /* (288) cmd ::= ANALYZE nm dbnm */ + 190, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 190, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 190, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 296, /* (292) add_column_fullname ::= fullname */ + 190, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 190, /* (294) cmd ::= create_vtab */ + 190, /* (295) cmd ::= create_vtab LP vtabarglist RP */ + 298, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 300, /* (297) vtabarg ::= */ + 301, /* (298) vtabargtoken ::= ANY */ + 301, /* (299) vtabargtoken ::= lp anylist RP */ + 302, /* (300) lp ::= LP */ + 266, /* (301) with ::= WITH wqlist */ + 266, /* (302) with ::= WITH RECURSIVE wqlist */ + 305, /* (303) wqas ::= AS */ + 305, /* (304) wqas ::= AS MATERIALIZED */ + 305, /* (305) wqas ::= AS NOT MATERIALIZED */ + 304, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ + 241, /* (307) wqlist ::= wqitem */ + 241, /* (308) wqlist ::= wqlist COMMA wqitem */ + 306, /* (309) windowdefn_list ::= windowdefn */ + 306, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 307, /* (311) windowdefn ::= nm AS LP window RP */ + 308, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (314) window ::= ORDER BY sortlist frame_opt */ + 308, /* (315) window ::= nm ORDER BY sortlist frame_opt */ + 308, /* (316) window ::= frame_opt */ + 308, /* (317) window ::= nm frame_opt */ + 309, /* (318) frame_opt ::= */ + 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 309, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 313, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ + 315, /* (322) frame_bound_s ::= frame_bound */ + 315, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ + 316, /* (324) frame_bound_e ::= frame_bound */ + 316, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 314, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ + 314, /* (327) frame_bound ::= CURRENT ROW */ + 317, /* (328) frame_exclude_opt ::= */ + 317, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 318, /* (330) frame_exclude ::= NO OTHERS */ + 318, /* (331) frame_exclude ::= CURRENT ROW */ + 318, /* (332) frame_exclude ::= GROUP|TIES */ + 251, /* (333) window_clause ::= WINDOW windowdefn_list */ + 273, /* (334) filter_over ::= filter_clause over_clause */ + 273, /* (335) filter_over ::= over_clause */ + 273, /* (336) filter_over ::= filter_clause */ + 312, /* (337) over_clause ::= OVER LP window RP */ + 312, /* (338) over_clause ::= OVER nm */ + 311, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ + 185, /* (340) input ::= cmdlist */ + 186, /* (341) cmdlist ::= cmdlist ecmd */ + 186, /* (342) cmdlist ::= ecmd */ + 187, /* (343) ecmd ::= SEMI */ + 187, /* (344) ecmd ::= cmdx SEMI */ + 187, /* (345) ecmd ::= explain cmdx SEMI */ + 192, /* (346) trans_opt ::= */ + 192, /* (347) trans_opt ::= TRANSACTION */ + 192, /* (348) trans_opt ::= TRANSACTION nm */ + 194, /* (349) savepoint_opt ::= SAVEPOINT */ + 194, /* (350) savepoint_opt ::= */ + 190, /* (351) cmd ::= create_table create_table_args */ + 203, /* (352) table_option_set ::= table_option */ + 201, /* (353) columnlist ::= columnlist COMMA columnname carglist */ + 201, /* (354) columnlist ::= columnname carglist */ + 193, /* (355) nm ::= ID|INDEXED|JOIN_KW */ + 193, /* (356) nm ::= STRING */ + 208, /* (357) typetoken ::= typename */ + 209, /* (358) typename ::= ID|STRING */ + 210, /* (359) signed ::= plus_num */ + 210, /* (360) signed ::= minus_num */ + 207, /* (361) carglist ::= carglist ccons */ + 207, /* (362) carglist ::= */ + 215, /* (363) ccons ::= NULL onconf */ + 215, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + 215, /* (365) ccons ::= AS generated */ + 202, /* (366) conslist_opt ::= COMMA conslist */ + 228, /* (367) conslist ::= conslist tconscomma tcons */ + 228, /* (368) conslist ::= tcons */ + 229, /* (369) tconscomma ::= */ + 233, /* (370) defer_subclause_opt ::= defer_subclause */ + 235, /* (371) resolvetype ::= raisetype */ + 239, /* (372) selectnowith ::= oneselect */ + 240, /* (373) oneselect ::= values */ + 254, /* (374) sclp ::= selcollist COMMA */ + 255, /* (375) as ::= ID|STRING */ + 264, /* (376) indexed_opt ::= indexed_by */ + 272, /* (377) returning ::= */ + 217, /* (378) expr ::= term */ + 274, /* (379) likeop ::= LIKE_KW|MATCH */ + 278, /* (380) case_operand ::= expr */ + 261, /* (381) exprlist ::= nexprlist */ + 284, /* (382) nmnum ::= plus_num */ + 284, /* (383) nmnum ::= nm */ + 284, /* (384) nmnum ::= ON */ + 284, /* (385) nmnum ::= DELETE */ + 284, /* (386) nmnum ::= DEFAULT */ + 211, /* (387) plus_num ::= INTEGER|FLOAT */ + 289, /* (388) foreach_clause ::= */ + 289, /* (389) foreach_clause ::= FOR EACH ROW */ + 292, /* (390) trnm ::= nm */ + 293, /* (391) tridxby ::= */ + 294, /* (392) database_kw_opt ::= DATABASE */ + 294, /* (393) database_kw_opt ::= */ + 297, /* (394) kwcolumn_opt ::= */ + 297, /* (395) kwcolumn_opt ::= COLUMNKW */ + 299, /* (396) vtabarglist ::= vtabarg */ + 299, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ + 300, /* (398) vtabarg ::= vtabarg vtabargtoken */ + 303, /* (399) anylist ::= */ + 303, /* (400) anylist ::= anylist LP anylist RP */ + 303, /* (401) anylist ::= anylist ANY */ + 266, /* (402) with ::= */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -166745,233 +171032,231 @@ static const signed char yyRuleInfoNRhs[] = { -3, /* (175) idlist ::= idlist COMMA nm */ -1, /* (176) idlist ::= nm */ -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED */ - -1, /* (179) expr ::= JOIN_KW */ - -3, /* (180) expr ::= nm DOT nm */ - -5, /* (181) expr ::= nm DOT nm DOT nm */ - -1, /* (182) term ::= NULL|FLOAT|BLOB */ - -1, /* (183) term ::= STRING */ - -1, /* (184) term ::= INTEGER */ - -1, /* (185) expr ::= VARIABLE */ - -3, /* (186) expr ::= expr COLLATE ID|STRING */ - -6, /* (187) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - -4, /* (189) expr ::= ID|INDEXED LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - -5, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - -1, /* (192) term ::= CTIME_KW */ - -5, /* (193) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (194) expr ::= expr AND expr */ - -3, /* (195) expr ::= expr OR expr */ - -3, /* (196) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (197) expr ::= expr EQ|NE expr */ - -3, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (199) expr ::= expr PLUS|MINUS expr */ - -3, /* (200) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (201) expr ::= expr CONCAT expr */ - -2, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (203) expr ::= expr likeop expr */ - -5, /* (204) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (205) expr ::= expr ISNULL|NOTNULL */ - -3, /* (206) expr ::= expr NOT NULL */ - -3, /* (207) expr ::= expr IS expr */ - -4, /* (208) expr ::= expr IS NOT expr */ - -6, /* (209) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (210) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (211) expr ::= NOT expr */ - -2, /* (212) expr ::= BITNOT expr */ - -2, /* (213) expr ::= PLUS|MINUS expr */ - -3, /* (214) expr ::= expr PTR expr */ - -1, /* (215) between_op ::= BETWEEN */ - -2, /* (216) between_op ::= NOT BETWEEN */ - -5, /* (217) expr ::= expr between_op expr AND expr */ - -1, /* (218) in_op ::= IN */ - -2, /* (219) in_op ::= NOT IN */ - -5, /* (220) expr ::= expr in_op LP exprlist RP */ - -3, /* (221) expr ::= LP select RP */ - -5, /* (222) expr ::= expr in_op LP select RP */ - -5, /* (223) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (224) expr ::= EXISTS LP select RP */ - -5, /* (225) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (227) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (228) case_else ::= ELSE expr */ - 0, /* (229) case_else ::= */ - -1, /* (230) case_operand ::= expr */ - 0, /* (231) case_operand ::= */ - 0, /* (232) exprlist ::= */ - -3, /* (233) nexprlist ::= nexprlist COMMA expr */ - -1, /* (234) nexprlist ::= expr */ - 0, /* (235) paren_exprlist ::= */ - -3, /* (236) paren_exprlist ::= LP exprlist RP */ - -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (238) uniqueflag ::= UNIQUE */ - 0, /* (239) uniqueflag ::= */ - 0, /* (240) eidlist_opt ::= */ - -3, /* (241) eidlist_opt ::= LP eidlist RP */ - -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (243) eidlist ::= nm collate sortorder */ - 0, /* (244) collate ::= */ - -2, /* (245) collate ::= COLLATE ID|STRING */ - -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (247) cmd ::= VACUUM vinto */ - -3, /* (248) cmd ::= VACUUM nm vinto */ - -2, /* (249) vinto ::= INTO expr */ - 0, /* (250) vinto ::= */ - -3, /* (251) cmd ::= PRAGMA nm dbnm */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (260) trigger_time ::= BEFORE|AFTER */ - -2, /* (261) trigger_time ::= INSTEAD OF */ - 0, /* (262) trigger_time ::= */ - -1, /* (263) trigger_event ::= DELETE|INSERT */ - -1, /* (264) trigger_event ::= UPDATE */ - -3, /* (265) trigger_event ::= UPDATE OF idlist */ - 0, /* (266) when_clause ::= */ - -2, /* (267) when_clause ::= WHEN expr */ - -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (270) trnm ::= nm DOT nm */ - -3, /* (271) tridxby ::= INDEXED BY nm */ - -2, /* (272) tridxby ::= NOT INDEXED */ - -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (276) trigger_cmd ::= scanpt select scanpt */ - -4, /* (277) expr ::= RAISE LP IGNORE RP */ - -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (279) raisetype ::= ROLLBACK */ - -1, /* (280) raisetype ::= ABORT */ - -1, /* (281) raisetype ::= FAIL */ - -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (284) cmd ::= DETACH database_kw_opt expr */ - 0, /* (285) key_opt ::= */ - -2, /* (286) key_opt ::= KEY expr */ - -1, /* (287) cmd ::= REINDEX */ - -3, /* (288) cmd ::= REINDEX nm dbnm */ - -1, /* (289) cmd ::= ANALYZE */ - -3, /* (290) cmd ::= ANALYZE nm dbnm */ - -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (294) add_column_fullname ::= fullname */ - -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (296) cmd ::= create_vtab */ - -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (299) vtabarg ::= */ - -1, /* (300) vtabargtoken ::= ANY */ - -3, /* (301) vtabargtoken ::= lp anylist RP */ - -1, /* (302) lp ::= LP */ - -2, /* (303) with ::= WITH wqlist */ - -3, /* (304) with ::= WITH RECURSIVE wqlist */ - -1, /* (305) wqas ::= AS */ - -2, /* (306) wqas ::= AS MATERIALIZED */ - -3, /* (307) wqas ::= AS NOT MATERIALIZED */ - -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (309) wqlist ::= wqitem */ - -3, /* (310) wqlist ::= wqlist COMMA wqitem */ - -1, /* (311) windowdefn_list ::= windowdefn */ - -3, /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (313) windowdefn ::= nm AS LP window RP */ - -5, /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (316) window ::= ORDER BY sortlist frame_opt */ - -5, /* (317) window ::= nm ORDER BY sortlist frame_opt */ - -1, /* (318) window ::= frame_opt */ - -2, /* (319) window ::= nm frame_opt */ - 0, /* (320) frame_opt ::= */ - -3, /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (324) frame_bound_s ::= frame_bound */ - -2, /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (326) frame_bound_e ::= frame_bound */ - -2, /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (329) frame_bound ::= CURRENT ROW */ - 0, /* (330) frame_exclude_opt ::= */ - -2, /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (332) frame_exclude ::= NO OTHERS */ - -2, /* (333) frame_exclude ::= CURRENT ROW */ - -1, /* (334) frame_exclude ::= GROUP|TIES */ - -2, /* (335) window_clause ::= WINDOW windowdefn_list */ - -2, /* (336) filter_over ::= filter_clause over_clause */ - -1, /* (337) filter_over ::= over_clause */ - -1, /* (338) filter_over ::= filter_clause */ - -4, /* (339) over_clause ::= OVER LP window RP */ - -2, /* (340) over_clause ::= OVER nm */ - -5, /* (341) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (342) input ::= cmdlist */ - -2, /* (343) cmdlist ::= cmdlist ecmd */ - -1, /* (344) cmdlist ::= ecmd */ - -1, /* (345) ecmd ::= SEMI */ - -2, /* (346) ecmd ::= cmdx SEMI */ - -3, /* (347) ecmd ::= explain cmdx SEMI */ - 0, /* (348) trans_opt ::= */ - -1, /* (349) trans_opt ::= TRANSACTION */ - -2, /* (350) trans_opt ::= TRANSACTION nm */ - -1, /* (351) savepoint_opt ::= SAVEPOINT */ - 0, /* (352) savepoint_opt ::= */ - -2, /* (353) cmd ::= create_table create_table_args */ - -1, /* (354) table_option_set ::= table_option */ - -4, /* (355) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (356) columnlist ::= columnname carglist */ - -1, /* (357) nm ::= ID|INDEXED */ - -1, /* (358) nm ::= STRING */ - -1, /* (359) nm ::= JOIN_KW */ - -1, /* (360) typetoken ::= typename */ - -1, /* (361) typename ::= ID|STRING */ - -1, /* (362) signed ::= plus_num */ - -1, /* (363) signed ::= minus_num */ - -2, /* (364) carglist ::= carglist ccons */ - 0, /* (365) carglist ::= */ - -2, /* (366) ccons ::= NULL onconf */ - -4, /* (367) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (368) ccons ::= AS generated */ - -2, /* (369) conslist_opt ::= COMMA conslist */ - -3, /* (370) conslist ::= conslist tconscomma tcons */ - -1, /* (371) conslist ::= tcons */ - 0, /* (372) tconscomma ::= */ - -1, /* (373) defer_subclause_opt ::= defer_subclause */ - -1, /* (374) resolvetype ::= raisetype */ - -1, /* (375) selectnowith ::= oneselect */ - -1, /* (376) oneselect ::= values */ - -2, /* (377) sclp ::= selcollist COMMA */ - -1, /* (378) as ::= ID|STRING */ - -1, /* (379) indexed_opt ::= indexed_by */ - 0, /* (380) returning ::= */ - -1, /* (381) expr ::= term */ - -1, /* (382) likeop ::= LIKE_KW|MATCH */ - -1, /* (383) exprlist ::= nexprlist */ - -1, /* (384) nmnum ::= plus_num */ - -1, /* (385) nmnum ::= nm */ - -1, /* (386) nmnum ::= ON */ - -1, /* (387) nmnum ::= DELETE */ - -1, /* (388) nmnum ::= DEFAULT */ - -1, /* (389) plus_num ::= INTEGER|FLOAT */ - 0, /* (390) foreach_clause ::= */ - -3, /* (391) foreach_clause ::= FOR EACH ROW */ - -1, /* (392) trnm ::= nm */ - 0, /* (393) tridxby ::= */ - -1, /* (394) database_kw_opt ::= DATABASE */ - 0, /* (395) database_kw_opt ::= */ - 0, /* (396) kwcolumn_opt ::= */ - -1, /* (397) kwcolumn_opt ::= COLUMNKW */ - -1, /* (398) vtabarglist ::= vtabarg */ - -3, /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (400) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (401) anylist ::= */ - -4, /* (402) anylist ::= anylist LP anylist RP */ - -2, /* (403) anylist ::= anylist ANY */ - 0, /* (404) with ::= */ + -1, /* (178) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (179) expr ::= nm DOT nm */ + -5, /* (180) expr ::= nm DOT nm DOT nm */ + -1, /* (181) term ::= NULL|FLOAT|BLOB */ + -1, /* (182) term ::= STRING */ + -1, /* (183) term ::= INTEGER */ + -1, /* (184) expr ::= VARIABLE */ + -3, /* (185) expr ::= expr COLLATE ID|STRING */ + -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -4, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -5, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (191) term ::= CTIME_KW */ + -5, /* (192) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (193) expr ::= expr AND expr */ + -3, /* (194) expr ::= expr OR expr */ + -3, /* (195) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (196) expr ::= expr EQ|NE expr */ + -3, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (198) expr ::= expr PLUS|MINUS expr */ + -3, /* (199) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (200) expr ::= expr CONCAT expr */ + -2, /* (201) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (202) expr ::= expr likeop expr */ + -5, /* (203) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (204) expr ::= expr ISNULL|NOTNULL */ + -3, /* (205) expr ::= expr NOT NULL */ + -3, /* (206) expr ::= expr IS expr */ + -4, /* (207) expr ::= expr IS NOT expr */ + -6, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (209) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (210) expr ::= NOT expr */ + -2, /* (211) expr ::= BITNOT expr */ + -2, /* (212) expr ::= PLUS|MINUS expr */ + -3, /* (213) expr ::= expr PTR expr */ + -1, /* (214) between_op ::= BETWEEN */ + -2, /* (215) between_op ::= NOT BETWEEN */ + -5, /* (216) expr ::= expr between_op expr AND expr */ + -1, /* (217) in_op ::= IN */ + -2, /* (218) in_op ::= NOT IN */ + -5, /* (219) expr ::= expr in_op LP exprlist RP */ + -3, /* (220) expr ::= LP select RP */ + -5, /* (221) expr ::= expr in_op LP select RP */ + -5, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (223) expr ::= EXISTS LP select RP */ + -5, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (226) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (227) case_else ::= ELSE expr */ + 0, /* (228) case_else ::= */ + 0, /* (229) case_operand ::= */ + 0, /* (230) exprlist ::= */ + -3, /* (231) nexprlist ::= nexprlist COMMA expr */ + -1, /* (232) nexprlist ::= expr */ + 0, /* (233) paren_exprlist ::= */ + -3, /* (234) paren_exprlist ::= LP exprlist RP */ + -12, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (236) uniqueflag ::= UNIQUE */ + 0, /* (237) uniqueflag ::= */ + 0, /* (238) eidlist_opt ::= */ + -3, /* (239) eidlist_opt ::= LP eidlist RP */ + -5, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (241) eidlist ::= nm collate sortorder */ + 0, /* (242) collate ::= */ + -2, /* (243) collate ::= COLLATE ID|STRING */ + -4, /* (244) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (245) cmd ::= VACUUM vinto */ + -3, /* (246) cmd ::= VACUUM nm vinto */ + -2, /* (247) vinto ::= INTO expr */ + 0, /* (248) vinto ::= */ + -3, /* (249) cmd ::= PRAGMA nm dbnm */ + -5, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (258) trigger_time ::= BEFORE|AFTER */ + -2, /* (259) trigger_time ::= INSTEAD OF */ + 0, /* (260) trigger_time ::= */ + -1, /* (261) trigger_event ::= DELETE|INSERT */ + -1, /* (262) trigger_event ::= UPDATE */ + -3, /* (263) trigger_event ::= UPDATE OF idlist */ + 0, /* (264) when_clause ::= */ + -2, /* (265) when_clause ::= WHEN expr */ + -3, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (268) trnm ::= nm DOT nm */ + -3, /* (269) tridxby ::= INDEXED BY nm */ + -2, /* (270) tridxby ::= NOT INDEXED */ + -9, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (274) trigger_cmd ::= scanpt select scanpt */ + -4, /* (275) expr ::= RAISE LP IGNORE RP */ + -6, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (277) raisetype ::= ROLLBACK */ + -1, /* (278) raisetype ::= ABORT */ + -1, /* (279) raisetype ::= FAIL */ + -4, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (282) cmd ::= DETACH database_kw_opt expr */ + 0, /* (283) key_opt ::= */ + -2, /* (284) key_opt ::= KEY expr */ + -1, /* (285) cmd ::= REINDEX */ + -3, /* (286) cmd ::= REINDEX nm dbnm */ + -1, /* (287) cmd ::= ANALYZE */ + -3, /* (288) cmd ::= ANALYZE nm dbnm */ + -6, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (292) add_column_fullname ::= fullname */ + -8, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (294) cmd ::= create_vtab */ + -4, /* (295) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (297) vtabarg ::= */ + -1, /* (298) vtabargtoken ::= ANY */ + -3, /* (299) vtabargtoken ::= lp anylist RP */ + -1, /* (300) lp ::= LP */ + -2, /* (301) with ::= WITH wqlist */ + -3, /* (302) with ::= WITH RECURSIVE wqlist */ + -1, /* (303) wqas ::= AS */ + -2, /* (304) wqas ::= AS MATERIALIZED */ + -3, /* (305) wqas ::= AS NOT MATERIALIZED */ + -6, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ + -1, /* (307) wqlist ::= wqitem */ + -3, /* (308) wqlist ::= wqlist COMMA wqitem */ + -1, /* (309) windowdefn_list ::= windowdefn */ + -3, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (311) windowdefn ::= nm AS LP window RP */ + -5, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (314) window ::= ORDER BY sortlist frame_opt */ + -5, /* (315) window ::= nm ORDER BY sortlist frame_opt */ + -1, /* (316) window ::= frame_opt */ + -2, /* (317) window ::= nm frame_opt */ + 0, /* (318) frame_opt ::= */ + -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (322) frame_bound_s ::= frame_bound */ + -2, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (324) frame_bound_e ::= frame_bound */ + -2, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (327) frame_bound ::= CURRENT ROW */ + 0, /* (328) frame_exclude_opt ::= */ + -2, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (330) frame_exclude ::= NO OTHERS */ + -2, /* (331) frame_exclude ::= CURRENT ROW */ + -1, /* (332) frame_exclude ::= GROUP|TIES */ + -2, /* (333) window_clause ::= WINDOW windowdefn_list */ + -2, /* (334) filter_over ::= filter_clause over_clause */ + -1, /* (335) filter_over ::= over_clause */ + -1, /* (336) filter_over ::= filter_clause */ + -4, /* (337) over_clause ::= OVER LP window RP */ + -2, /* (338) over_clause ::= OVER nm */ + -5, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (340) input ::= cmdlist */ + -2, /* (341) cmdlist ::= cmdlist ecmd */ + -1, /* (342) cmdlist ::= ecmd */ + -1, /* (343) ecmd ::= SEMI */ + -2, /* (344) ecmd ::= cmdx SEMI */ + -3, /* (345) ecmd ::= explain cmdx SEMI */ + 0, /* (346) trans_opt ::= */ + -1, /* (347) trans_opt ::= TRANSACTION */ + -2, /* (348) trans_opt ::= TRANSACTION nm */ + -1, /* (349) savepoint_opt ::= SAVEPOINT */ + 0, /* (350) savepoint_opt ::= */ + -2, /* (351) cmd ::= create_table create_table_args */ + -1, /* (352) table_option_set ::= table_option */ + -4, /* (353) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (354) columnlist ::= columnname carglist */ + -1, /* (355) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (356) nm ::= STRING */ + -1, /* (357) typetoken ::= typename */ + -1, /* (358) typename ::= ID|STRING */ + -1, /* (359) signed ::= plus_num */ + -1, /* (360) signed ::= minus_num */ + -2, /* (361) carglist ::= carglist ccons */ + 0, /* (362) carglist ::= */ + -2, /* (363) ccons ::= NULL onconf */ + -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (365) ccons ::= AS generated */ + -2, /* (366) conslist_opt ::= COMMA conslist */ + -3, /* (367) conslist ::= conslist tconscomma tcons */ + -1, /* (368) conslist ::= tcons */ + 0, /* (369) tconscomma ::= */ + -1, /* (370) defer_subclause_opt ::= defer_subclause */ + -1, /* (371) resolvetype ::= raisetype */ + -1, /* (372) selectnowith ::= oneselect */ + -1, /* (373) oneselect ::= values */ + -2, /* (374) sclp ::= selcollist COMMA */ + -1, /* (375) as ::= ID|STRING */ + -1, /* (376) indexed_opt ::= indexed_by */ + 0, /* (377) returning ::= */ + -1, /* (378) expr ::= term */ + -1, /* (379) likeop ::= LIKE_KW|MATCH */ + -1, /* (380) case_operand ::= expr */ + -1, /* (381) exprlist ::= nexprlist */ + -1, /* (382) nmnum ::= plus_num */ + -1, /* (383) nmnum ::= nm */ + -1, /* (384) nmnum ::= ON */ + -1, /* (385) nmnum ::= DELETE */ + -1, /* (386) nmnum ::= DEFAULT */ + -1, /* (387) plus_num ::= INTEGER|FLOAT */ + 0, /* (388) foreach_clause ::= */ + -3, /* (389) foreach_clause ::= FOR EACH ROW */ + -1, /* (390) trnm ::= nm */ + 0, /* (391) tridxby ::= */ + -1, /* (392) database_kw_opt ::= DATABASE */ + 0, /* (393) database_kw_opt ::= */ + 0, /* (394) kwcolumn_opt ::= */ + -1, /* (395) kwcolumn_opt ::= COLUMNKW */ + -1, /* (396) vtabarglist ::= vtabarg */ + -3, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (398) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (399) anylist ::= */ + -4, /* (400) anylist ::= anylist LP anylist RP */ + -2, /* (401) anylist ::= anylist ANY */ + 0, /* (402) with ::= */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -167031,7 +171316,7 @@ static YYACTIONTYPE yy_reduce( case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 323: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==323); + case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321); {yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ @@ -167068,7 +171353,7 @@ static YYACTIONTYPE yy_reduce( case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 244: /* collate ::= */ yytestcase(yyruleno==244); + case 242: /* collate ::= */ yytestcase(yyruleno==242); {yymsp[1].minor.yy394 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ @@ -167252,9 +171537,9 @@ static YYACTIONTYPE yy_reduce( break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 216: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==216); - case 219: /* in_op ::= NOT IN */ yytestcase(yyruleno==219); - case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); + case 215: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==215); + case 218: /* in_op ::= NOT IN */ yytestcase(yyruleno==218); + case 243: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==243); {yymsp[-1].minor.yy394 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ @@ -167404,9 +171689,9 @@ static YYACTIONTYPE yy_reduce( case 99: /* sclp ::= */ case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 232: /* exprlist ::= */ yytestcase(yyruleno==232); - case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); - case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); + case 230: /* exprlist ::= */ yytestcase(yyruleno==230); + case 233: /* paren_exprlist ::= */ yytestcase(yyruleno==233); + case 238: /* eidlist_opt ::= */ yytestcase(yyruleno==238); {yymsp[1].minor.yy322 = 0;} break; case 100: /* selcollist ::= sclp scanpt expr scanpt as */ @@ -167432,8 +171717,8 @@ static YYACTIONTYPE yy_reduce( break; case 103: /* as ::= AS nm */ case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); + case 254: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==254); + case 255: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==255); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; case 105: /* from ::= */ @@ -167477,7 +171762,7 @@ static YYACTIONTYPE yy_reduce( { if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){ yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131; - }else if( yymsp[-3].minor.yy131->nSrc==1 ){ + }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){ yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); if( yymsp[-5].minor.yy131 ){ SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1]; @@ -167605,16 +171890,16 @@ static YYACTIONTYPE yy_reduce( case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); case 151: /* where_opt ::= */ yytestcase(yyruleno==151); case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 229: /* case_else ::= */ yytestcase(yyruleno==229); - case 231: /* case_operand ::= */ yytestcase(yyruleno==231); - case 250: /* vinto ::= */ yytestcase(yyruleno==250); + case 228: /* case_else ::= */ yytestcase(yyruleno==228); + case 229: /* case_operand ::= */ yytestcase(yyruleno==229); + case 248: /* vinto ::= */ yytestcase(yyruleno==248); {yymsp[1].minor.yy528 = 0;} break; case 145: /* having_opt ::= HAVING expr */ case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 228: /* case_else ::= ELSE expr */ yytestcase(yyruleno==228); - case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); + case 227: /* case_else ::= ELSE expr */ yytestcase(yyruleno==227); + case 247: /* vinto ::= INTO expr */ yytestcase(yyruleno==247); {yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} break; case 147: /* limit_opt ::= LIMIT expr */ @@ -167726,11 +172011,10 @@ static YYACTIONTYPE yy_reduce( case 177: /* expr ::= LP expr RP */ {yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;} break; - case 178: /* expr ::= ID|INDEXED */ - case 179: /* expr ::= JOIN_KW */ yytestcase(yyruleno==179); + case 178: /* expr ::= ID|INDEXED|JOIN_KW */ {yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 180: /* expr ::= nm DOT nm */ + case 179: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); @@ -167738,7 +172022,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy528 = yylhsminor.yy528; break; - case 181: /* expr ::= nm DOT nm DOT nm */ + case 180: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); @@ -167751,18 +172035,18 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 182: /* term ::= NULL|FLOAT|BLOB */ - case 183: /* term ::= STRING */ yytestcase(yyruleno==183); + case 181: /* term ::= NULL|FLOAT|BLOB */ + case 182: /* term ::= STRING */ yytestcase(yyruleno==182); {yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 184: /* term ::= INTEGER */ + case 183: /* term ::= INTEGER */ { yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } yymsp[0].minor.yy528 = yylhsminor.yy528; break; - case 185: /* expr ::= VARIABLE */ + case 184: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; @@ -167784,50 +172068,50 @@ static YYACTIONTYPE yy_reduce( } } break; - case 186: /* expr ::= expr COLLATE ID|STRING */ + case 185: /* expr ::= expr COLLATE ID|STRING */ { yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1); } break; - case 187: /* expr ::= CAST LP expr AS typetoken RP */ + case 186: /* expr ::= CAST LP expr AS typetoken RP */ { yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0); } break; - case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394); } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 189: /* expr ::= ID|INDEXED LP STAR RP */ + case 188: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } yymsp[-3].minor.yy528 = yylhsminor.yy528; break; - case 190: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-5].minor.yy528 = yylhsminor.yy528; break; - case 191: /* expr ::= ID|INDEXED LP STAR RP filter_over */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 192: /* term ::= CTIME_KW */ + case 191: /* term ::= CTIME_KW */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } yymsp[0].minor.yy528 = yylhsminor.yy528; break; - case 193: /* expr ::= LP nexprlist COMMA expr RP */ + case 192: /* expr ::= LP nexprlist COMMA expr RP */ { ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -167841,22 +172125,22 @@ static YYACTIONTYPE yy_reduce( } } break; - case 194: /* expr ::= expr AND expr */ + case 193: /* expr ::= expr AND expr */ {yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 195: /* expr ::= expr OR expr */ - case 196: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==196); - case 197: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==201); + case 194: /* expr ::= expr OR expr */ + case 195: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==195); + case 196: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==196); + case 197: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==197); + case 198: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==198); + case 199: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==200); {yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 202: /* likeop ::= NOT LIKE_KW|MATCH */ + case 201: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 203: /* expr ::= expr likeop expr */ + case 202: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; @@ -167868,7 +172152,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; } break; - case 204: /* expr ::= expr likeop expr ESCAPE expr */ + case 203: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; @@ -167881,47 +172165,47 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr ISNULL|NOTNULL */ + case 204: /* expr ::= expr ISNULL|NOTNULL */ {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} break; - case 206: /* expr ::= expr NOT NULL */ + case 205: /* expr ::= expr NOT NULL */ {yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} break; - case 207: /* expr ::= expr IS expr */ + case 206: /* expr ::= expr IS expr */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); } break; - case 208: /* expr ::= expr IS NOT expr */ + case 207: /* expr ::= expr IS NOT expr */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); } break; - case 209: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 208: /* expr ::= expr IS NOT DISTINCT FROM expr */ { yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); } break; - case 210: /* expr ::= expr IS DISTINCT FROM expr */ + case 209: /* expr ::= expr IS DISTINCT FROM expr */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); } break; - case 211: /* expr ::= NOT expr */ - case 212: /* expr ::= BITNOT expr */ yytestcase(yyruleno==212); + case 210: /* expr ::= NOT expr */ + case 211: /* expr ::= BITNOT expr */ yytestcase(yyruleno==211); {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} break; - case 213: /* expr ::= PLUS|MINUS expr */ + case 212: /* expr ::= PLUS|MINUS expr */ { yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); /*A-overwrites-B*/ } break; - case 214: /* expr ::= expr PTR expr */ + case 213: /* expr ::= expr PTR expr */ { ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); @@ -167929,11 +172213,11 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy528 = yylhsminor.yy528; break; - case 215: /* between_op ::= BETWEEN */ - case 218: /* in_op ::= IN */ yytestcase(yyruleno==218); + case 214: /* between_op ::= BETWEEN */ + case 217: /* in_op ::= IN */ yytestcase(yyruleno==217); {yymsp[0].minor.yy394 = 0;} break; - case 217: /* expr ::= expr between_op expr AND expr */ + case 216: /* expr ::= expr between_op expr AND expr */ { ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); @@ -167946,7 +172230,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 220: /* expr ::= expr in_op LP exprlist RP */ + case 219: /* expr ::= expr in_op LP exprlist RP */ { if( yymsp[-1].minor.yy322==0 ){ /* Expressions of the form @@ -167967,6 +172251,11 @@ static YYACTIONTYPE yy_reduce( sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy528, pRHS); + }else if( yymsp[-1].minor.yy322->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pRHS->x.pSelect); + pRHS->x.pSelect = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); }else{ yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); if( yymsp[-4].minor.yy528==0 ){ @@ -167987,20 +172276,20 @@ static YYACTIONTYPE yy_reduce( } } break; - case 221: /* expr ::= LP select RP */ + case 220: /* expr ::= LP select RP */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); } break; - case 222: /* expr ::= expr in_op LP select RP */ + case 221: /* expr ::= expr in_op LP select RP */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 223: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 222: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); @@ -168010,14 +172299,14 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 224: /* expr ::= EXISTS LP select RP */ + case 223: /* expr ::= EXISTS LP select RP */ { Expr *p; p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); } break; - case 225: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 224: /* expr ::= CASE case_operand case_exprlist case_else END */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); if( yymsp[-4].minor.yy528 ){ @@ -168029,32 +172318,29 @@ static YYACTIONTYPE yy_reduce( } } break; - case 226: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 225: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); } break; - case 227: /* case_exprlist ::= WHEN expr THEN expr */ + case 226: /* case_exprlist ::= WHEN expr THEN expr */ { yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); } break; - case 230: /* case_operand ::= expr */ -{yymsp[0].minor.yy528 = yymsp[0].minor.yy528; /*A-overwrites-X*/} - break; - case 233: /* nexprlist ::= nexprlist COMMA expr */ + case 231: /* nexprlist ::= nexprlist COMMA expr */ {yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} break; - case 234: /* nexprlist ::= expr */ + case 232: /* nexprlist ::= expr */ {yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} break; - case 236: /* paren_exprlist ::= LP exprlist RP */ - case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); + case 234: /* paren_exprlist ::= LP exprlist RP */ + case 239: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==239); {yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} break; - case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 235: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, @@ -168064,48 +172350,48 @@ static YYACTIONTYPE yy_reduce( } } break; - case 238: /* uniqueflag ::= UNIQUE */ - case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); + case 236: /* uniqueflag ::= UNIQUE */ + case 278: /* raisetype ::= ABORT */ yytestcase(yyruleno==278); {yymsp[0].minor.yy394 = OE_Abort;} break; - case 239: /* uniqueflag ::= */ + case 237: /* uniqueflag ::= */ {yymsp[1].minor.yy394 = OE_None;} break; - case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 240: /* eidlist ::= eidlist COMMA nm collate sortorder */ { yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); } break; - case 243: /* eidlist ::= nm collate sortorder */ + case 241: /* eidlist ::= nm collate sortorder */ { yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ } break; - case 246: /* cmd ::= DROP INDEX ifexists fullname */ + case 244: /* cmd ::= DROP INDEX ifexists fullname */ {sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} break; - case 247: /* cmd ::= VACUUM vinto */ + case 245: /* cmd ::= VACUUM vinto */ {sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} break; - case 248: /* cmd ::= VACUUM nm vinto */ + case 246: /* cmd ::= VACUUM nm vinto */ {sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 249: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 250: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 251: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 252: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 253: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 256: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; @@ -168113,50 +172399,50 @@ static YYACTIONTYPE yy_reduce( sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); } break; - case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 257: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 260: /* trigger_time ::= BEFORE|AFTER */ + case 258: /* trigger_time ::= BEFORE|AFTER */ { yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 261: /* trigger_time ::= INSTEAD OF */ + case 259: /* trigger_time ::= INSTEAD OF */ { yymsp[-1].minor.yy394 = TK_INSTEAD;} break; - case 262: /* trigger_time ::= */ + case 260: /* trigger_time ::= */ { yymsp[1].minor.yy394 = TK_BEFORE; } break; - case 263: /* trigger_event ::= DELETE|INSERT */ - case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); + case 261: /* trigger_event ::= DELETE|INSERT */ + case 262: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==262); {yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} break; - case 265: /* trigger_event ::= UPDATE OF idlist */ + case 263: /* trigger_event ::= UPDATE OF idlist */ {yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} break; - case 266: /* when_clause ::= */ - case 285: /* key_opt ::= */ yytestcase(yyruleno==285); + case 264: /* when_clause ::= */ + case 283: /* key_opt ::= */ yytestcase(yyruleno==283); { yymsp[1].minor.yy528 = 0; } break; - case 267: /* when_clause ::= WHEN expr */ - case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); + case 265: /* when_clause ::= WHEN expr */ + case 284: /* key_opt ::= KEY expr */ yytestcase(yyruleno==284); { yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } break; - case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 266: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { assert( yymsp[-2].minor.yy33!=0 ); yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 267: /* trigger_cmd_list ::= trigger_cmd SEMI */ { assert( yymsp[-1].minor.yy33!=0 ); yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 270: /* trnm ::= nm DOT nm */ + case 268: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -168164,39 +172450,39 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 271: /* tridxby ::= INDEXED BY nm */ + case 269: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 272: /* tridxby ::= NOT INDEXED */ + case 270: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + case 271: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-8].minor.yy33 = yylhsminor.yy33; break; - case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 272: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ } yymsp[-7].minor.yy33 = yylhsminor.yy33; break; - case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + case 273: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-5].minor.yy33 = yylhsminor.yy33; break; - case 276: /* trigger_cmd ::= scanpt select scanpt */ + case 274: /* trigger_cmd ::= scanpt select scanpt */ {yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} yymsp[-2].minor.yy33 = yylhsminor.yy33; break; - case 277: /* expr ::= RAISE LP IGNORE RP */ + case 275: /* expr ::= RAISE LP IGNORE RP */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); if( yymsp[-3].minor.yy528 ){ @@ -168204,7 +172490,7 @@ static YYACTIONTYPE yy_reduce( } } break; - case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 276: /* expr ::= RAISE LP raisetype COMMA nm RP */ { yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); if( yymsp[-5].minor.yy528 ) { @@ -168212,118 +172498,118 @@ static YYACTIONTYPE yy_reduce( } } break; - case 279: /* raisetype ::= ROLLBACK */ + case 277: /* raisetype ::= ROLLBACK */ {yymsp[0].minor.yy394 = OE_Rollback;} break; - case 281: /* raisetype ::= FAIL */ + case 279: /* raisetype ::= FAIL */ {yymsp[0].minor.yy394 = OE_Fail;} break; - case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 280: /* cmd ::= DROP TRIGGER ifexists fullname */ { sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); } break; - case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 281: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); } break; - case 284: /* cmd ::= DETACH database_kw_opt expr */ + case 282: /* cmd ::= DETACH database_kw_opt expr */ { sqlite3Detach(pParse, yymsp[0].minor.yy528); } break; - case 287: /* cmd ::= REINDEX */ + case 285: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 288: /* cmd ::= REINDEX nm dbnm */ + case 286: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ANALYZE */ + case 287: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 290: /* cmd ::= ANALYZE nm dbnm */ + case 288: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 289: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); } break; - case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 290: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 291: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); } break; - case 294: /* add_column_fullname ::= fullname */ + case 292: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); } break; - case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 296: /* cmd ::= create_vtab */ + case 294: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 297: /* cmd ::= create_vtab LP vtabarglist RP */ + case 295: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 296: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); } break; - case 299: /* vtabarg ::= */ + case 297: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 300: /* vtabargtoken ::= ANY */ - case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); - case 302: /* lp ::= LP */ yytestcase(yyruleno==302); + case 298: /* vtabargtoken ::= ANY */ + case 299: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==299); + case 300: /* lp ::= LP */ yytestcase(yyruleno==300); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 303: /* with ::= WITH wqlist */ - case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); + case 301: /* with ::= WITH wqlist */ + case 302: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==302); { sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } break; - case 305: /* wqas ::= AS */ + case 303: /* wqas ::= AS */ {yymsp[0].minor.yy516 = M10d_Any;} break; - case 306: /* wqas ::= AS MATERIALIZED */ + case 304: /* wqas ::= AS MATERIALIZED */ {yymsp[-1].minor.yy516 = M10d_Yes;} break; - case 307: /* wqas ::= AS NOT MATERIALIZED */ + case 305: /* wqas ::= AS NOT MATERIALIZED */ {yymsp[-2].minor.yy516 = M10d_No;} break; - case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 306: /* wqitem ::= nm eidlist_opt wqas LP select RP */ { yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ } break; - case 309: /* wqlist ::= wqitem */ + case 307: /* wqlist ::= wqitem */ { yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ } break; - case 310: /* wqlist ::= wqlist COMMA wqitem */ + case 308: /* wqlist ::= wqlist COMMA wqitem */ { yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); } break; - case 311: /* windowdefn_list ::= windowdefn */ + case 309: /* windowdefn_list ::= windowdefn */ { yylhsminor.yy41 = yymsp[0].minor.yy41; } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 312: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 310: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { assert( yymsp[0].minor.yy41!=0 ); sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); @@ -168332,7 +172618,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy41 = yylhsminor.yy41; break; - case 313: /* windowdefn ::= nm AS LP window RP */ + case 311: /* windowdefn ::= nm AS LP window RP */ { if( ALWAYS(yymsp[-1].minor.yy41) ){ yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); @@ -168341,90 +172627,90 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 314: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 312: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); } break; - case 315: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 313: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); } yymsp[-5].minor.yy41 = yylhsminor.yy41; break; - case 316: /* window ::= ORDER BY sortlist frame_opt */ + case 314: /* window ::= ORDER BY sortlist frame_opt */ { yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); } break; - case 317: /* window ::= nm ORDER BY sortlist frame_opt */ + case 315: /* window ::= nm ORDER BY sortlist frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 318: /* window ::= frame_opt */ - case 337: /* filter_over ::= over_clause */ yytestcase(yyruleno==337); + case 316: /* window ::= frame_opt */ + case 335: /* filter_over ::= over_clause */ yytestcase(yyruleno==335); { yylhsminor.yy41 = yymsp[0].minor.yy41; } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 319: /* window ::= nm frame_opt */ + case 317: /* window ::= nm frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); } yymsp[-1].minor.yy41 = yylhsminor.yy41; break; - case 320: /* frame_opt ::= */ + case 318: /* frame_opt ::= */ { yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 321: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516); } yymsp[-2].minor.yy41 = yylhsminor.yy41; break; - case 322: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516); } yymsp[-5].minor.yy41 = yylhsminor.yy41; break; - case 324: /* frame_bound_s ::= frame_bound */ - case 326: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==326); + case 322: /* frame_bound_s ::= frame_bound */ + case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324); {yylhsminor.yy595 = yymsp[0].minor.yy595;} yymsp[0].minor.yy595 = yylhsminor.yy595; break; - case 325: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 327: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==327); - case 329: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==329); + case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325); + case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327); {yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;} yymsp[-1].minor.yy595 = yylhsminor.yy595; break; - case 328: /* frame_bound ::= expr PRECEDING|FOLLOWING */ + case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */ {yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;} yymsp[-1].minor.yy595 = yylhsminor.yy595; break; - case 330: /* frame_exclude_opt ::= */ + case 328: /* frame_exclude_opt ::= */ {yymsp[1].minor.yy516 = 0;} break; - case 331: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ + case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ {yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;} break; - case 332: /* frame_exclude ::= NO OTHERS */ - case 333: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==333); + case 330: /* frame_exclude ::= NO OTHERS */ + case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331); {yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 334: /* frame_exclude ::= GROUP|TIES */ + case 332: /* frame_exclude ::= GROUP|TIES */ {yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/} break; - case 335: /* window_clause ::= WINDOW windowdefn_list */ + case 333: /* window_clause ::= WINDOW windowdefn_list */ { yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; } break; - case 336: /* filter_over ::= filter_clause over_clause */ + case 334: /* filter_over ::= filter_clause over_clause */ { if( yymsp[0].minor.yy41 ){ yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528; @@ -168435,7 +172721,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-1].minor.yy41 = yylhsminor.yy41; break; - case 338: /* filter_over ::= filter_clause */ + case 336: /* filter_over ::= filter_clause */ { yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); if( yylhsminor.yy41 ){ @@ -168447,13 +172733,13 @@ static YYACTIONTYPE yy_reduce( } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 339: /* over_clause ::= OVER LP window RP */ + case 337: /* over_clause ::= OVER LP window RP */ { yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41; assert( yymsp[-3].minor.yy41!=0 ); } break; - case 340: /* over_clause ::= OVER nm */ + case 338: /* over_clause ::= OVER nm */ { yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); if( yymsp[-1].minor.yy41 ){ @@ -168461,73 +172747,73 @@ static YYACTIONTYPE yy_reduce( } } break; - case 341: /* filter_clause ::= FILTER LP WHERE expr RP */ + case 339: /* filter_clause ::= FILTER LP WHERE expr RP */ { yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; } break; default: - /* (342) input ::= cmdlist */ yytestcase(yyruleno==342); - /* (343) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==343); - /* (344) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=344); - /* (345) ecmd ::= SEMI */ yytestcase(yyruleno==345); - /* (346) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==346); - /* (347) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=347); - /* (348) trans_opt ::= */ yytestcase(yyruleno==348); - /* (349) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==349); - /* (350) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==350); - /* (351) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==351); - /* (352) savepoint_opt ::= */ yytestcase(yyruleno==352); - /* (353) cmd ::= create_table create_table_args */ yytestcase(yyruleno==353); - /* (354) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=354); - /* (355) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==355); - /* (356) columnlist ::= columnname carglist */ yytestcase(yyruleno==356); - /* (357) nm ::= ID|INDEXED */ yytestcase(yyruleno==357); - /* (358) nm ::= STRING */ yytestcase(yyruleno==358); - /* (359) nm ::= JOIN_KW */ yytestcase(yyruleno==359); - /* (360) typetoken ::= typename */ yytestcase(yyruleno==360); - /* (361) typename ::= ID|STRING */ yytestcase(yyruleno==361); - /* (362) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=362); - /* (363) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); - /* (364) carglist ::= carglist ccons */ yytestcase(yyruleno==364); - /* (365) carglist ::= */ yytestcase(yyruleno==365); - /* (366) ccons ::= NULL onconf */ yytestcase(yyruleno==366); - /* (367) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==367); - /* (368) ccons ::= AS generated */ yytestcase(yyruleno==368); - /* (369) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==369); - /* (370) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==370); - /* (371) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) tconscomma ::= */ yytestcase(yyruleno==372); - /* (373) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=373); - /* (374) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=374); - /* (375) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=375); - /* (376) oneselect ::= values */ yytestcase(yyruleno==376); - /* (377) sclp ::= selcollist COMMA */ yytestcase(yyruleno==377); - /* (378) as ::= ID|STRING */ yytestcase(yyruleno==378); - /* (379) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=379); - /* (380) returning ::= */ yytestcase(yyruleno==380); - /* (381) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=381); - /* (382) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==382); - /* (383) exprlist ::= nexprlist */ yytestcase(yyruleno==383); - /* (384) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=384); - /* (385) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=385); - /* (386) nmnum ::= ON */ yytestcase(yyruleno==386); - /* (387) nmnum ::= DELETE */ yytestcase(yyruleno==387); - /* (388) nmnum ::= DEFAULT */ yytestcase(yyruleno==388); - /* (389) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==389); - /* (390) foreach_clause ::= */ yytestcase(yyruleno==390); - /* (391) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==391); - /* (392) trnm ::= nm */ yytestcase(yyruleno==392); - /* (393) tridxby ::= */ yytestcase(yyruleno==393); - /* (394) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==394); - /* (395) database_kw_opt ::= */ yytestcase(yyruleno==395); - /* (396) kwcolumn_opt ::= */ yytestcase(yyruleno==396); - /* (397) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==397); - /* (398) vtabarglist ::= vtabarg */ yytestcase(yyruleno==398); - /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==399); - /* (400) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==400); - /* (401) anylist ::= */ yytestcase(yyruleno==401); - /* (402) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==402); - /* (403) anylist ::= anylist ANY */ yytestcase(yyruleno==403); - /* (404) with ::= */ yytestcase(yyruleno==404); + /* (340) input ::= cmdlist */ yytestcase(yyruleno==340); + /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341); + /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342); + /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343); + /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344); + /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345); + /* (346) trans_opt ::= */ yytestcase(yyruleno==346); + /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347); + /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348); + /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349); + /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350); + /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351); + /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352); + /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353); + /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354); + /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355); + /* (356) nm ::= STRING */ yytestcase(yyruleno==356); + /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); + /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); + /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); + /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); + /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); + /* (362) carglist ::= */ yytestcase(yyruleno==362); + /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); + /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); + /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); + /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); + /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); + /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); + /* (369) tconscomma ::= */ yytestcase(yyruleno==369); + /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); + /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); + /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) oneselect ::= values */ yytestcase(yyruleno==373); + /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); + /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); + /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) returning ::= */ yytestcase(yyruleno==377); + /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378); + /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379); + /* (380) case_operand ::= expr */ yytestcase(yyruleno==380); + /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381); + /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383); + /* (384) nmnum ::= ON */ yytestcase(yyruleno==384); + /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385); + /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386); + /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387); + /* (388) foreach_clause ::= */ yytestcase(yyruleno==388); + /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389); + /* (390) trnm ::= nm */ yytestcase(yyruleno==390); + /* (391) tridxby ::= */ yytestcase(yyruleno==391); + /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392); + /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393); + /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394); + /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395); + /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396); + /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397); + /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398); + /* (399) anylist ::= */ yytestcase(yyruleno==399); + /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); + /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); + /* (402) with ::= */ yytestcase(yyruleno==402); break; /********** End reduce actions ************************************************/ }; @@ -169103,7 +173389,7 @@ static const unsigned char aKWHash[127] = { /* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 ** then the i-th keyword has no more hash collisions. Otherwise, ** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[147] = { +static const unsigned char aKWNext[148] = {0, 0, 0, 0, 0, 4, 0, 43, 0, 0, 106, 114, 0, 0, 0, 2, 0, 0, 143, 0, 0, 0, 13, 0, 0, 0, 0, 141, 0, 0, 119, 52, 0, 0, 137, 12, 0, 0, 62, 0, @@ -169118,7 +173404,7 @@ static const unsigned char aKWNext[147] = { 102, 0, 0, 87, }; /* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[147] = { +static const unsigned char aKWLen[148] = {0, 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, @@ -169134,7 +173420,7 @@ static const unsigned char aKWLen[147] = { }; /* aKWOffset[i] is the index into zKWText[] of the start of ** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[147] = { +static const unsigned short int aKWOffset[148] = {0, 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, @@ -169149,7 +173435,7 @@ static const unsigned short int aKWOffset[147] = { 648, 650, 655, 659, }; /* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[147] = { +static const unsigned char aKWCode[148] = {0, TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, @@ -169318,7 +173604,7 @@ static int keywordCode(const char *z, int n, int *pType){ const char *zKW; if( n>=2 ){ i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; - for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ + for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ if( aKWLen[i]!=n ) continue; zKW = &zKWText[aKWOffset[i]]; #ifdef SQLITE_ASCII @@ -169334,153 +173620,153 @@ static int keywordCode(const char *z, int n, int *pType){ while( j<n && toupper(z[j])==zKW[j] ){ j++; } #endif if( j<n ) continue; - testcase( i==0 ); /* REINDEX */ - testcase( i==1 ); /* INDEXED */ - testcase( i==2 ); /* INDEX */ - testcase( i==3 ); /* DESC */ - testcase( i==4 ); /* ESCAPE */ - testcase( i==5 ); /* EACH */ - testcase( i==6 ); /* CHECK */ - testcase( i==7 ); /* KEY */ - testcase( i==8 ); /* BEFORE */ - testcase( i==9 ); /* FOREIGN */ - testcase( i==10 ); /* FOR */ - testcase( i==11 ); /* IGNORE */ - testcase( i==12 ); /* REGEXP */ - testcase( i==13 ); /* EXPLAIN */ - testcase( i==14 ); /* INSTEAD */ - testcase( i==15 ); /* ADD */ - testcase( i==16 ); /* DATABASE */ - testcase( i==17 ); /* AS */ - testcase( i==18 ); /* SELECT */ - testcase( i==19 ); /* TABLE */ - testcase( i==20 ); /* LEFT */ - testcase( i==21 ); /* THEN */ - testcase( i==22 ); /* END */ - testcase( i==23 ); /* DEFERRABLE */ - testcase( i==24 ); /* ELSE */ - testcase( i==25 ); /* EXCLUDE */ - testcase( i==26 ); /* DELETE */ - testcase( i==27 ); /* TEMPORARY */ - testcase( i==28 ); /* TEMP */ - testcase( i==29 ); /* OR */ - testcase( i==30 ); /* ISNULL */ - testcase( i==31 ); /* NULLS */ - testcase( i==32 ); /* SAVEPOINT */ - testcase( i==33 ); /* INTERSECT */ - testcase( i==34 ); /* TIES */ - testcase( i==35 ); /* NOTNULL */ - testcase( i==36 ); /* NOT */ - testcase( i==37 ); /* NO */ - testcase( i==38 ); /* NULL */ - testcase( i==39 ); /* LIKE */ - testcase( i==40 ); /* EXCEPT */ - testcase( i==41 ); /* TRANSACTION */ - testcase( i==42 ); /* ACTION */ - testcase( i==43 ); /* ON */ - testcase( i==44 ); /* NATURAL */ - testcase( i==45 ); /* ALTER */ - testcase( i==46 ); /* RAISE */ - testcase( i==47 ); /* EXCLUSIVE */ - testcase( i==48 ); /* EXISTS */ - testcase( i==49 ); /* CONSTRAINT */ - testcase( i==50 ); /* INTO */ - testcase( i==51 ); /* OFFSET */ - testcase( i==52 ); /* OF */ - testcase( i==53 ); /* SET */ - testcase( i==54 ); /* TRIGGER */ - testcase( i==55 ); /* RANGE */ - testcase( i==56 ); /* GENERATED */ - testcase( i==57 ); /* DETACH */ - testcase( i==58 ); /* HAVING */ - testcase( i==59 ); /* GLOB */ - testcase( i==60 ); /* BEGIN */ - testcase( i==61 ); /* INNER */ - testcase( i==62 ); /* REFERENCES */ - testcase( i==63 ); /* UNIQUE */ - testcase( i==64 ); /* QUERY */ - testcase( i==65 ); /* WITHOUT */ - testcase( i==66 ); /* WITH */ - testcase( i==67 ); /* OUTER */ - testcase( i==68 ); /* RELEASE */ - testcase( i==69 ); /* ATTACH */ - testcase( i==70 ); /* BETWEEN */ - testcase( i==71 ); /* NOTHING */ - testcase( i==72 ); /* GROUPS */ - testcase( i==73 ); /* GROUP */ - testcase( i==74 ); /* CASCADE */ - testcase( i==75 ); /* ASC */ - testcase( i==76 ); /* DEFAULT */ - testcase( i==77 ); /* CASE */ - testcase( i==78 ); /* COLLATE */ - testcase( i==79 ); /* CREATE */ - testcase( i==80 ); /* CURRENT_DATE */ - testcase( i==81 ); /* IMMEDIATE */ - testcase( i==82 ); /* JOIN */ - testcase( i==83 ); /* INSERT */ - testcase( i==84 ); /* MATCH */ - testcase( i==85 ); /* PLAN */ - testcase( i==86 ); /* ANALYZE */ - testcase( i==87 ); /* PRAGMA */ - testcase( i==88 ); /* MATERIALIZED */ - testcase( i==89 ); /* DEFERRED */ - testcase( i==90 ); /* DISTINCT */ - testcase( i==91 ); /* IS */ - testcase( i==92 ); /* UPDATE */ - testcase( i==93 ); /* VALUES */ - testcase( i==94 ); /* VIRTUAL */ - testcase( i==95 ); /* ALWAYS */ - testcase( i==96 ); /* WHEN */ - testcase( i==97 ); /* WHERE */ - testcase( i==98 ); /* RECURSIVE */ - testcase( i==99 ); /* ABORT */ - testcase( i==100 ); /* AFTER */ - testcase( i==101 ); /* RENAME */ - testcase( i==102 ); /* AND */ - testcase( i==103 ); /* DROP */ - testcase( i==104 ); /* PARTITION */ - testcase( i==105 ); /* AUTOINCREMENT */ - testcase( i==106 ); /* TO */ - testcase( i==107 ); /* IN */ - testcase( i==108 ); /* CAST */ - testcase( i==109 ); /* COLUMN */ - testcase( i==110 ); /* COMMIT */ - testcase( i==111 ); /* CONFLICT */ - testcase( i==112 ); /* CROSS */ - testcase( i==113 ); /* CURRENT_TIMESTAMP */ - testcase( i==114 ); /* CURRENT_TIME */ - testcase( i==115 ); /* CURRENT */ - testcase( i==116 ); /* PRECEDING */ - testcase( i==117 ); /* FAIL */ - testcase( i==118 ); /* LAST */ - testcase( i==119 ); /* FILTER */ - testcase( i==120 ); /* REPLACE */ - testcase( i==121 ); /* FIRST */ - testcase( i==122 ); /* FOLLOWING */ - testcase( i==123 ); /* FROM */ - testcase( i==124 ); /* FULL */ - testcase( i==125 ); /* LIMIT */ - testcase( i==126 ); /* IF */ - testcase( i==127 ); /* ORDER */ - testcase( i==128 ); /* RESTRICT */ - testcase( i==129 ); /* OTHERS */ - testcase( i==130 ); /* OVER */ - testcase( i==131 ); /* RETURNING */ - testcase( i==132 ); /* RIGHT */ - testcase( i==133 ); /* ROLLBACK */ - testcase( i==134 ); /* ROWS */ - testcase( i==135 ); /* ROW */ - testcase( i==136 ); /* UNBOUNDED */ - testcase( i==137 ); /* UNION */ - testcase( i==138 ); /* USING */ - testcase( i==139 ); /* VACUUM */ - testcase( i==140 ); /* VIEW */ - testcase( i==141 ); /* WINDOW */ - testcase( i==142 ); /* DO */ - testcase( i==143 ); /* BY */ - testcase( i==144 ); /* INITIALLY */ - testcase( i==145 ); /* ALL */ - testcase( i==146 ); /* PRIMARY */ + testcase( i==1 ); /* REINDEX */ + testcase( i==2 ); /* INDEXED */ + testcase( i==3 ); /* INDEX */ + testcase( i==4 ); /* DESC */ + testcase( i==5 ); /* ESCAPE */ + testcase( i==6 ); /* EACH */ + testcase( i==7 ); /* CHECK */ + testcase( i==8 ); /* KEY */ + testcase( i==9 ); /* BEFORE */ + testcase( i==10 ); /* FOREIGN */ + testcase( i==11 ); /* FOR */ + testcase( i==12 ); /* IGNORE */ + testcase( i==13 ); /* REGEXP */ + testcase( i==14 ); /* EXPLAIN */ + testcase( i==15 ); /* INSTEAD */ + testcase( i==16 ); /* ADD */ + testcase( i==17 ); /* DATABASE */ + testcase( i==18 ); /* AS */ + testcase( i==19 ); /* SELECT */ + testcase( i==20 ); /* TABLE */ + testcase( i==21 ); /* LEFT */ + testcase( i==22 ); /* THEN */ + testcase( i==23 ); /* END */ + testcase( i==24 ); /* DEFERRABLE */ + testcase( i==25 ); /* ELSE */ + testcase( i==26 ); /* EXCLUDE */ + testcase( i==27 ); /* DELETE */ + testcase( i==28 ); /* TEMPORARY */ + testcase( i==29 ); /* TEMP */ + testcase( i==30 ); /* OR */ + testcase( i==31 ); /* ISNULL */ + testcase( i==32 ); /* NULLS */ + testcase( i==33 ); /* SAVEPOINT */ + testcase( i==34 ); /* INTERSECT */ + testcase( i==35 ); /* TIES */ + testcase( i==36 ); /* NOTNULL */ + testcase( i==37 ); /* NOT */ + testcase( i==38 ); /* NO */ + testcase( i==39 ); /* NULL */ + testcase( i==40 ); /* LIKE */ + testcase( i==41 ); /* EXCEPT */ + testcase( i==42 ); /* TRANSACTION */ + testcase( i==43 ); /* ACTION */ + testcase( i==44 ); /* ON */ + testcase( i==45 ); /* NATURAL */ + testcase( i==46 ); /* ALTER */ + testcase( i==47 ); /* RAISE */ + testcase( i==48 ); /* EXCLUSIVE */ + testcase( i==49 ); /* EXISTS */ + testcase( i==50 ); /* CONSTRAINT */ + testcase( i==51 ); /* INTO */ + testcase( i==52 ); /* OFFSET */ + testcase( i==53 ); /* OF */ + testcase( i==54 ); /* SET */ + testcase( i==55 ); /* TRIGGER */ + testcase( i==56 ); /* RANGE */ + testcase( i==57 ); /* GENERATED */ + testcase( i==58 ); /* DETACH */ + testcase( i==59 ); /* HAVING */ + testcase( i==60 ); /* GLOB */ + testcase( i==61 ); /* BEGIN */ + testcase( i==62 ); /* INNER */ + testcase( i==63 ); /* REFERENCES */ + testcase( i==64 ); /* UNIQUE */ + testcase( i==65 ); /* QUERY */ + testcase( i==66 ); /* WITHOUT */ + testcase( i==67 ); /* WITH */ + testcase( i==68 ); /* OUTER */ + testcase( i==69 ); /* RELEASE */ + testcase( i==70 ); /* ATTACH */ + testcase( i==71 ); /* BETWEEN */ + testcase( i==72 ); /* NOTHING */ + testcase( i==73 ); /* GROUPS */ + testcase( i==74 ); /* GROUP */ + testcase( i==75 ); /* CASCADE */ + testcase( i==76 ); /* ASC */ + testcase( i==77 ); /* DEFAULT */ + testcase( i==78 ); /* CASE */ + testcase( i==79 ); /* COLLATE */ + testcase( i==80 ); /* CREATE */ + testcase( i==81 ); /* CURRENT_DATE */ + testcase( i==82 ); /* IMMEDIATE */ + testcase( i==83 ); /* JOIN */ + testcase( i==84 ); /* INSERT */ + testcase( i==85 ); /* MATCH */ + testcase( i==86 ); /* PLAN */ + testcase( i==87 ); /* ANALYZE */ + testcase( i==88 ); /* PRAGMA */ + testcase( i==89 ); /* MATERIALIZED */ + testcase( i==90 ); /* DEFERRED */ + testcase( i==91 ); /* DISTINCT */ + testcase( i==92 ); /* IS */ + testcase( i==93 ); /* UPDATE */ + testcase( i==94 ); /* VALUES */ + testcase( i==95 ); /* VIRTUAL */ + testcase( i==96 ); /* ALWAYS */ + testcase( i==97 ); /* WHEN */ + testcase( i==98 ); /* WHERE */ + testcase( i==99 ); /* RECURSIVE */ + testcase( i==100 ); /* ABORT */ + testcase( i==101 ); /* AFTER */ + testcase( i==102 ); /* RENAME */ + testcase( i==103 ); /* AND */ + testcase( i==104 ); /* DROP */ + testcase( i==105 ); /* PARTITION */ + testcase( i==106 ); /* AUTOINCREMENT */ + testcase( i==107 ); /* TO */ + testcase( i==108 ); /* IN */ + testcase( i==109 ); /* CAST */ + testcase( i==110 ); /* COLUMN */ + testcase( i==111 ); /* COMMIT */ + testcase( i==112 ); /* CONFLICT */ + testcase( i==113 ); /* CROSS */ + testcase( i==114 ); /* CURRENT_TIMESTAMP */ + testcase( i==115 ); /* CURRENT_TIME */ + testcase( i==116 ); /* CURRENT */ + testcase( i==117 ); /* PRECEDING */ + testcase( i==118 ); /* FAIL */ + testcase( i==119 ); /* LAST */ + testcase( i==120 ); /* FILTER */ + testcase( i==121 ); /* REPLACE */ + testcase( i==122 ); /* FIRST */ + testcase( i==123 ); /* FOLLOWING */ + testcase( i==124 ); /* FROM */ + testcase( i==125 ); /* FULL */ + testcase( i==126 ); /* LIMIT */ + testcase( i==127 ); /* IF */ + testcase( i==128 ); /* ORDER */ + testcase( i==129 ); /* RESTRICT */ + testcase( i==130 ); /* OTHERS */ + testcase( i==131 ); /* OVER */ + testcase( i==132 ); /* RETURNING */ + testcase( i==133 ); /* RIGHT */ + testcase( i==134 ); /* ROLLBACK */ + testcase( i==135 ); /* ROWS */ + testcase( i==136 ); /* ROW */ + testcase( i==137 ); /* UNBOUNDED */ + testcase( i==138 ); /* UNION */ + testcase( i==139 ); /* USING */ + testcase( i==140 ); /* VACUUM */ + testcase( i==141 ); /* VIEW */ + testcase( i==142 ); /* WINDOW */ + testcase( i==143 ); /* DO */ + testcase( i==144 ); /* BY */ + testcase( i==145 ); /* INITIALLY */ + testcase( i==146 ); /* ALL */ + testcase( i==147 ); /* PRIMARY */ *pType = aKWCode[i]; break; } @@ -169495,6 +173781,7 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ #define SQLITE_N_KEYWORD 147 SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){ if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + i++; *pzName = zKWText + aKWOffset[i]; *pnName = aKWLen[i]; return SQLITE_OK; @@ -170071,7 +174358,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){ sqlite3DeleteTrigger(db, pParse->pNewTrigger); } - if( pParse->pVList ) sqlite3DbFreeNN(db, pParse->pVList); + if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList); db->pParse = pParentParse; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; @@ -171033,9 +175320,21 @@ SQLITE_API int sqlite3_config(int op, ...){ va_list ap; int rc = SQLITE_OK; - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes + ** are allowed. + */ + if( sqlite3GlobalConfig.isInit ){ + static const u64 mAnytimeConfigOption = 0 + | MASKBIT64( SQLITE_CONFIG_LOG ) + | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) + ; + if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ + return SQLITE_MISUSE_BKPT; + } + testcase( op==SQLITE_CONFIG_LOG ); + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } va_start(ap, op); switch( op ){ @@ -171104,6 +175403,7 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_MEMSTATUS: { + assert( !sqlite3GlobalConfig.isInit ); /* Cannot change at runtime */ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ @@ -171227,8 +175527,10 @@ SQLITE_API int sqlite3_config(int op, ...){ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); */ typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t); + void *pLogArg = va_arg(ap, void*); + AtomicStore(&sqlite3GlobalConfig.xLog, xLog); + AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg); break; } @@ -171242,7 +175544,8 @@ SQLITE_API int sqlite3_config(int op, ...){ ** argument of type int. If non-zero, then URI handling is globally ** enabled. If the parameter is zero, then URI handling is globally ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + int bOpenUri = va_arg(ap, int); + AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri); break; } @@ -171427,18 +175730,19 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ db->lookaside.bMalloced = pBuf==0 ?1:0; db->lookaside.nSlot = nBig+nSm; }else{ - db->lookaside.pStart = db; + db->lookaside.pStart = 0; #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE db->lookaside.pSmallInit = 0; db->lookaside.pSmallFree = 0; - db->lookaside.pMiddle = db; + db->lookaside.pMiddle = 0; #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - db->lookaside.pEnd = db; + db->lookaside.pEnd = 0; db->lookaside.bDisable = 1; db->lookaside.sz = 0; db->lookaside.bMalloced = 0; db->lookaside.nSlot = 0; } + db->lookaside.pTrueEnd = db->lookaside.pEnd; assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; @@ -171517,6 +175821,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_MAINDBNAME: { @@ -171555,6 +175860,8 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, + { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -171582,6 +175889,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ } } va_end(ap); + sqlite3_mutex_leave(db->mutex); return rc; } @@ -172166,6 +176474,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; case SQLITE_NOTICE_RECOVER_ROLLBACK: zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_NOTICE_RBU: zName = "SQLITE_NOTICE_RBU"; break; case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; case SQLITE_DONE: zName = "SQLITE_DONE"; break; @@ -172395,7 +176704,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ */ SQLITE_API void sqlite3_interrupt(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) ){ + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ (void)SQLITE_MISUSE_BKPT; return; } @@ -172403,6 +176714,21 @@ SQLITE_API void sqlite3_interrupt(sqlite3 *db){ AtomicStore(&db->u1.isInterrupted, 1); } +/* +** Return true or false depending on whether or not an interrupt is +** pending on connection db. +*/ +SQLITE_API int sqlite3_is_interrupted(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return AtomicLoad(&db->u1.isInterrupted)!=0; +} /* ** This function is exactly the same as sqlite3_create_function(), except @@ -172447,7 +176773,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But ** the meaning is inverted. So flip the bit. */ assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); - extraFlags ^= SQLITE_FUNC_UNSAFE; + extraFlags ^= SQLITE_FUNC_UNSAFE; /* tag-20230109-1 */ #ifndef SQLITE_OMIT_UTF16 @@ -172465,11 +176791,11 @@ SQLITE_PRIVATE int sqlite3CreateFunc( case SQLITE_ANY: { int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, - (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, + (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, /* tag-20230109-1 */ pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, - (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, + (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, /* tag-20230109-1*/ pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); } if( rc!=SQLITE_OK ){ @@ -172718,7 +177044,7 @@ SQLITE_API int sqlite3_overload_function( rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; sqlite3_mutex_leave(db->mutex); if( rc ) return SQLITE_OK; - zCopy = sqlite3_mprintf(zName); + zCopy = sqlite3_mprintf("%s", zName); if( zCopy==0 ) return SQLITE_NOMEM; return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); @@ -173521,9 +177847,9 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -173930,6 +178256,9 @@ static int openDatabase( #if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) | SQLITE_LegacyAlter #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) + | SQLITE_StmtScanStatus +#endif ; sqlite3HashInit(&db->aCollSeq); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -173952,6 +178281,19 @@ static int openDatabase( goto opendb_out; } +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) + /* Process magic filenames ":localStorage:" and ":sessionStorage:" */ + if( zFilename && zFilename[0]==':' ){ + if( strcmp(zFilename, ":localStorage:")==0 ){ + zFilename = "file:local?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + }else if( strcmp(zFilename, ":sessionStorage:")==0 ){ + zFilename = "file:session?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + } + } +#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */ + /* Parse the filename/URI argument ** ** Only allow sensible combinations of bits in the flags argument. @@ -173982,6 +178324,12 @@ static int openDatabase( sqlite3_free(zErrMsg); goto opendb_out; } + assert( db->pVfs!=0 ); +#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL) + if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){ + db->temp_store = 2; + } +#endif /* Open the backend database driver */ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, @@ -174475,7 +178823,7 @@ SQLITE_API int sqlite3_sleep(int ms){ /* This function works in milliseconds, but the underlying OsSleep() ** API uses microseconds. Hence the 1000's. */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); return rc; } @@ -174531,6 +178879,9 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); } rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; }else{ int nSave = db->busyHandler.nBusy; rc = sqlite3OsFileControl(fd, op, pArg); @@ -175091,7 +179442,7 @@ static char *appendText(char *p, const char *z){ ** Memory layout must be compatible with that generated by the pager ** and expected by sqlite3_uri_parameter() and databaseName(). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API const char *sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, @@ -175127,10 +179478,10 @@ SQLITE_API char *sqlite3_create_filename( ** error to call this routine with any parameter other than a pointer ** previously obtained from sqlite3_create_filename() or a NULL pointer. */ -SQLITE_API void sqlite3_free_filename(char *p){ +SQLITE_API void sqlite3_free_filename(const char *p){ if( p==0 ) return; - p = (char*)databaseName(p); - sqlite3_free(p - 4); + p = databaseName(p); + sqlite3_free((char*)p - 4); } @@ -175381,8 +179732,8 @@ SQLITE_API int sqlite3_snapshot_open( */ SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ int rc = SQLITE_ERROR; - int iDb; #ifndef SQLITE_OMIT_WAL + int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -176937,7 +181288,7 @@ struct Fts3MultiSegReader { int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; @@ -177029,6 +181380,8 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); #endif +SQLITE_PRIVATE int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); + #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ @@ -179633,7 +183986,7 @@ static int fts3TermSelectMerge( ** ** Similar padding is added in the fts3DoclistOrMerge() function. */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -181121,7 +185474,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -181490,7 +185843,7 @@ static int fts3EvalIncrPhraseNext( if( bEof==0 ){ int nList = 0; int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); if( !aDoclist ) return SQLITE_NOMEM; memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); @@ -182032,9 +186385,8 @@ static void fts3EvalNextRow( Fts3Expr *pExpr, /* Expr. to advance to next matching row */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ + if( *pRc==SQLITE_OK && pExpr->bEof==0 ){ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -182511,6 +186863,22 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ } /* +** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array +** has not yet been allocated, allocate and zero it. Otherwise, just zero +** it. +*/ +static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){ + Fts3Table *pTab = (Fts3Table*)pCtx; + UNUSED_PARAMETER(iPhrase); + if( pExpr->aMI==0 ){ + pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( pExpr->aMI==0 ) return SQLITE_NOMEM; + } + memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + return SQLITE_OK; +} + +/* ** Expression pExpr must be of type FTSQUERY_PHRASE. ** ** If it is not already allocated and populated, this function allocates and @@ -182531,7 +186899,6 @@ static int fts3EvalGatherStats( if( pExpr->aMI==0 ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ sqlite3_int64 iPrevId = pCsr->iPrevId; sqlite3_int64 iDocid; @@ -182539,7 +186906,9 @@ static int fts3EvalGatherStats( /* Find the root of the NEAR expression */ pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + while( pRoot->pParent + && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) + ){ pRoot = pRoot->pParent; } iDocid = pRoot->iDocid; @@ -182547,14 +186916,8 @@ static int fts3EvalGatherStats( assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); - } - + rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab); + if( rc!=SQLITE_OK ) return rc; fts3EvalRestart(pCsr, pRoot, &rc); while( pCsr->isEof==0 && rc==SQLITE_OK ){ @@ -182710,6 +187073,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */ int bMatch; /* Check if this phrase descends from an OR expression node. If not, @@ -182724,25 +187088,30 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( if( p->bEof ) bTreeEof = 1; } if( bOr==0 ) return SQLITE_OK; + pRun = pNear; + while( pRun->bDeferred ){ + assert( pRun->pParent ); + pRun = pRun->pParent; + } /* This is the descendent of an OR node. In this case we cannot use ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; + int bEofSave = pRun->bEof; + fts3EvalRestart(pCsr, pRun, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); + if( bEofSave==0 && pRun->iDocid==iDocid ) break; } assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){ + if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){ rc = FTS_CORRUPT_VTAB; } } if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); } } if( rc!=SQLITE_OK ) return rc; @@ -185726,7 +190095,7 @@ static int porterNext( if( n>c->nAllocated ){ char *pNew; c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); if( !pNew ) return SQLITE_NOMEM; c->zToken = pNew; } @@ -186478,7 +190847,7 @@ static int simpleNext( if( n>c->nTokenAllocated ){ char *pNew; c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); if( !pNew ) return SQLITE_NOMEM; c->pToken = pNew; } @@ -187640,7 +192009,7 @@ static int fts3PendingListAppendVarint( /* Allocate or grow the PendingList as required. */ if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); + p = sqlite3_malloc64(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } @@ -187649,14 +192018,14 @@ static int fts3PendingListAppendVarint( p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } - p->nSpace = nNew; + p->nSpace = (int)nNew; p->aData = (char *)&p[1]; } @@ -188213,7 +192582,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int nByte = sqlite3_blob_bytes(p->pSegments); *pnBlob = nByte; if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ @@ -188330,7 +192699,7 @@ static int fts3SegReaderNext( int nTerm = fts3HashKeysize(pElem); if( (nTerm+1)>pReader->nTermAlloc ){ sqlite3_free(pReader->zTerm); - pReader->zTerm = (char*)sqlite3_malloc((nTerm+1)*2); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); if( !pReader->zTerm ) return SQLITE_NOMEM; pReader->nTermAlloc = (nTerm+1)*2; } @@ -188338,7 +192707,7 @@ static int fts3SegReaderNext( pReader->zTerm[nTerm] = '\0'; pReader->nTerm = nTerm; - aCopy = (char*)sqlite3_malloc(nCopy); + aCopy = (char*)sqlite3_malloc64(nCopy); if( !aCopy ) return SQLITE_NOMEM; memcpy(aCopy, pList->aData, nCopy); pReader->nNode = pReader->nDoclist = nCopy; @@ -188625,7 +192994,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( nExtra = nRoot + FTS3_NODE_PADDING; } - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } @@ -188717,7 +193086,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( if( nElem==nAlloc ){ Fts3HashElem **aElem2; nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem2 = (Fts3HashElem **)sqlite3_realloc64( aElem, nAlloc*sizeof(Fts3HashElem *) ); if( !aElem2 ){ @@ -189051,7 +193420,7 @@ static int fts3NodeAddTerm( ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); + pTree->aData = (char *)sqlite3_malloc64(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } @@ -189069,7 +193438,7 @@ static int fts3NodeAddTerm( if( isCopyTerm ){ if( pTree->nMalloc<nTerm ){ - char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189095,7 +193464,7 @@ static int fts3NodeAddTerm( ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); if( !pNew ){ return SQLITE_NOMEM; } @@ -189233,7 +193602,7 @@ static int fts3SegWriterAdd( ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ + i64 nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; @@ -189242,13 +193611,13 @@ static int fts3SegWriterAdd( sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = p->nNodeSize; @@ -189323,7 +193692,7 @@ static int fts3SegWriterAdd( ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; @@ -189348,7 +193717,7 @@ static int fts3SegWriterAdd( */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189656,18 +194025,20 @@ static void fts3ColumnFilter( static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, - int nList + i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } @@ -189717,7 +194088,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); if( rc!=SQLITE_OK ) return rc; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); pList = pMsr->aBuffer; @@ -189854,11 +194225,11 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ return SQLITE_OK; } -static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ if( nReq>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = nReq*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } @@ -189949,7 +194320,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; @@ -190002,7 +194374,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; if( isFirst ){ @@ -190028,7 +194401,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ - rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); pCsr->aDoclist = pCsr->aBuffer; @@ -190741,7 +195114,7 @@ struct NodeReader { static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); if( a ){ pBlob->nAlloc = nAlloc; pBlob->a = a; @@ -191538,7 +195911,7 @@ static int fts3RepackSegdirLevel( if( nIdx>=nAlloc ){ int *aNew; nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); if( !aNew ){ rc = SQLITE_NOMEM; break; @@ -191912,7 +196285,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; @@ -192548,7 +196921,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( return SQLITE_OK; } - pRet = (char *)sqlite3_malloc(p->pList->nData); + pRet = (char *)sqlite3_malloc64(p->pList->nData); if( !pRet ) return SQLITE_NOMEM; nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); @@ -192568,7 +196941,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } @@ -192847,7 +197220,7 @@ typedef sqlite3_int64 i64; /* -** Used as an fts3ExprIterate() context when loading phrase doclists to +** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; @@ -192891,7 +197264,7 @@ struct SnippetFragment { }; /* -** This type is used as an fts3ExprIterate() context object while +** This type is used as an sqlite3Fts3ExprIterate() context object while ** accumulating the data returned by the matchinfo() function. */ typedef struct MatchInfo MatchInfo; @@ -193050,7 +197423,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){ } /* -** Helper function for fts3ExprIterate() (see below). +** Helper function for sqlite3Fts3ExprIterate() (see below). */ static int fts3ExprIterate2( Fts3Expr *pExpr, /* Expression to iterate phrases of */ @@ -193084,7 +197457,7 @@ static int fts3ExprIterate2( ** Otherwise, SQLITE_OK is returned after a callback has been made for ** all eligible phrase nodes. */ -static int fts3ExprIterate( +SQLITE_PRIVATE int sqlite3Fts3ExprIterate( Fts3Expr *pExpr, /* Expression to iterate phrases of */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ @@ -193093,10 +197466,9 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } - /* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** This is an sqlite3Fts3ExprIterate() callback used while loading the +** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also ** fts3ExprLoadDoclists(). */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ @@ -193128,9 +197500,9 @@ static int fts3ExprLoadDoclists( int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ - LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */ sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx); if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; @@ -193143,7 +197515,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); return nPhrase; } @@ -193271,8 +197643,9 @@ static void fts3SnippetDetails( } /* -** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). -** Each invocation populates an element of the SnippetIter.aPhrase[] array. +** This function is an sqlite3Fts3ExprIterate() callback used by +** fts3BestSnippet(). Each invocation populates an element of the +** SnippetIter.aPhrase[] array. */ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ SnippetIter *p = (SnippetIter *)ctx; @@ -193362,7 +197735,9 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter + ); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ @@ -193723,10 +198098,10 @@ static int fts3ExprLHitGather( } /* -** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. +** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo +** stats for a single query. ** -** fts3ExprIterate() callback to load the 'global' elements of a +** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements ** of the matchinfo array that are constant for all rows returned by the ** current query. @@ -193761,7 +198136,7 @@ static int fts3ExprGlobalHitsCb( } /* -** fts3ExprIterate() callback used to collect the "local" part of the +** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the ** array that are different for each row returned by the query. */ @@ -193957,7 +198332,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ **/ aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase); if( !aIter ) return SQLITE_NOMEM; - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); for(i=0; i<pInfo->nPhrase; i++){ LcsIterator *pIter = &aIter[i]; @@ -194134,11 +198509,11 @@ static int fts3MatchinfoValues( rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); if( rc!=SQLITE_OK ) break; } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); sqlite3Fts3EvalTestDeferred(pCsr, &rc); if( rc!=SQLITE_OK ) break; } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); break; } } @@ -194361,7 +198736,7 @@ struct TermOffsetCtx { }; /* -** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). +** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets(). */ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ TermOffsetCtx *p = (TermOffsetCtx *)ctx; @@ -194443,7 +198818,9 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( */ sCtx.iCol = iCol; sCtx.iTerm = 0; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx + ); if( rc!=SQLITE_OK ) goto offsets_out; /* Retreive the text stored in column iCol. If an SQL NULL is stored @@ -195450,6 +199827,7 @@ static const char * const jsonType[] = { #define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ #define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ #define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ /* A single node of parsed JSON @@ -195476,10 +199854,12 @@ struct JsonParse { JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ u16 iDepth; /* Nesting depth */ + u8 nErr; /* Number of errors seen */ + u8 oom; /* Set to true if out of memory */ + u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ int nJson; /* Length of the zJson string in bytes */ + u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ }; @@ -195487,10 +199867,10 @@ struct JsonParse { ** Maximum nesting depth of JSON for this implementation. ** ** This limit is needed to avoid a stack overflow in the recursive -** descent parser. A depth of 2000 is far deeper than any sane JSON -** should go. +** descent parser. A depth of 1000 is far deeper than any sane JSON +** should go. Historical note: This limit was 2000 prior to version 3.42.0 */ -#define JSON_MAX_DEPTH 2000 +#define JSON_MAX_DEPTH 1000 /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -195641,6 +200021,129 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ } /* +** The zIn[0..N] string is a JSON5 string literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ + u32 i; + jsonAppendChar(p, '"'); + zIn++; + N -= 2; + while( N>0 ){ + for(i=0; i<N && zIn[i]!='\\'; i++){} + if( i>0 ){ + jsonAppendRaw(p, zIn, i); + zIn += i; + N -= i; + if( N==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(p, '\''); + break; + case 'v': + jsonAppendRaw(p, "\\u0009", 6); + break; + case 'x': + jsonAppendRaw(p, "\\u00", 4); + jsonAppendRaw(p, &zIn[2], 2); + zIn += 2; + N -= 2; + break; + case '0': + jsonAppendRaw(p, "\\u0000", 6); + break; + case '\r': + if( zIn[2]=='\n' ){ + zIn++; + N--; + } + break; + case '\n': + break; + case 0xe2: + assert( N>=4 ); + assert( 0x80==(u8)zIn[2] ); + assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + zIn += 2; + N -= 2; + break; + default: + jsonAppendRaw(p, zIn, 2); + break; + } + zIn += 2; + N -= 2; + } + jsonAppendChar(p, '"'); +} + +/* +** The zIn[0..N] string is a JSON5 integer literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){ + sqlite3_int64 i = 0; + int rc = sqlite3DecOrHexToI64(zIn, &i); + if( rc<=1 ){ + jsonPrintf(100,p,"%lld",i); + }else{ + assert( rc==2 ); + jsonAppendRaw(p, "9.0e999", 7); + } + return; + } + jsonAppendRaw(p, zIn, N); +} + +/* +** The zIn[0..N] string is a JSON5 real literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + if( zIn[0]=='.' ){ + jsonAppendChar(p, '0'); + } + for(i=0; i<N; i++){ + if( zIn[i]=='.' && (i+1==N || !sqlite3Isdigit(zIn[i+1])) ){ + i++; + jsonAppendRaw(p, zIn, i); + zIn += i; + N -= i; + jsonAppendChar(p, '0'); + break; + } + } + if( N>0 ){ + jsonAppendRaw(p, zIn, N); + } +} + + + +/* ** Append a function parameter value to the JSON string under ** construction. */ @@ -195653,8 +200156,11 @@ static void jsonAppendValue( jsonAppendRaw(p, "null", 4); break; } - case SQLITE_INTEGER: case SQLITE_FLOAT: { + jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); + break; + } + case SQLITE_INTEGER: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); jsonAppendRaw(p, z, n); @@ -195767,17 +200273,38 @@ static void jsonRenderNode( break; } case JSON_STRING: { + assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; + if( pNode->jnFlags & JNODE_LABEL ){ + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); + }else{ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); } - /* no break */ deliberate_fall_through + break; + } + case JSON_REAL: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } + break; } - case JSON_REAL: case JSON_INT: { assert( pNode->eU==1 ); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } break; } case JSON_ARRAY: { @@ -195893,59 +200420,41 @@ static void jsonReturn( } case JSON_INT: { sqlite3_int64 i = 0; + int rc; + int bNeg = 0; const char *z; + + assert( pNode->eU==1 ); z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; - } - } - } - i = i*10 + v; + if( z[0]=='-' ){ z++; bNeg = 1; } + else if( z[0]=='+' ){ z++; } + rc = sqlite3DecOrHexToI64(z, &i); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -i : i); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: break; - int_as_real: ; /* no break */ deliberate_fall_through } case JSON_REAL: { double r; -#ifdef SQLITE_AMALGAMATION const char *z; assert( pNode->eU==1 ); + to_double: z = pNode->u.zJContent; sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - assert( pNode->eU==1 ); - r = strtod(pNode->u.zJContent, 0); -#endif sqlite3_result_double(pCtx, r); break; } case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ if( pNode->jnFlags & JNODE_RAW ){ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, @@ -195957,18 +200466,17 @@ static void jsonReturn( const char *z; char *zOut; u32 j; + u32 nOut = n; assert( pNode->eU==1 ); z = pNode->u.zJContent; - zOut = sqlite3_malloc( n+1 ); + zOut = sqlite3_malloc( nOut+1 ); if( zOut==0 ){ sqlite3_result_error_nomem(pCtx); break; } for(i=1, j=0; i<n-1; i++){ char c = z[i]; - if( c!='\\' ){ - zOut[j++] = c; - }else{ + if( c=='\\' ){ c = z[++i]; if( c=='u' ){ u32 v = jsonHexToInt4(z+i+1); @@ -196000,22 +200508,40 @@ static void jsonReturn( zOut[j++] = 0x80 | (v&0x3f); } } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); + i += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + i++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + i += 2; + continue; }else{ - if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - } - zOut[j++] = c; + continue; } - } - } + } /* end if( c=='\\' ) */ + zOut[j++] = c; + } /* end for() */ zOut[j] = 0; sqlite3_result_text(pCtx, zOut, j, sqlite3_free); } @@ -196083,8 +200609,8 @@ static int jsonParseAddNode( return jsonParseAddNodeExpand(pParse, eType, n, zContent); } p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; + p->eType = (u8)(eType & 0xff); + p->jnFlags = (u8)(eType >> 8); VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; @@ -196092,21 +200618,177 @@ static int jsonParseAddNode( } /* +** Return true if z[] begins with 2 (or more) hexadecimal digits +*/ +static int jsonIs2Hex(const char *z){ + return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]); +} + +/* ** Return true if z[] begins with 4 (or more) hexadecimal digits */ static int jsonIs4Hex(const char *z){ - int i; - for(i=0; i<4; i++) if( !sqlite3Isxdigit(z[i]) ) return 0; - return 1; + return jsonIs2Hex(z) && jsonIs2Hex(&z[2]); +} + +/* +** Return the number of bytes of JSON5 whitespace at the beginning of +** the input string z[]. +** +** JSON5 whitespace consists of any of the following characters: +** +** Unicode UTF-8 Name +** U+0009 09 horizontal tab +** U+000a 0a line feed +** U+000b 0b vertical tab +** U+000c 0c form feed +** U+000d 0d carriage return +** U+0020 20 space +** U+00a0 c2 a0 non-breaking space +** U+1680 e1 9a 80 ogham space mark +** U+2000 e2 80 80 en quad +** U+2001 e2 80 81 em quad +** U+2002 e2 80 82 en space +** U+2003 e2 80 83 em space +** U+2004 e2 80 84 three-per-em space +** U+2005 e2 80 85 four-per-em space +** U+2006 e2 80 86 six-per-em space +** U+2007 e2 80 87 figure space +** U+2008 e2 80 88 punctuation space +** U+2009 e2 80 89 thin space +** U+200a e2 80 8a hair space +** U+2028 e2 80 a8 line separator +** U+2029 e2 80 a9 paragraph separator +** U+202f e2 80 af narrow no-break space (NNBSP) +** U+205f e2 81 9f medium mathematical space (MMSP) +** U+3000 e3 80 80 ideographical space +** U+FEFF ef bb bf byte order mark +** +** In addition, comments between '/', '*' and '*', '/' and +** from '/', '/' to end-of-line are also considered to be whitespace. +*/ +static int json5Whitespace(const char *zIn){ + int n = 0; + const u8 *z = (u8*)zIn; + while( 1 /*exit by "goto whitespace_done"*/ ){ + switch( z[n] ){ + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x20: { + n++; + break; + } + case '/': { + if( z[n+1]=='*' && z[n+2]!=0 ){ + int j; + for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ + if( z[j]==0 ) goto whitespace_done; + } + n = j+1; + break; + }else if( z[n+1]=='/' ){ + int j; + char c; + for(j=n+2; (c = z[j])!=0; j++){ + if( c=='\n' || c=='\r' ) break; + if( 0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]) + ){ + j += 2; + break; + } + } + n = j; + if( z[n] ) n++; + break; + } + goto whitespace_done; + } + case 0xc2: { + if( z[n+1]==0xa0 ){ + n += 2; + break; + } + goto whitespace_done; + } + case 0xe1: { + if( z[n+1]==0x9a && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe2: { + if( z[n+1]==0x80 ){ + u8 c = z[n+2]; + if( c<0x80 ) goto whitespace_done; + if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){ + n += 3; + break; + } + }else if( z[n+1]==0x81 && z[n+2]==0x9f ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe3: { + if( z[n+1]==0x80 && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xef: { + if( z[n+1]==0xbb && z[n+2]==0xbf ){ + n += 3; + break; + } + goto whitespace_done; + } + default: { + goto whitespace_done; + } + } + } + whitespace_done: + return n; } /* +** Extra floating-point literals to allow in JSON. +*/ +static const struct NanInfName { + char c1; + char c2; + char n; + char eType; + char nRepl; + char *zMatch; + char *zRepl; +} aNanInfName[] = { + { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, +}; + +/* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. ** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. +** Special return values: +** +** 0 End if input +** -1 Syntax error +** -2 '}' seen +** -3 ']' seen +** -4 ',' seen +** -5 ':' seen */ static int jsonParseValue(JsonParse *pParse, u32 i){ char c; @@ -196115,151 +200797,430 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ int x; JsonNode *pNode; const char *z = pParse->zJson; - while( fast_isspace(z[i]) ){ i++; } - if( (c = z[i])=='{' ){ +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + u32 nNode = pParse->nNode; x = jsonParseValue(pParse, j); - if( x<0 ){ - pParse->iDepth--; - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; + if( x<=0 ){ + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + ){ + k++; + } + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } } if( pParse->oom ) return -1; - pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; + pNode = &pParse->aNode[nNode]; + if( pNode->eType!=JSON_STRING ){ + pParse->iErr = j; + return -1; + } pNode->jnFlags |= JNODE_LABEL; j = x; - while( fast_isspace(z[j]) ){ j++; } - if( z[j]!=':' ) return -1; - j++; + if( z[j]==':' ){ + j++; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonParseValue(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ) return -1; + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + pParse->iDepth--; return j+1; - }else if( c=='[' ){ + } + case '[': { /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); if( iThis<0 ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; return -1; } j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + pParse->iDepth--; return j+1; - }else if( c=='"' ){ + } + case '\'': { + u8 jnFlags; + char cDelim; + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + goto parse_string; + case '"': /* Parse string */ - u8 jnFlags = 0; + jnFlags = 0; + parse_string: + cDelim = z[i]; j = i+1; for(;;){ c = z[j]; if( (c & ~0x1f)==0 ){ /* Control characters are not allowed in strings */ + pParse->iErr = j; return -1; } if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(z+j+1)) ){ - jnFlags = JNODE_ESCAPE; + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + jnFlags |= JNODE_ESCAPE; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->hasNonstd = 1; }else{ + pParse->iErr = j; return -1; } - }else if( c=='"' ){ + }else if( c==cDelim ){ break; } j++; } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); return j+1; - }else if( c=='n' - && strncmp(z+i,"null",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(z+i,"true",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(z+i,"false",5)==0 - && !sqlite3Isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenDP, seenE, jnFlags; + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + seenE = 0; + seenDP = JSON_REAL; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; + jnFlags = 0; + parse_number: + seenDP = JSON_INT; + seenE = 0; assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + if( c<='0' ){ - j = c=='-' ? i+1 : i; - if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( seenDP==JSON_INT ); + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + }else{ + jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } } - j = i+1; - for(;; j++){ + parse_number_2: + for(j=i+1;; j++){ c = z[j]; - if( c>='0' && c<='9' ) continue; + if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( z[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; + if( seenDP==JSON_REAL ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; continue; } if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ) return -1; - if( seenE ) return -1; - seenDP = seenE = 1; + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; + seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ j++; c = z[j+1]; } - if( c<'0' || c>'9' ) return -1; + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } continue; } break; } - if( z[j-1]<'0' ) return -1; - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &z[i]); + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); return j; - }else if( c=='}' ){ + } + case '}': { + pParse->iErr = i; return -2; /* End of {...} */ - }else if( c==']' ){ + } + case ']': { + pParse->iErr = i; return -3; /* End of [...] */ - }else if( c==0 ){ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { return 0; /* End of file */ - }else{ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; k<sizeof(aNanInfName)/sizeof(aNanInfName[0]); k++){ + if( c!=aNanInfName[k].c1 && c!=aNanInfName[k].c2 ) continue; + nn = aNanInfName[k].n; + if( sqlite3StrNICmp(&z[i], aNanInfName[k].zMatch, nn)!=0 ){ + continue; + } + if( sqlite3Isalnum(z[i+nn]) ) continue; + jsonParseAddNode(pParse, aNanInfName[k].eType, + aNanInfName[k].nRepl, aNanInfName[k].zRepl); + pParse->hasNonstd = 1; + return i + nn; + } + pParse->iErr = i; return -1; /* Syntax error */ } + } /* End switch(z[i]) */ } /* @@ -196283,7 +201244,14 @@ static int jsonParse( if( i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } } if( i<=0 ){ if( pCtx!=0 ){ @@ -196354,6 +201322,15 @@ static int jsonParseFindParents(JsonParse *pParse){ ** is no longer valid, parse the JSON again and return the new parse, ** and also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. +** +** If an error occurs and pErrCtx!=0 then report the error on pErrCtx +** and return NULL. +** +** If an error occurs and pErrCtx==0 then return the Parse object with +** JsonParse.nErr non-zero. If the caller invokes this routine with +** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller +** is responsible for invoking jsonParseFree() on the returned value. +** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. */ static JsonParse *jsonParseCached( sqlite3_context *pCtx, @@ -196403,6 +201380,10 @@ static JsonParse *jsonParseCached( p->zJson = (char*)&p[1]; memcpy((char*)p->zJson, zJson, nJson+1); if( jsonParse(p, pErrCtx, p->zJson) ){ + if( pErrCtx==0 ){ + p->nErr = 1; + return p; + } sqlite3_free(p); return 0; } @@ -196417,7 +201398,7 @@ static JsonParse *jsonParseCached( ** Compare the OBJECT label at pNode against zKey,nKey. Return true on ** a match. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ +static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ if( pNode->n!=nKey ) return 0; @@ -196427,6 +201408,15 @@ static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; } } +static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ + if( p1->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p2, p1->u.zJContent, p1->n); + }else if( p2->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p1, p2->u.zJContent, p2->n); + }else{ + return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; + } +} /* forward declaration */ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); @@ -196897,7 +201887,7 @@ static void jsonExtractFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) return; if( flags & JSON_ABPATH ){ - if( zPath[0]!='$' ){ + if( zPath[0]!='$' || (zPath[1]!='.' && zPath[1]!='[' && zPath[1]!=0) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for ** convenience. @@ -196988,12 +201978,10 @@ static JsonNode *jsonMergePatch( assert( pPatch[i].eU==1 ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){ assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; @@ -197280,8 +202268,8 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. +** Return 1 if JSON is a well-formed canonical JSON string according +** to RFC-7159. Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -197290,8 +202278,69 @@ static void jsonValidFunc( ){ JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + p = jsonParseCached(ctx, argv, 0); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else{ + sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); + if( p->nErr ) jsonParseFree(p); + } +} + +/* +** json_error_position(JSON) +** +** If the argument is not an interpretable JSON string, then return the 1-based +** character position at which the parser first recognized that the input +** was in error. The left-most character is 1. If the string is valid +** JSON, then return 0. +** +** Note that json_valid() is only true for strictly conforming canonical JSON. +** But this routine returns zero if the input contains extension. Thus: +** +** (1) If the input X is strictly conforming canonical JSON: +** +** json_valid(X) returns true +** json_error_position(X) returns 0 +** +** (2) If the input X is JSON but it includes extension (such as JSON5) that +** are not part of RFC-8259: +** +** json_valid(X) returns false +** json_error_position(X) return 0 +** +** (3) If the input X cannot be interpreted as JSON even taking extensions +** into account: +** +** json_valid(X) return false +** json_error_position(X) returns 1 or more +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else if( p->nErr==0 ){ + sqlite3_result_int(ctx, 0); + }else{ + int n = 1; + u32 i; + const char *z = p->zJson; + for(i=0; i<p->iErr && ALWAYS(z[i]); i++){ + if( (z[i]&0xc0)!=0x80 ) n++; + } + sqlite3_result_int(ctx, n); + jsonParseFree(p); + } } @@ -197635,14 +202684,16 @@ static void jsonAppendObjectPathElement( assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - assert( nn>=2 ); - assert( z[0]=='"' ); - assert( z[nn-1]=='"' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){} - if( jj==nn-1 ){ - z++; - nn -= 2; + if( (pNode->jnFlags & JNODE_RAW)==0 ){ + assert( nn>=2 ); + assert( z[0]=='"' || z[0]=='\'' ); + assert( z[nn-1]=='"' || z[0]=='\'' ); + if( nn>2 && sqlite3Isalpha(z[1]) ){ + for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){} + if( jj==nn-1 ){ + z++; + nn -= 2; + } } } jsonPrintf(nn+2, pStr, ".%.*s", nn, z); @@ -197819,6 +202870,13 @@ static int jsonEachBestIndex( idxMask |= iMask; } } + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + if( (unusableMask & ~idxMask)!=0 ){ /* If there are any unusable constraints on JSON or ROOT, then reject ** this entire plan */ @@ -197995,6 +203053,7 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_array, -1, 0, jsonArrayFunc), JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1, 0, jsonErrorFunc), JFUNCTION(json_extract, -1, 0, jsonExtractFunc), JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), @@ -198014,10 +203073,10 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS), + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) }; sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); #endif @@ -198519,16 +203578,17 @@ struct RtreeMatchArg { ** at run-time. */ #ifndef SQLITE_BYTEORDER -#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SQLITE_BYTEORDER 1234 -#elif defined(sparc) || defined(__ppc__) -# define SQLITE_BYTEORDER 4321 -#else -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ -#endif +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif @@ -198549,7 +203609,7 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ - assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( (((sqlite3_uint64)p)&3)==0 ); /* p is always 4-byte aligned */ #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 pCoord->u = _byteswap_ulong(*(u32*)p); #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -198603,7 +203663,7 @@ static void writeInt16(u8 *p, int i){ } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; - assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( (((sqlite3_uint64)p)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); #if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -199331,7 +204391,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ case RTREE_FALSE: break; /* Never satisfied */ @@ -199384,7 +204444,7 @@ static void rtreeLeafConstraint( || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ @@ -201283,7 +206343,7 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + memset(&cell, 0, sizeof(cell)); /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: @@ -202756,7 +207816,7 @@ static GeoPoly *geopolyFuncParam( int nByte; testcase( pCtx==0 ); if( sqlite3_value_type(pVal)==SQLITE_BLOB - && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + && (nByte = sqlite3_value_bytes(pVal))>=(int)(4+6*sizeof(GeoCoord)) ){ const unsigned char *a = sqlite3_value_blob(pVal); int nVertex; @@ -202814,6 +207874,7 @@ static void geopolyBlobFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); @@ -202833,6 +207894,7 @@ static void geopolyJsonFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3 *db = sqlite3_context_db_handle(context); sqlite3_str *x = sqlite3_str_new(db); @@ -202914,6 +207976,7 @@ static void geopolyXformFunc( double F = sqlite3_value_double(argv[6]); GeoCoord x1, y1, x0, y0; int ii; + (void)argc; if( p ){ for(ii=0; ii<p->nVertex; ii++){ x0 = GeoX(p,ii); @@ -202964,6 +208027,7 @@ static void geopolyAreaFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_double(context, geopolyArea(p)); sqlite3_free(p); @@ -202989,6 +208053,7 @@ static void geopolyCcwFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ if( geopolyArea(p)<0.0 ){ int ii, jj; @@ -203043,6 +208108,7 @@ static void geopolyRegularFunc( int n = sqlite3_value_int(argv[3]); int i; GeoPoly *p; + (void)argc; if( n<3 || r<=0.0 ) return; if( n>1000 ) n = 1000; @@ -203152,6 +208218,7 @@ static void geopolyBBoxFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); @@ -203179,6 +208246,7 @@ static void geopolyBBoxStep( ){ RtreeCoord a[4]; int rc = SQLITE_OK; + (void)argc; (void)geopolyBBox(context, argv[0], a, &rc); if( rc==SQLITE_OK ){ GeoBBox *pBBox; @@ -203267,6 +208335,8 @@ static void geopolyContainsPointFunc( int v = 0; int cnt = 0; int ii; + (void)argc; + if( p1==0 ) return; for(ii=0; ii<p1->nVertex-1; ii++){ v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), @@ -203306,6 +208376,7 @@ static void geopolyWithinFunc( ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ @@ -203636,6 +208707,7 @@ static void geopolyOverlapFunc( ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ @@ -203656,8 +208728,12 @@ static void geopolyDebugFunc( int argc, sqlite3_value **argv ){ + (void)context; + (void)argc; #ifdef GEOPOLY_ENABLE_DEBUG geo_debug = sqlite3_value_int(argv[0]); +#else + (void)argv; #endif } @@ -203685,6 +208761,7 @@ static int geopolyInit( sqlite3_str *pSql; char *zSql; int ii; + (void)pAux; sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); @@ -203801,6 +208878,7 @@ static int geopolyFilter( RtreeNode *pRoot = 0; int rc = SQLITE_OK; int iCell = 0; + (void)idxStr; rtreeReference(pRtree); @@ -203927,6 +209005,7 @@ static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iRowidTerm = -1; int iFuncTerm = -1; int idxNum = 0; + (void)tab; for(ii=0; ii<pIdxInfo->nConstraint; ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; @@ -204147,7 +209226,7 @@ static int geopolyUpdate( sqlite3_free(p); nChange = 1; } - for(jj=1; jj<pRtree->nAux; jj++){ + for(jj=1; jj<nData-2; jj++){ nChange++; sqlite3_bind_value(pUp, jj+2, aData[jj+2]); } @@ -204173,6 +209252,8 @@ static int geopolyFindFunction( void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ + (void)pVtab; + (void)nArg; if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ *pxFunc = geopolyOverlapFunc; *ppArg = 0; @@ -204242,7 +209323,7 @@ static int sqlite3_geopoly_init(sqlite3 *db){ } aAgg[] = { { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, }; - int i; + unsigned int i; for(i=0; i<sizeof(aFunc)/sizeof(aFunc[0]) && rc==SQLITE_OK; i++){ int enc; if( aFunc[i].bPure ){ @@ -204750,8 +209831,9 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ if( U_SUCCESS(status) ){ sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); - }else{ - assert(!pExpr); + pExpr = sqlite3_get_auxdata(p, 0); + } + if( !pExpr ){ icuFunctionError(p, "uregex_open", status); return; } @@ -205462,7 +210544,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** The order of the columns in the data_% table does not matter. ** ** Instead of a regular table, the RBU database may also contain virtual -** tables or view named using the data_<target> naming scheme. +** tables or views named using the data_<target> naming scheme. ** ** Instead of the plain data_<target> naming scheme, RBU database tables ** may also be named data<integer>_<target>, where <integer> is any sequence @@ -205475,7 +210557,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column -** named "rbu_rowid". This column is mapped to the tables implicit primary +** named "rbu_rowid". This column is mapped to the table's implicit primary ** key column - "rowid". Virtual tables for which the "rowid" column does ** not function like a primary key value cannot be updated using RBU. For ** example, if the target db contains either of the following: @@ -205909,6 +210991,34 @@ SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); /* +** As part of applying an RBU update or performing an RBU vacuum operation, +** the system must at one point move the *-oal file to the equivalent *-wal +** path. Normally, it does this by invoking POSIX function rename(2) directly. +** Except on WINCE platforms, where it uses win32 API MoveFileW(). This +** function may be used to register a callback that the RBU module will invoke +** instead of one of these APIs. +** +** If a callback is registered with an RBU handle, it invokes it instead +** of rename(2) when it needs to move a file within the file-system. The +** first argument passed to the xRename() callback is a copy of the second +** argument (pArg) passed to this function. The second is the full path +** to the file to move and the third the full path to which it should be +** moved. The callback function should return SQLITE_OK to indicate +** success. If an error occurs, it should return an SQLite error code. +** In this case the RBU operation will be abandoned and the error returned +** to the RBU user. +** +** Passing a NULL pointer in place of the xRename argument to this function +** restores the default behaviour. +*/ +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +); + + +/* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, ** then the new RBU VFS uses the default system VFS to access the file-system. @@ -206275,6 +211385,8 @@ struct sqlite3rbu { int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; + void *pRenameArg; + int (*xRename)(void*, const char*, const char*); /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -208663,7 +213775,7 @@ static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->zState==0 ){ const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); } } @@ -208911,11 +214023,11 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ ** no-ops. These locks will not be released until the connection ** is closed. ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** * Attempting to xSync() the database file causes an SQLITE_NOTICE ** error. ** ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[] ** array populated with a set of (frame -> page) mappings. Because the ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy ** data from the wal file into the database file according to the @@ -208925,7 +214037,7 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ int rc2; p->eStage = RBU_STAGE_CAPTURE; rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + if( rc2!=SQLITE_NOTICE ) p->rc = rc2; } if( p->rc==SQLITE_OK && p->nFrame>0 ){ @@ -208971,7 +214083,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->mLock!=mReq ){ pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } pRbu->pgsz = iAmt; @@ -209021,6 +214133,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if @@ -209029,9 +214146,20 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ static int rbuLockDatabase(sqlite3 *db){ int rc = SQLITE_OK; sqlite3_file *fd = 0; - sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); - if( fd->pMethods ){ + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE); + } + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + }else{ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + } + + if( rc==SQLITE_OK && fd->pMethods ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); if( rc==SQLITE_OK ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE); @@ -209123,32 +214251,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ -#if defined(_WIN32_WCE) - { - LPWSTR zWideOal; - LPWSTR zWideWal; - - zWideOal = rbuWinUtf8ToUnicode(zOal); - if( zWideOal ){ - zWideWal = rbuWinUtf8ToUnicode(zWal); - if( zWideWal ){ - if( MoveFileW(zWideOal, zWideWal) ){ - p->rc = SQLITE_OK; - }else{ - p->rc = SQLITE_IOERR; - } - sqlite3_free(zWideWal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - sqlite3_free(zWideOal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - } -#else - p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; -#endif + p->rc = p->xRename(p->pRenameArg, zOal, zWal); } if( p->rc!=SQLITE_OK @@ -209735,7 +214838,8 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ static void rbuDeleteOalFile(sqlite3rbu *p){ char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget); if( zOal ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + sqlite3_vfs *pVfs = 0; + sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 ); pVfs->xDelete(pVfs, zOal, 0); sqlite3_free(zOal); @@ -209887,6 +214991,7 @@ static sqlite3rbu *openRbuHandle( /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); + sqlite3rbu_rename_handler(p, 0, 0); rbuCreateVfs(p); /* Open the target, RBU and state databases */ @@ -210278,6 +215383,54 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ return rc; } +/* +** Default xRename callback for RBU. +*/ +static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; +#if defined(_WIN32_WCE) + { + LPWSTR zWideOld; + LPWSTR zWideNew; + + zWideOld = rbuWinUtf8ToUnicode(zOld); + if( zWideOld ){ + zWideNew = rbuWinUtf8ToUnicode(zNew); + if( zWideNew ){ + if( MoveFileW(zWideOld, zWideNew) ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR; + } + sqlite3_free(zWideNew); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOld); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + } +#else + rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK; +#endif + return rc; +} + +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +){ + if( xRename ){ + pRbu->xRename = xRename; + pRbu->pRenameArg = pArg; + }else{ + pRbu->xRename = xDefaultRename; + pRbu->pRenameArg = 0; + } +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: @@ -210334,7 +215487,7 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ ** database file are recorded. xShmLock() calls to unlock the same ** locks are no-ops (so that once obtained, these locks are never ** relinquished). Finally, calls to xSync() on the target database -** file fail with SQLITE_INTERNAL errors. +** file fail with SQLITE_NOTICE errors. */ static void rbuUnlockShm(rbu_file *p){ @@ -210443,9 +215596,12 @@ static int rbuVfsClose(sqlite3_file *pFile){ sqlite3_free(p->zDel); if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ + const sqlite3_io_methods *pMeth = p->pReal->pMethods; rbuMainlistRemove(p); rbuUnlockShm(p); - p->pReal->pMethods->xShmUnmap(p->pReal, 0); + if( pMeth->iVersion>1 && pMeth->xShmUnmap ){ + pMeth->xShmUnmap(p->pReal, 0); + } } else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ rbuUpdateTempSize(p, 0); @@ -210613,7 +215769,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ rbu_file *p = (rbu_file *)pFile; if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } return SQLITE_OK; } @@ -210904,6 +216060,25 @@ static int rbuVfsOpen( rbuVfsShmUnmap, /* xShmUnmap */ 0, 0 /* xFetch, xUnfetch */ }; + static sqlite3_io_methods rbuvfs_io_methods1 = { + 1, /* iVersion */ + rbuVfsClose, /* xClose */ + rbuVfsRead, /* xRead */ + rbuVfsWrite, /* xWrite */ + rbuVfsTruncate, /* xTruncate */ + rbuVfsSync, /* xSync */ + rbuVfsFileSize, /* xFileSize */ + rbuVfsLock, /* xLock */ + rbuVfsUnlock, /* xUnlock */ + rbuVfsCheckReservedLock, /* xCheckReservedLock */ + rbuVfsFileControl, /* xFileControl */ + rbuVfsSectorSize, /* xSectorSize */ + rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, 0, 0, 0, 0, 0 + }; + + + rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs; rbu_file *pFd = (rbu_file *)pFile; @@ -210958,10 +216133,15 @@ static int rbuVfsOpen( rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ + const sqlite3_io_methods *pMeth = pFd->pReal->pMethods; /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods ** pointer and, if the file is a main database file, link it into the ** mutex protected linked list of all such files. */ - pFile->pMethods = &rbuvfs_io_methods; + if( pMeth->iVersion<2 || pMeth->xShmLock==0 ){ + pFile->pMethods = &rbuvfs_io_methods1; + }else{ + pFile->pMethods = &rbuvfs_io_methods; + } if( flags & SQLITE_OPEN_MAIN_DB ){ rbuMainlistAdd(pFd); } @@ -211394,6 +216574,7 @@ static int statConnect( StatTable *pTab = 0; int rc = SQLITE_OK; int iDb; + (void)pAux; if( argc>=4 ){ Token nm; @@ -211447,6 +216628,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iSchema = -1; int iName = -1; int iAgg = -1; + (void)tab; /* Look for a valid schema=? constraint. If found, change the idxNum to ** 1 and request the value of that constraint be sent to xFilter. And @@ -211972,6 +217154,8 @@ static int statFilter( int iArg = 0; /* Count of argv[] parameters used so far */ int rc = SQLITE_OK; /* Result of this operation */ const char *zName = 0; /* Only provide analysis of this table */ + (void)argc; + (void)idxStr; statResetCsr(pCsr); sqlite3_finalize(pCsr->pStmt); @@ -212055,16 +217239,16 @@ static int statColumn( } break; case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); + sqlite3_result_int64(ctx, pCsr->nCell); break; case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); + sqlite3_result_int64(ctx, pCsr->nPayload); break; case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); + sqlite3_result_int64(ctx, pCsr->nUnused); break; case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); + sqlite3_result_int64(ctx, pCsr->nMxPayload); break; case 8: /* pgoffset */ if( !pCsr->isAgg ){ @@ -212072,7 +217256,7 @@ static int statColumn( } break; case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); + sqlite3_result_int64(ctx, pCsr->szPage); break; case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -212206,8 +217390,13 @@ static int dbpageConnect( ){ DbpageTable *pTab = 0; int rc = SQLITE_OK; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -212244,6 +217433,7 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; int iPlan = 0; + (void)tab; /* If there is a schema= constraint, it must be honored. Report a ** ridiculously large estimated cost if the schema= constraint is @@ -212290,7 +217480,6 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - sqlite3VtabUsesAllSchemas(pIdxInfo); return SQLITE_OK; } @@ -212359,6 +217548,8 @@ static int dbpageFilter( sqlite3 *db = pTab->db; Btree *pBt; + (void)idxStr; + /* Default setting is no rows of result */ pCsr->pgno = 1; pCsr->mxPgno = 0; @@ -212373,7 +217564,7 @@ static int dbpageFilter( pCsr->iDb = 0; } pBt = db->aDb[pCsr->iDb].pBt; - if( pBt==0 ) return SQLITE_OK; + if( NEVER(pBt==0) ) return SQLITE_OK; pCsr->pPager = sqlite3BtreePager(pBt); pCsr->szPage = sqlite3BtreeGetPageSize(pBt); pCsr->mxPgno = sqlite3BtreeLastPage(pBt); @@ -212408,12 +217599,18 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, - SQLITE_TRANSIENT); + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); } - sqlite3PagerUnref(pDbPage); break; } default: { /* schema */ @@ -212422,7 +217619,7 @@ static int dbpageColumn( break; } } - return SQLITE_OK; + return rc; } static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -212448,6 +217645,7 @@ static int dbpageUpdate( Pager *pPager; int szPage; + (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ zErr = "read-only"; goto update_fail; @@ -212457,18 +217655,20 @@ static int dbpageUpdate( goto update_fail; } pgno = sqlite3_value_int(argv[0]); - if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + if( sqlite3_value_type(argv[0])==SQLITE_NULL + || (Pgno)sqlite3_value_int(argv[1])!=pgno + ){ zErr = "cannot insert"; goto update_fail; } zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( iDb<0 ){ + iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; + if( NEVER(iDb<0) ){ zErr = "no such schema"; goto update_fail; } pBt = pTab->db->aDb[iDb].pBt; - if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){ + if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ zErr = "bad page number"; goto update_fail; } @@ -212482,11 +217682,12 @@ static int dbpageUpdate( pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - memcpy(sqlite3PagerGetData(pDbPage), - sqlite3_value_blob(argv[3]), - szPage); + const void *pData = sqlite3_value_blob(argv[3]); + assert( pData!=0 || pTab->db->mallocFailed ); + if( pData + && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK + ){ + memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -212508,7 +217709,7 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ int i; for(i=0; i<db->nDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); + if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); } return SQLITE_OK; } @@ -212579,6 +217780,8 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; typedef struct SessionHook SessionHook; @@ -212600,6 +217803,7 @@ struct sqlite3_session { int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); @@ -212676,6 +217880,7 @@ struct SessionTable { char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ @@ -213068,6 +218273,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -213076,48 +218282,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; i<pTab->nCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; - - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; i<pTab->nCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); + if( rc!=SQLITE_OK ) return rc; + + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - z = (const u8 *)sqlite3_value_blob(pVal); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - assert( pTab->bStat1==0 || i!=1 ); - *pbNullPK = 1; } } } @@ -213400,6 +218611,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -213407,6 +218619,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iCol<pTab->nCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -213551,7 +218768,8 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; @@ -213563,6 +218781,7 @@ static int sessionTableInfo( u8 *pAlloc = 0; char **azCol = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); @@ -213607,10 +218826,15 @@ static int sessionTableInfo( } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nByte += sqlite3_column_bytes(pStmt, 1); nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ @@ -213632,6 +218856,14 @@ static int sessionTableInfo( } i = 0; + if( bRowid ){ + size_t nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); const unsigned char *zName = sqlite3_column_text(pStmt, 1); @@ -213643,7 +218875,6 @@ static int sessionTableInfo( i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and @@ -213660,6 +218891,7 @@ static int sessionTableInfo( if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } @@ -213681,7 +218913,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK, + (pSession->bImplicitPK ? &pTab->bRowid : 0) ); if( pSession->rc==SQLITE_OK ){ int i; @@ -213753,6 +218986,7 @@ static int sessionUpdateMaxSize( ){ i64 nNew = 2; if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; if( op!=SQLITE_DELETE ){ int ii; for(ii=0; ii<pTab->nCol; ii++){ @@ -213769,12 +219003,16 @@ static int sessionUpdateMaxSize( }else{ int ii; u8 *pCsr = pC->aRecord; - for(ii=0; ii<pTab->nCol; ii++){ + if( pTab->bRowid ){ + nNew += 9 + 1; + pCsr += 9; + } + for(ii=pTab->bRowid; ii<pTab->nCol; ii++){ int bChanged = 1; int nOld = 0; int eType; sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); if( p==0 ){ return SQLITE_NOMEM; } @@ -213853,6 +219091,7 @@ static int sessionUpdateMaxSize( */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ @@ -213868,7 +219107,7 @@ static void sessionPreupdateOneChange( /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -213901,14 +219140,16 @@ static void sessionPreupdateOneChange( /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ @@ -213923,7 +219164,7 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; i<pTab->nCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -213938,6 +219179,9 @@ static void sessionPreupdateOneChange( rc = sessionSerializeValue(0, p, &nByte); if( rc!=SQLITE_OK ) goto error_out; } + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } /* Allocate the change object */ pC = (SessionChange *)sessionMalloc64(pSession, nByte); @@ -213954,7 +219198,12 @@ static void sessionPreupdateOneChange( ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; i<pTab->nCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -214053,6 +219302,8 @@ static void xPreUpdate( int nDb = sqlite3Strlen30(zDb); assert( sqlite3_mutex_held(db->mutex) ); + (void)iKey1; + (void)iKey2; for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ SessionTable *pTab; @@ -214067,9 +219318,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -214108,6 +219360,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -214116,19 +219369,20 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ + (void)pCtx; return 0; } @@ -214202,17 +219456,18 @@ static char *sessionExprCompareOther( } static char *sessionSelectFindNew( - int nCol, const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -214226,7 +219481,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(pTab->nCol, zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -214237,8 +219494,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -214248,6 +219507,27 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; ii<pTab->nCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( sqlite3_session *pSession, SessionTable *pTab, @@ -214262,11 +219542,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -214277,12 +219559,15 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; @@ -214321,9 +219606,12 @@ SQLITE_API int sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -214665,9 +219953,10 @@ static void sessionAppendStr( int *pRc ){ int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ memcpy(&p->aBuf[p->nBuf], zStr, nStr); p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; } } @@ -214689,6 +219978,27 @@ static void sessionAppendInteger( sessionAppendStr(p, aBuf, pRc); } +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and @@ -214703,7 +220013,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -214714,6 +220024,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -214849,7 +220160,7 @@ static int sessionAppendUpdate( /* If at least one field has been modified, this is not a no-op. */ if( bChanged ) bNoop = 0; - /* Add a field to the old.* record. This is omitted if this modules is + /* Add a field to the old.* record. This is omitted if this module is ** currently generating a patchset. */ if( bPatchset==0 ){ if( bChanged || abPK[i] ){ @@ -214938,12 +220249,20 @@ static int sessionAppendDelete( ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. ** -** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +** SELECT *, <noop-test> FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where <noop-test> is: +** +** 1 AND (?A OR ?1 IS <column>) AND ... +** +** for each non-pk <column>. */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ @@ -214951,8 +220270,50 @@ static int sessionSelectStmt( ){ int rc = SQLITE_OK; char *zSql = 0; + const char *zSep = ""; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; int nSql = -1; + int i; + + SessionBuffer nooptest = {0, 0, 0}; + SessionBuffer pkfield = {0, 0, 0}; + SessionBuffer pkvar = {0, 0, 0}; + + sessionAppendStr(&nooptest, ", 1", &rc); + + if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ + sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc); + sessionAppendStr(&pkfield, "tbl, idx", &rc); + sessionAppendStr(&pkvar, + "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc + ); + zCols = "tbl, ?2, stat"; + }else{ + for(i=0; i<nCol; i++){ + if( abPK[i] ){ + sessionAppendStr(&pkfield, zSep, &rc); + sessionAppendStr(&pkvar, zSep, &rc); + zSep = ", "; + sessionAppendIdent(&pkfield, azCol[i], &rc); + sessionAppendPrintf(&pkvar, &rc, "?%d", i+1); + }else{ + sessionAppendPrintf(&nooptest, &rc, + " AND (?%d OR ?%d IS %w.%w)", i+1+nCol, i+1, zTab, azCol[i] + ); + } + } + } + if( rc==SQLITE_OK ){ + zSql = sqlite3_mprintf( + "SELECT %s%s FROM %Q.%Q WHERE (%s) IS (%s)", + zCols, (bIgnoreNoop ? (char*)nooptest.aBuf : ""), + zDb, zTab, (char*)pkfield.aBuf, (char*)pkvar.aBuf + ); + if( zSql==0 ) rc = SQLITE_NOMEM; + } + +#if 0 if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ zSql = sqlite3_mprintf( "SELECT tbl, ?2, stat FROM %Q.sqlite_stat1 WHERE tbl IS ?1 AND " @@ -214960,7 +220321,6 @@ static int sessionSelectStmt( ); if( zSql==0 ) rc = SQLITE_NOMEM; }else{ - int i; const char *zSep = ""; SessionBuffer buf = {0, 0, 0}; @@ -214981,11 +220341,15 @@ static int sessionSelectStmt( zSql = (char*)buf.aBuf; nSql = buf.nBuf; } +#endif if( rc==SQLITE_OK ){ rc = sqlite3_prepare_v2(db, zSql, nSql, ppStmt, 0); } sqlite3_free(zSql); + sqlite3_free(nooptest.aBuf); + sqlite3_free(pkfield.aBuf); + sqlite3_free(pkvar.aBuf); return rc; } @@ -215132,10 +220496,18 @@ static int sessionGenerateChangeset( sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ int nRewind = buf.nBuf; /* Initial size of write buffer */ int nNoop; /* Size of buffer after writing tbl header */ + int bRowid = 0; /* Check the table schema is still Ok. */ - rc = sessionTableInfo(0, db, pSession->zDb, zName, &nCol, 0,&azCol,&abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = sessionTableInfo( + 0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( rc==SQLITE_OK && ( + pTab->nCol!=nCol + || pTab->bRowid!=bRowid + || memcmp(abPK, pTab->abPK, nCol) + )){ rc = SQLITE_SCHEMA; } @@ -215145,7 +220517,8 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt( - db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel + ); } nNoop = buf.nBuf; @@ -215228,7 +220601,7 @@ SQLITE_API int sqlite3session_changeset( int rc; if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; - rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset); + rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); assert( rc || pnChangeset==0 || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize ); @@ -215346,6 +220719,19 @@ SQLITE_API int sqlite3session_object_config(sqlite3_session *pSession, int op, v break; } + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + default: rc = SQLITE_MISUSE; } @@ -215881,6 +221267,22 @@ static int sessionChangesetNextOne( if( p->op==SQLITE_INSERT ) p->op = SQLITE_DELETE; else if( p->op==SQLITE_DELETE ) p->op = SQLITE_INSERT; } + + /* If this is an UPDATE that is part of a changeset, then check that + ** there are no fields in the old.* record that are not (a) PK fields, + ** or (b) also present in the new.* record. + ** + ** Such records are technically corrupt, but the rebaser was at one + ** point generating them. Under most circumstances this is benign, but + ** can cause spurious SQLITE_RANGE errors when applying the changeset. */ + if( p->bPatchset==0 && p->op==SQLITE_UPDATE){ + for(i=0; i<p->nCol; i++){ + if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){ + sqlite3ValueFree(p->apValue[i]); + p->apValue[i] = 0; + } + } + } } return SQLITE_ROW; @@ -216318,6 +221720,8 @@ struct SessionApplyCtx { SessionBuffer rebase; /* Rebase information (if any) here */ u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; /* Number of prepared UPDATE statements to cache. */ @@ -216568,8 +221972,10 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + /* TODO */ + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -216727,22 +222133,34 @@ static int sessionBindRow( ** UPDATE, bind values from the old.* record. */ static int sessionSeekToRow( - sqlite3 *db, /* Database handle */ sqlite3_changeset_iter *pIter, /* Changeset iterator */ - u8 *abPK, /* Primary key flags array */ - sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ + SessionApplyCtx *p ){ + sqlite3_stmt *pSelect = p->pSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && ii<nCol; ii++){ + if( p->abPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -216857,16 +222275,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -216974,7 +222398,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -217031,7 +222455,7 @@ static int sessionApplyOneOp( /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); @@ -217208,6 +222632,7 @@ static int sessionChangesetApply( memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); @@ -217245,6 +222670,7 @@ static int sessionChangesetApply( sApply.bStat1 = 0; sApply.bDeferConstraints = 1; sApply.bRebaseStarted = 0; + sApply.bRowid = 0; memset(&sApply.constraints, 0, sizeof(SessionBuffer)); /* If an xFilter() callback was specified, invoke it now. If the @@ -217264,8 +222690,8 @@ static int sessionChangesetApply( int i; sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo(0, - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; i<sApply.nCol; i++){ @@ -218077,7 +223503,7 @@ static void sessionAppendPartialUpdate( if( !pIter->abPK[i] && a1[0] ) bData = 1; memcpy(pOut, a1, n1); pOut += n1; - }else if( a2[0]!=0xFF ){ + }else if( a2[0]!=0xFF && a1[0] ){ bData = 1; memcpy(pOut, a2, n2); pOut += n2; @@ -219147,6 +224573,7 @@ struct Fts5Config { int ePattern; /* FTS_PATTERN_XXX constant */ /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ @@ -219155,6 +224582,7 @@ struct Fts5Config { int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -219164,8 +224592,11 @@ struct Fts5Config { #endif }; -/* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 4 +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -219234,7 +224665,7 @@ static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) @@ -219331,6 +224762,7 @@ struct Fts5IndexIter { ** above. */ #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 /* ** Create/destroy an Fts5Index object. @@ -219485,7 +224917,7 @@ static int sqlite3Fts5GetVarintLen(u32 iVal); static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); -#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b)) #define fts5GetVarint sqlite3Fts5GetVarint #define fts5FastGetVarint32(a, iOff, nVal) { \ @@ -221464,7 +226896,7 @@ static int fts5HighlightCb( if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } @@ -221476,7 +226908,7 @@ static int fts5HighlightCb( } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){ + if( p->iRangeEnd>=0 && p->iter.iStart<p->iRangeStart ){ fts5HighlightAppend(&rc, p, p->zOpen, -1); } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); @@ -221487,7 +226919,7 @@ static int fts5HighlightCb( } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){ @@ -221522,6 +226954,7 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); if( ctx.zIn ){ @@ -221707,6 +227140,7 @@ static void fts5SnippetFunction( iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); @@ -222975,6 +228409,7 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } + assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK ); for(i=3; rc==SQLITE_OK && i<nArg; i++){ const char *zOrig = azArg[i]; const char *z; @@ -223328,6 +228763,18 @@ static int sqlite3Fts5ConfigSetValue( rc = SQLITE_OK; *pbBadkey = 1; } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -223372,15 +228819,20 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = sqlite3_finalize(p); } - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION + *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); } + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ @@ -223408,6 +228860,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* #include "fts5Int.h" */ /* #include "fts5parse.h" */ +#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH +# define SQLITE_FTS5_MAX_EXPR_DEPTH 256 +#endif + /* ** All token types in the generated fts5parse.h file are greater than 0. */ @@ -223448,11 +228904,17 @@ struct Fts5Expr { ** FTS5_NOT (nChild, apChild valid) ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) +** +** iHeight: +** Distance from this node to furthest leaf. This is always 0 for nodes +** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one +** greater than the largest child value. */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ + int iHeight; /* Distance to tree leaf nodes */ /* Next method for this node. */ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); @@ -223522,6 +228984,31 @@ struct Fts5Parse { int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ }; +/* +** Check that the Fts5ExprNode.iHeight variables are set correctly in +** the expression tree passed as the only argument. +*/ +#ifndef NDEBUG +static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){ + if( rc==SQLITE_OK ){ + if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){ + assert( p->iHeight==0 ); + }else{ + int ii; + int iMaxChild = 0; + for(ii=0; ii<p->nChild; ii++){ + Fts5ExprNode *pChild = p->apChild[ii]; + iMaxChild = MAX(iMaxChild, pChild->iHeight); + assert_expr_depth_ok(SQLITE_OK, pChild); + } + assert( p->iHeight==iMaxChild+1 ); + } + } +} +#else +# define assert_expr_depth_ok(rc, p) +#endif + static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); @@ -223636,6 +229123,8 @@ static int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert_expr_depth_ok(sParse.rc, sParse.pExpr); + /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ @@ -223681,6 +229170,19 @@ static int sqlite3Fts5ExprNew( } /* +** Assuming that buffer z is at least nByte bytes in size and contains a +** valid utf-8 string, return the number of characters in the string. +*/ +static int fts5ExprCountChar(const char *z, int nByte){ + int nRet = 0; + int ii; + for(ii=0; ii<nByte; ii++){ + if( (z[ii] & 0xC0)!=0x80 ) nRet++; + } + return nRet; +} + +/* ** This function is only called when using the special 'trigram' tokenizer. ** Argument zText contains the text of a LIKE or GLOB pattern matched ** against column iCol. This function creates and compiles an FTS5 MATCH @@ -223717,7 +229219,8 @@ static int sqlite3Fts5ExprPattern( if( i==nText || zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2] ){ - if( i-iFirst>=3 ){ + + if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){ int jj; zExpr[iOut++] = '"'; for(jj=iFirst; jj<i; jj++){ @@ -223784,7 +229287,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ Fts5Parse sParse; memset(&sParse, 0, sizeof(sParse)); - if( *pp1 ){ + if( *pp1 && p2 ){ Fts5Expr *p1 = *pp1; int nPhrase = p1->nPhrase + p2->nPhrase; @@ -223809,7 +229312,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ } sqlite3_free(p2->apExprPhrase); sqlite3_free(p2); - }else{ + }else if( p2 ){ *pp1 = p2; } @@ -225583,6 +231086,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); @@ -225591,6 +231095,9 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ }else{ p->apChild[p->nChild++] = pSub; } + for( ; ii<p->nChild; ii++){ + p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1); + } } /* @@ -225621,6 +231128,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( if( pRet ){ pRet->eType = FTS5_AND; pRet->nChild = nTerm; + pRet->iHeight = 1; fts5ExprAssignXNext(pRet); pParse->nPhrase--; for(ii=0; ii<nTerm; ii++){ @@ -225726,6 +231234,14 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); + if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ + sqlite3Fts5ParseError(pParse, + "fts5 expression tree is too large (maximum depth %d)", + SQLITE_FTS5_MAX_EXPR_DEPTH + ); + sqlite3_free(pRet); + pRet = 0; + } } } } @@ -227078,6 +232594,8 @@ static void sqlite3Fts5HashScanEntry( # error "FTS5_MAX_PREFIX_INDEXES is too large" #endif +#define FTS5_MAX_LEVEL 64 + /* ** Details: ** @@ -227324,6 +232842,8 @@ struct Fts5Index { sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -227416,9 +232936,6 @@ struct Fts5CResult { ** iLeafOffset: ** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -227977,6 +233494,7 @@ static int fts5StructureDecode( rc = FTS5_CORRUPT; break; } + assert( pSeg!=0 ); i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); @@ -228007,6 +233525,7 @@ static int fts5StructureDecode( */ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; @@ -228465,42 +233984,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } - - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; + + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -228696,7 +234198,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -228795,10 +234297,12 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); @@ -228992,7 +234496,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -229185,7 +234689,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); @@ -229746,7 +235250,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -229761,21 +235266,23 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } @@ -230490,7 +235997,7 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -230511,7 +236018,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -231111,7 +236618,9 @@ static void fts5WriteAppendRowid( fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); - fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; @@ -231264,7 +236773,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); if( p->rc==SQLITE_OK ){ /* Set the szLeaf field */ fts5PutU16(&buf.p[2], (u16)buf.n); @@ -231542,16 +237051,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -231586,6 +237095,413 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ } /* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1<pLeaf->nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + u64 iNextDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iKeyOff = 0; + int iPrevKeyOff = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** iDelta: + */ + { + int iSOP; + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOP<pSeg->iLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOP<pSeg->iLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Set bLastInDoclist to true if the entry being removed is the last + ** in its doclist. */ + for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){ + u32 iVal = 0; + iIdx += fts5GetVarint32(&aIdx[iIdx], iVal); + iKeyOff += iVal; + if( iKeyOff==iNextOff ){ + bLastInDoclist = 1; + } + } + } + + if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){ + fts5PutU16(&aPg[0], 0); + } + + if( bLastInDoclist==0 ){ + if( iNextOff!=iPgIdx ){ + iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); + } + }else if( + iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){ + u32 iVal = 0; + iIdx += fts5GetVarint32(&aIdx[iIdx], iVal); + if( (iKeyOff+iVal)>(u32)iStart ) break; + iKeyOff += iVal; + } + + iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; + int nShift = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){ + u32 iVal = 0; + iIdx += fts5GetVarint32(&aIdx[iIdx], iVal); + iKeyOff += iVal; + if( iKeyOff!=iDelKeyOff ){ + if( iKeyOff>iOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + int nTerm = (int)strlen(zTerm); + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThis<iRowid ){ + fts5MultiIterNextFrom(p, pIter, iRowid); + } + + if( p->rc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + +/* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. ** @@ -231607,6 +237523,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -231629,40 +237546,77 @@ static void fts5FlushOneHash(Fts5Index *p){ } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - /* Write the term for this entry to disk. */ + /* Get the term and doclist for this entry. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ + int bTermWritten = !bSecureDelete; i64 iRowid = 0; - u64 iDelta = 0; + i64 iPrev = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ + u64 iDelta = 0; iOff += fts5GetVarint(&pDoclist[iOff], &iDelta); iRowid += iDelta; + /* If in secure delete mode, and if this entry in the poslist is + ** in fact a delete, then edit the existing segments directly + ** using fts5FlushSecureDelete(). */ + if( bSecureDelete ){ + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ + fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); + iOff++; + if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ + iOff++; + nDoclist = 0; + }else{ + continue; + } + } + }else if( (pDoclist[iOff] & 0x01) ){ + fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); + if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); - if( p->rc!=SQLITE_OK ) break; }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); } + if( p->rc!=SQLITE_OK ) break; assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; if( eDetail==FTS5_DETAIL_NONE ){ if( iOff<nDoclist && pDoclist[iOff]==0 ){ @@ -231721,20 +237675,23 @@ static void fts5FlushOneHash(Fts5Index *p){ sqlite3Fts5HashClear(pHash); fts5WriteFinish(p, &writer, &pgnoLast); - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); @@ -231790,10 +237747,10 @@ static Fts5Structure *fts5IndexOptimizeStruct( if( pNew ){ Fts5StructureLevel *pLvl; nByte = nSeg * sizeof(Fts5StructureSegment); - pNew->nLevel = pStruct->nLevel+1; + pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; - pLvl = &pNew->aLevel[pStruct->nLevel]; + pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; @@ -231875,7 +237832,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ static void fts5AppendRowid( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ @@ -231885,7 +237842,7 @@ static void fts5AppendRowid( static void fts5AppendPoslist( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ @@ -231960,10 +237917,10 @@ static void fts5MergeAppendDocid( } #endif -#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ - assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ - fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ - (iLastRowid) = (iRowid); \ +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ } /* @@ -232095,7 +238052,7 @@ static void fts5MergePrefixLists( /* Initialize a doclist-iterator for each input buffer. Arrange them in ** a linked-list starting at pHead in ascending order of rowid. Avoid ** linking any iterators already at EOF into the linked list at all. */ - assert( nBuf+1<=sizeof(aMerger)/sizeof(aMerger[0]) ); + assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) ); memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); pHead = &aMerger[nBuf]; fts5DoclistIterInit(p1, &pHead->iter); @@ -232234,7 +238191,7 @@ static void fts5SetupPrefixIter( int nMerge = 1; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -232273,7 +238230,7 @@ static void fts5SetupPrefixIter( Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; p1->xSetOutputs(p1, pSeg); if( p1->base.nData ){ - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } } @@ -232321,7 +238278,7 @@ static void fts5SetupPrefixIter( iLastRowid = 0; } - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } @@ -232475,6 +238432,7 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -233105,6 +239063,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -233140,7 +239099,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -233204,9 +239175,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){ + p->rc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } @@ -233300,6 +239274,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ @@ -234104,7 +240079,7 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ break; case FTS5_SYNC: - assert( p->ts.eState==1 ); + assert( p->ts.eState==1 || p->ts.eState==2 ); p->ts.eState = 2; break; @@ -234119,21 +240094,21 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ break; case FTS5_SAVEPOINT: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); assert( iSavepoint>=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint; break; case FTS5_RELEASE: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); assert( iSavepoint<=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint-1; break; case FTS5_ROLLBACKTO: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=-1 ); /* The following assert() can fail if another vtab strikes an error ** within an xSavepoint() call then SQLite calls xRollbackTo() - without @@ -235467,9 +241442,11 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; + /* A transaction must be open when this is called. */ - assert( pTab->ts.eState==1 ); + assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); assert( pVtab->zErrMsg==0 ); assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); @@ -235477,6 +241454,11 @@ static int fts5UpdateMethod( || sqlite3_value_type(apVal[0])==SQLITE_NULL ); assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ @@ -235529,6 +241511,7 @@ static int fts5UpdateMethod( else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + bUpdateOrDelete = 1; } /* INSERT or UPDATE */ @@ -235544,6 +241527,7 @@ static int fts5UpdateMethod( if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); } @@ -235572,10 +241556,24 @@ static int fts5UpdateMethod( rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } + bUpdateOrDelete = 1; } } } + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -236435,6 +242433,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); + pTab->p.pConfig->pgsz = 0; return sqlite3Fts5StorageRollback(pTab->pStorage); } @@ -236637,7 +242636,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0", -1, SQLITE_TRANSIENT); } /* @@ -236710,7 +242709,9 @@ static int fts5Init(sqlite3 *db){ } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( - db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 + db, "fts5_source_id", 0, + SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS, + p, fts5SourceIdFunc, 0, 0 ); } } @@ -241375,6 +247376,10 @@ static int stmtConnect( #define STMT_COLUMN_MEM 10 /* SQLITE_STMTSTATUS_MEMUSED */ + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; rc = sqlite3_declare_vtab(db, "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep," "reprep,run,mem)"); @@ -241494,6 +247499,10 @@ static int stmtFilter( sqlite3_int64 iRowid = 1; StmtRow **ppRow = 0; + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; stmtCsrReset(pCur); ppRow = &pCur->pRow; for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ @@ -241549,6 +247558,7 @@ static int stmtBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ + (void)tab; pIdxInfo->estimatedCost = (double)500; pIdxInfo->estimatedRows = 500; return SQLITE_OK; |