diff --git a/code/game/sqlite3.c b/code/game/sqlite3.c index 55039b9..51e54a1 100644 --- a/code/game/sqlite3.c +++ b/code/game/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.14.1. By combining all the individual C code files into this +** version 3.7.16.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -306,6 +306,10 @@ # define _GNU_SOURCE #endif +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** Include standard header files as necessary */ @@ -440,7 +444,8 @@ ** ** See also ticket #2741. */ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) \ + && !defined(__APPLE__) && SQLITE_THREADSAFE # define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ #endif @@ -673,9 +678,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.14.1" -#define SQLITE_VERSION_NUMBER 3007014 -#define SQLITE_SOURCE_ID "2012-10-04 19:37:12 091570e46d04e84b67228e0bdbcd6e1fb60c6bdb" +#define SQLITE_VERSION "3.7.16.2" +#define SQLITE_VERSION_NUMBER 3007016 +#define SQLITE_SOURCE_ID "2013-04-12 11:52:43 cbea02d93865ce0e06789db95fd9168ebac970c7" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -854,7 +859,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated ** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has +** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or ** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], @@ -1040,14 +1045,26 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -1421,6 +1438,26 @@ struct sqlite3_io_methods { ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] ** file control occurs at the beginning of pragma statement analysis and so ** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^This file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke this file-control to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1437,6 +1474,8 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 /* ** CAPI3REF: Mutex Handle @@ -2133,11 +2172,39 @@ struct sqlite3_mem_methods { ** disabled. The default value may be changed by compiling with the ** [SQLITE_USE_URI] symbol defined. ** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** malfunction when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** ** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. ** +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -2158,6 +2225,8 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_URI 17 /* int */ #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3166,7 +3235,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both @@ -3182,7 +3251,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** ** @@ -3318,6 +3387,11 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -3336,6 +3410,7 @@ SQLITE_API int sqlite3_errcode(sqlite3 *db); SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -4494,7 +4569,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4574,14 +4650,17 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -5298,6 +5377,9 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ SQLITE_API int sqlite3_enable_shared_cache(int); @@ -6872,7 +6954,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** ** -**
    createFlag Behaviour when page is not already in cache +**
    createFlag Behavior when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -7677,7 +7759,7 @@ struct sqlite3_rtree_geometry { ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ @@ -8177,6 +8259,11 @@ struct BusyHandler { */ #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + /* ** The following value as a destructor means to use sqlite3DbFree(). ** The sqlite3DbFree() routine requires two parameters instead of the @@ -8263,6 +8350,7 @@ typedef struct Parse Parse; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; @@ -8359,6 +8447,9 @@ SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree*); +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); +#endif SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); @@ -8402,6 +8493,8 @@ SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); + /* ** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta ** should be one of the following values. The integer values are assigned @@ -8869,7 +8962,7 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT3 0x0040 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x01, 0x01, 0x04, 0x04, 0x10, 0x00, 0x02,\ -/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x00, 0x24, 0x24,\ +/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x24,\ /* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\ /* 24 */ 0x00, 0x01, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00,\ /* 32 */ 0x02, 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00,\ @@ -8916,7 +9009,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); @@ -9106,11 +9199,14 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); -SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); -SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); #endif @@ -9128,6 +9224,7 @@ SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); @@ -9825,6 +9922,7 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -9929,48 +10027,60 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ -#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000008 /* Show short columns names */ +#define SQLITE_CountRows 0x00000010 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00000020 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ - /* 0x00020000 Unused */ -#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ -#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ +#define SQLITE_SqlTrace 0x00000040 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000080 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000100 /* OK to update SQLITE_MASTER */ +#define SQLITE_VdbeAddopTrace 0x00000200 /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_IgnoreChecks 0x00000400 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0000800 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00001000 /* Create new databases in format 1 */ +#define SQLITE_FullFSync 0x00002000 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00004000 /* Use full fsync for checkpoint */ +#define SQLITE_RecoveryMode 0x00008000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00010000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00020000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00040000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00080000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00100000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00200000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00400000 /* True to enable triggers */ /* -** Bits of the sqlite3.flags field that are used by the -** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. -** These must be the low-order bits of the flags field. +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. */ -#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ -#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ -#define SQLITE_IndexCover 0x10 /* Disable index covering table */ -#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ -#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ -#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ -#define SQLITE_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif /* ** Possible values for the sqlite.magic field. @@ -10121,32 +10231,22 @@ struct Column { char *zDflt; /* Original text of the default value */ char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ - u8 notNull; /* True if there is a NOT NULL constraint */ - u8 isPrimKey; /* True if this column is part of the PRIMARY KEY */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - u8 isHidden; /* True if this column is 'hidden' */ -#endif + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; +/* Allowed values for Column.colFlags: +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ + /* ** A "Collating Sequence" is defined by an instance of the following ** structure. Conceptually, a collating sequence consists of a name and ** a comparison routine that defines the order of that sequence. ** -** There may two separate implementations of the collation function, one -** that processes text in UTF-8 encoding (CollSeq.xCmp) and another that -** processes text encoded in UTF-16 (CollSeq.xCmp16), using the machine -** native byte order. When a collation sequence is invoked, SQLite selects -** the version that will require the least expensive encoding -** translations, if any. -** -** The CollSeq.pUser member variable is an extra parameter that passed in -** as the first argument to the UTF-8 comparison function, xCmp. -** CollSeq.pUser16 is the equivalent for the UTF-16 comparison function, -** xCmp16. -** -** If both CollSeq.xCmp and CollSeq.xCmp16 are NULL, it means that the +** If CollSeq.xCmp is NULL, it means that the ** collating sequence is undefined. Indices built on an undefined ** collating sequence may not be read or written. */ @@ -10284,28 +10384,28 @@ struct VTable { */ struct Table { char *zName; /* Name of the table or view */ - int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ - int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ - int tnum; /* Root BTree node for this table (see note above) */ - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ - u16 nRef; /* Number of pointers to this Table */ - u8 tabFlags; /* Mask of TF_* values */ - u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif + tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + int tnum; /* Root BTree node for this table (see note above) */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + i16 nCol; /* Number of columns in this table */ + u16 nRef; /* Number of pointers to this Table */ + u8 tabFlags; /* Mask of TF_* values */ + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - VTable *pVTable; /* List of VTable objects. */ int nModuleArg; /* Number of arguments to the module */ char **azModuleArg; /* Text of all module args. [0] is module name */ + VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ Schema *pSchema; /* Schema that contains this table */ @@ -10329,7 +10429,7 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) -# define IsHiddenColumn(X) ((X)->isHidden) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 # define IsHiddenColumn(X) 0 @@ -10480,20 +10580,20 @@ struct UnpackedRecord { ** element. */ struct Index { - char *zName; /* Name of this index */ - int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ - Table *pTable; /* The SQL table being indexed */ - char *zColAff; /* String defining the affinity of each column */ - Index *pNext; /* The next index associated with the same table */ - Schema *pSchema; /* Schema containing this index */ - u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ - int nColumn; /* Number of columns in the table used by this index */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ + char *zName; /* Name of this index */ + int *aiColumn; /* Which columns are used by this index. 1st is 0 */ + tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + int tnum; /* DB Page containing root of this index */ + u16 nColumn; /* Number of columns in table used by this index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -10674,13 +10774,15 @@ struct Expr { ExprList *pList; /* Function arguments or in " IN ( IN ( ** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephermeral table instead of an +** existing table. +** ** If the prNotFound parameter is 0, then the b-tree will be used to iterate ** through the set members, skipping any duplicates. In this case an ** epheremal table must be used unless the selected is guaranteed @@ -74764,8 +75541,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** comparison is the same as the affinity of the column. If ** it is not, it is not possible to use any index. */ - char aff = comparisonAffinity(pX); - int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) @@ -74781,7 +75557,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, pKey,P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ @@ -74893,6 +75670,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( case TK_IN: { char affinity; /* Affinity of the LHS of the IN */ KeyInfo keyInfo; /* Keyinfo for the generated table */ + static u8 sortOrder = 0; /* Fake aSortOrder for keyInfo */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ @@ -74920,6 +75698,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; + keyInfo.aSortOrder = &sortOrder; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -74960,6 +75739,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( affinity = SQLITE_AFF_NONE; } keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + keyInfo.aSortOrder = &sortOrder; /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); @@ -75286,7 +76066,7 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** for testing only - to verify that SQLite always gets the same answer ** with and without the column cache. */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; /* First replace any existing entry. ** @@ -75483,8 +76263,8 @@ SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, in SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ int i; struct yColCache *p; - if( NEVER(iFrom==iTo) ) return; - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); for(i=0, p=pParse->aColCache; iiReg; if( x>=iFrom && xpVdbe, OP_Copy, iFrom+i, iTo+i); - } -} - #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* ** Return true if any register in the range iFrom..iTo (inclusive) @@ -75976,6 +76744,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ReleaseTempReg(pParse, r4); break; } + case TK_COLLATE: case TK_UPLUS: { inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; @@ -76142,7 +76911,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0); } break; @@ -76345,6 +77115,12 @@ SQLITE_PRIVATE void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_COLLATE: { + sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + break; + } + case TK_AGG_FUNCTION: case TK_CONST_FUNC: case TK_FUNCTION: { @@ -76482,6 +77258,12 @@ SQLITE_PRIVATE void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); sqlite3ExplainPop(pOut); + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } if( inExpr-1 ){ sqlite3ExplainNL(pOut); } @@ -76563,6 +77345,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ case TK_REGISTER: { return WRC_Prune; } + case TK_COLLATE: { + return WRC_Continue; + } case TK_FUNCTION: case TK_AGG_FUNCTION: case TK_CONST_FUNC: { @@ -76584,9 +77369,11 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ } if( isAppropriateForFactoring(pExpr) ){ int r1 = ++pParse->nMem; - int r2; - r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); + int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + /* If r2!=r1, it means that register r1 is never used. That is harmless + ** but suboptimal, so we want to know about the situation to fix it. + ** Hence the following assert: */ + assert( r2==r1 ); pExpr->op2 = pExpr->op; pExpr->op = TK_REGISTER; pExpr->iTable = r2; @@ -76614,7 +77401,7 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; + if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; w.xExprCallback = evalConstExpr; w.xSelectCallback = 0; w.pParse = pParse; @@ -76729,7 +77516,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ @@ -76849,7 +77636,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: @@ -77003,7 +77790,15 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ return 2; } if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; - if( pA->op!=pB->op ) return 2; + if( pA->op!=pB->op ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft)<2 ){ + return 1; + } + return 2; + } if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; @@ -77015,11 +77810,9 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ }else if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 2; + return pA->op==TK_COLLATE ? 1 : 2; } } - if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; - if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; return 0; } @@ -77260,8 +78053,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ExprSetIrreducible(pExpr); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; } - return WRC_Prune; } } return WRC_Continue; @@ -77273,9 +78068,10 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ } /* -** Analyze the given expression looking for aggregate functions and -** for variables that need to be added to the pParse->aAgg[] array. -** Make additional entries to the pParse->aAgg[] array as necessary. +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. ** ** This routine should only be called after the expression has been ** analyzed by sqlite3ResolveExprNames(). @@ -77788,7 +78584,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; @@ -78038,7 +78834,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** If there is a NOT NULL constraint, then the default value for the ** column must not be NULL. */ - if( pCol->isPrimKey ){ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); return; } @@ -78131,7 +78927,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -78675,7 +79471,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -79085,7 +79881,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; - if( memcmp(z, "unordered", 10)==0 ){ + if( strcmp(z, "unordered")==0 ){ pIndex->bUnordered = 1; break; } @@ -79437,7 +80233,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -79454,7 +80250,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); @@ -79762,6 +80558,7 @@ SQLITE_PRIVATE int sqlite3FixInit( assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zName; + pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; return 1; @@ -79792,14 +80589,15 @@ SQLITE_PRIVATE int sqlite3FixSrcList( if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ + if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } + sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; @@ -80265,6 +81063,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + assert( pParse->pToplevel==0 ); db = pParse->db; if( db->mallocFailed ) return; if( pParse->nested ) return; @@ -80457,6 +81256,31 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( return p; } +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTableItem( + Parse *pParse, + int isView, + struct SrcList_item *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, isView, p->zName, zDb); +} + /* ** Locate the in-memory structure that describes ** a particular index given the name of that index @@ -81307,7 +82131,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; }else{ for(i=0; inExpr; i++){ for(iCol=0; iColnCol; iCol++){ @@ -81316,7 +82140,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( } } if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; } } if( pList->nExpr>1 ) iCol = -1; @@ -81433,10 +82257,7 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } return pColl; @@ -82135,6 +82956,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ return; }else{ int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); destroyRootPage(pParse, iLargest, iDb); iDestroyed = iLargest; } @@ -82214,7 +83036,7 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in /* Drop all SQLITE_MASTER table and index entries that refer to the ** table. The program name loops through the master table and deletes ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be + ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ @@ -82252,8 +83074,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ @@ -82507,9 +83328,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ -#ifdef SQLITE_OMIT_MERGE_SORT - int regIdxKey; /* Registers containing the index key */ -#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -82537,13 +83355,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ (char *)pKey, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); -#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); -#else - iSorter = iTab; -#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -82551,7 +83365,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); -#ifndef SQLITE_OMIT_MERGE_SORT sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); @@ -82562,8 +83375,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + OE_Abort, "indexed columns are not unique", P4_STATIC ); }else{ addr2 = sqlite3VdbeCurrentAddr(v); @@ -82571,30 +83384,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#else - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - addr2 = addr1 + 1; - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); - } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#endif sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); @@ -82693,9 +83482,9 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); @@ -82709,7 +83498,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -82806,10 +83595,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = pExpr->pColl; - /* Either pColl!=0 or there was an OOM failure. But if an OOM - ** failure we have quit before reaching this point. */ - if( ALWAYS(pColl) ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr); + if( pColl ){ nExtra += (1 + sqlite3Strlen30(pColl->zName)); } } @@ -82872,6 +83659,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( const char *zColName = pListItem->zName; Column *pTabCol; int requestedSortOrder; + CollSeq *pColl; /* Collating sequence */ char *zColl; /* Collation sequence name */ for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ @@ -82884,14 +83672,11 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + if( pListItem->pExpr + && (pColl = sqlite3ExprCollSeq(pParse, pListItem->pExpr))!=0 + ){ int nColl; - zColl = pListItem->pExpr->pColl->zName; + zColl = pColl->zName; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -82958,7 +83743,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, @@ -83705,6 +84490,15 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); +#ifndef SQLITE_OMIT_TRIGGER + if( pToplevel!=pParse ){ + /* This branch is taken if a trigger is currently being coded. In this + ** case, set cookieGoto to a non-zero value to show that this function + ** has been called. This is used by the sqlite3ExprCodeConstants() + ** function. */ + pParse->cookieGoto = -1; + } +#endif if( pToplevel->cookieGoto==0 ){ Vdbe *v = sqlite3GetVdbe(pToplevel); if( v==0 ) return; /* This only happens if there was a prior error */ @@ -83802,12 +84596,19 @@ SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ ** error. The onError parameter determines which (if any) of the statement ** and/or current transaction is rolled back. */ -SQLITE_PRIVATE void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ +SQLITE_PRIVATE void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + int p4type /* P4_STATIC or P4_TRANSIENT */ +){ Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); } /* @@ -84054,17 +84855,18 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ ** ** The return value is either the collation sequence to be used in database ** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** sequence can be found. If no collation is found, leave an error message. ** ** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ + Parse *pParse, /* Parsing context */ u8 enc, /* The desired encoding for the collating sequence */ CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ const char *zName /* Collating sequence name */ ){ CollSeq *p; + sqlite3 *db = pParse->db; p = pColl; if( !p ){ @@ -84081,6 +84883,9 @@ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( p = 0; } assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } return p; } @@ -84099,10 +84904,8 @@ SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ if( pColl ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; return SQLITE_ERROR; } assert( p==pColl ); @@ -84489,7 +85292,7 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ struct SrcList_item *pItem = pSrc->a; Table *pTab; assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ @@ -84550,29 +85353,28 @@ SQLITE_PRIVATE void sqlite3MaterializeView( int iCur /* Cursor number for ephemerial table */ ){ SelectDest dest; - Select *pDup; + Select *pSel; + SrcList *pFrom; sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); - }else{ - sqlite3SelectDelete(db, pDup); - } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); } + + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + if( pSel ) pSel->selFlags |= SF_Materialize; + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); } #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ @@ -85095,7 +85897,9 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( } if( doMakeRec ){ const char *zAff; - if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + if( pTab->pSelect + || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) + ){ zAff = 0; }else{ zAff = sqlite3IndexAffinityStr(v, pIdx); @@ -85277,6 +86081,56 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + sqlite3_result_int(context, N); +} + /* ** Implementation of the substr() function. ** @@ -85476,33 +86330,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } - -#if 0 /* This function is never used. */ /* -** The COALESCE() and IFNULL() functions used to be implemented as shown -** here. But now they are implemented as VDBE code so that unused arguments -** do not have to be computed. This legacy implementation is retained as -** comment. +** The COALESCE() and IFNULL() functions are implemented as VDBE code so +** that unused argument values do not have to be computed. However, we +** still need some kind of function implementation for this routines in +** the function table. That function implementation will never be called +** so it doesn't matter what the implementation is. We might as well use +** the "version()" function as a substitute. */ -/* -** Implementation of the IFNULL(), NVL(), and COALESCE() functions. -** All three do the same thing. They return the first non-NULL -** argument. -*/ -static void ifnullFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - for(i=0; inoCase; int prevEscape = 0; /* True if the previous character was 'escape' */ - while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ - if( !prevEscape && c==matchAll ){ - while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + if( c==matchAll && !prevEscape ){ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ - if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } } if( c==0 ){ return 1; }else if( c==esc ){ - c = sqlite3Utf8Read(zPattern, &zPattern); + c = sqlite3Utf8Read(&zPattern); if( c==0 ){ return 0; } @@ -85701,25 +86536,25 @@ static int patternCompare( } return *zString!=0; } - while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ if( noCase ){ GlogUpperToLower(c2); GlogUpperToLower(c); while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); GlogUpperToLower(c2); } }else{ while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); } } if( c2==0 ) return 0; if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( !prevEscape && c==matchOne ){ - if( sqlite3Utf8Read(zString, &zString)==0 ){ + }else if( c==matchOne && !prevEscape ){ + if( sqlite3Utf8Read(&zString)==0 ){ return 0; } }else if( c==matchSet ){ @@ -85727,20 +86562,20 @@ static int patternCompare( assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; - c = sqlite3Utf8Read(zString, &zString); + c = sqlite3Utf8Read(&zString); if( c==0 ) return 0; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c2=='^' ){ invert = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==']' ){ if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } while( c2 && c2!=']' ){ if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); if( c>=prior_c && c<=c2 ) seen = 1; prior_c = 0; }else{ @@ -85749,7 +86584,7 @@ static int patternCompare( } prior_c = c2; } - c2 = sqlite3Utf8Read(zPattern, &zPattern); + c2 = sqlite3Utf8Read(&zPattern); } if( c2==0 || (seen ^ invert)==0 ){ return 0; @@ -85757,7 +86592,7 @@ static int patternCompare( }else if( esc==c && !prevEscape ){ prevEscape = 1; }else{ - c2 = sqlite3Utf8Read(zString, &zString); + c2 = sqlite3Utf8Read(&zString); if( noCase ){ GlogUpperToLower(c); GlogUpperToLower(c2); @@ -85829,7 +86664,7 @@ static void likeFunc( "ESCAPE expression must be a single character", -1); return; } - escape = sqlite3Utf8Read(zEsc, &zEsc); + escape = sqlite3Utf8Read(&zEsc); } if( zA && zB ){ struct compareInfo *pInfo = sqlite3_user_data(context); @@ -86040,6 +86875,62 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); +} + /* ** The hex() function. Interpret the argument as a blob. Return ** a hexadecimal rendering as text. @@ -86664,8 +87555,11 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), @@ -86757,8 +87651,9 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** -------------------------- ** ** Foreign keys in SQLite come in two flavours: deferred and immediate. -** If an immediate foreign key constraint is violated, SQLITE_CONSTRAINT -** is returned and the current statement transaction rolled back. If a +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a ** deferred foreign key constraint is violated, no action is taken ** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. @@ -86822,7 +87717,8 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate -** constraint counter is greater than zero, it returns SQLITE_CONSTRAINT +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY ** and the statement transaction is rolled back. An exception is an INSERT ** statement that inserts a single row only (no triggers). In this case, ** instead of using a counter, an exception is thrown immediately if the @@ -86878,7 +87774,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. ** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. +** search the schema for a unique index on the parent key columns. ** ** If successful, zero is returned. If the parent key is an INTEGER PRIMARY ** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx @@ -86914,7 +87810,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** into pParse. If an OOM error occurs, non-zero is returned and the ** pParse->db->mallocFailed flag is set. */ -static int locateFkeyIndex( +SQLITE_PRIVATE int sqlite3FkLocateIndex( Parse *pParse, /* Parse context to store any error in */ Table *pParent, /* Parent table of FK constraint pFKey */ FKey *pFKey, /* Foreign key to find index for */ @@ -87011,7 +87907,9 @@ static int locateFkeyIndex( if( !pIdx ){ if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); } sqlite3DbFree(pParse->db, aiCol); return 1; @@ -87160,8 +88058,8 @@ static void fkLookupParent( ** incrementing a counter. This is necessary as the VM code is being ** generated for will not open a statement transaction. */ assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ @@ -87247,12 +88145,15 @@ static void fkScanChildren( ** expression to the parent key column defaults. */ if( pIdx ){ Column *pCol; + const char *zColl; iCol = pIdx->aiColumn[i]; pCol = &pTab->aCol[iCol]; if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; - pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pLeft = sqlite3ExprAddCollateString(pParse, pLeft, zColl); }else{ pLeft->iTable = regData; pLeft->affinity = SQLITE_AFF_INTEGER; @@ -87398,8 +88299,8 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa ** any modifications to the schema are made. This is because statement ** transactions are not able to rollback schema changes. */ sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); if( iSkip ){ @@ -87469,7 +88370,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( }else{ pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); if( !isIgnoreErrors || db->mallocFailed ) return; if( pTo==0 ){ @@ -87549,7 +88450,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( continue; } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ if( !isIgnoreErrors || db->mallocFailed ) return; continue; } @@ -87604,7 +88505,7 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ for(i=0; inColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } @@ -87661,7 +88562,8 @@ SQLITE_PRIVATE int sqlite3FkRequired( int iKey; for(iKey=0; iKeynCol; iKey++){ Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) + : (pCol->colFlags & COLFLAG_PRIMKEY)!=0) ){ if( aChange[iKey]>=0 ) return 1; if( iKey==pTab->iPKey && chngRowid ) return 1; } @@ -87729,7 +88631,7 @@ static Trigger *fkActionTrigger( int i; /* Iterator variable */ Expr *pWhen = 0; /* WHEN clause for the trigger */ - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; assert( aiCol || pFKey->nCol==1 ); for(i=0; inCol; i++){ @@ -87982,7 +88884,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( int opcode /* OP_OpenRead or OP_OpenWrite */ ){ Vdbe *v; - if( IsVirtual(pTab) ) return; + assert( !IsVirtual(pTab) ); v = sqlite3GetVdbe(p); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); @@ -88277,6 +89179,97 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** Generate code for a co-routine that will evaluate a subquery one +** row at a time. +** +** The pSelect parameter is the subquery that the co-routine will evaluation. +** Information about the location of co-routine and the registers it will use +** is returned by filling in the pDest object. +** +** Registers are allocated as follows: +** +** pDest->iSDParm The register holding the next entry-point of the +** co-routine. Run the co-routine to its next breakpoint +** by calling "OP_Yield $X" where $X is pDest->iSDParm. +** +** pDest->iSDParm+1 The register holding the "completed" flag for the +** co-routine. This register is 0 if the previous Yield +** generated a new result row, or 1 if the subquery +** has completed. If the Yield is called again +** after this register becomes 1, then the VDBE will +** halt with an SQLITE_INTERNAL error. +** +** pDest->iSdst First result register. +** +** pDest->nSdst Number of result registers. +** +** This routine handles all of the register allocation and fills in the +** pDest structure appropriately. +** +** Here is a schematic of the generated code assuming that X is the +** co-routine entry-point register reg[pDest->iSDParm], that EOF is the +** completed flag reg[pDest->iSDParm+1], and R and S are the range of +** registers that hold the result set, reg[pDest->iSdst] through +** reg[pDest->iSdst+pDest->nSdst-1]: +** +** X <- A +** EOF <- 0 +** goto B +** A: setup for the SELECT +** loop rows in the SELECT +** load results into registers R..S +** yield X +** end loop +** cleanup after the SELECT +** EOF <- 1 +** yield X +** halt-error +** B: +** +** To use this subroutine, the caller generates code as follows: +** +** [ Co-routine generated by this subroutine, shown above ] +** S: yield X +** if EOF goto E +** if skip this row, goto C +** if terminate loop, goto E +** deal with this row +** C: goto S +** E: +*/ +SQLITE_PRIVATE int sqlite3CodeCoroutine(Parse *pParse, Select *pSelect, SelectDest *pDest){ + int regYield; /* Register holding co-routine entry-point */ + int regEof; /* Register holding co-routine completion flag */ + int addrTop; /* Top of the co-routine */ + int j1; /* Jump instruction */ + int rc; /* Result code */ + Vdbe *v; /* VDBE under construction */ + + regYield = ++pParse->nMem; + regEof = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + addrTop = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, addrTop+2, regYield); /* X <- A */ + VdbeComment((v, "Co-routine entry point")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ + VdbeComment((v, "Co-routine completion flag")); + sqlite3SelectDestInit(pDest, SRT_Coroutine, regYield); + j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + rc = sqlite3Select(pParse, pSelect, pDest); + assert( pParse->nErr==0 || rc ); + if( pParse->db->mallocFailed && rc==SQLITE_OK ) rc = SQLITE_NOMEM; + if( rc ) return rc; + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ + sqlite3VdbeAddOp1(v, OP_Yield, regYield); /* yield X */ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); + VdbeComment((v, "End of coroutine")); + sqlite3VdbeJumpHere(v, j1); /* label B: */ + return rc; +} + + + /* Forward declaration */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -88525,51 +89518,12 @@ SQLITE_PRIVATE void sqlite3Insert( ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ - /* Data is coming from a SELECT. Generate code to implement that SELECT - ** as a co-routine. The code is common to both the 3rd and 4th - ** templates: - ** - ** EOF <- 0 - ** X <- A - ** goto B - ** A: setup for the SELECT - ** loop over the tables in the SELECT - ** load value into register R..R+n - ** yield X - ** end loop - ** cleanup after the SELECT - ** EOF <- 1 - ** yield X - ** halt-error - ** - ** On each invocation of the co-routine, it puts a single row of the - ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. - ** (These output registers are allocated by sqlite3Select().) When - ** the SELECT completes, it sets the EOF flag stored in regEof. - */ - int rc, j1; - - regEof = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "SELECT eof flag")); - sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); - addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iSDParm); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - VdbeComment((v, "Jump over SELECT coroutine")); - - /* Resolve the expressions in the SELECT statement and execute it. */ - rc = sqlite3Select(pParse, pSelect, &dest); - assert( pParse->nErr==0 || rc ); - if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ - goto insert_cleanup; - } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of SELECT coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ + /* Data is coming from a SELECT. Generate a co-routine to run that + ** SELECT. */ + int rc = sqlite3CodeCoroutine(pParse, pSelect, &dest); + if( rc ) goto insert_cleanup; + regEof = dest.iSDParm + 1; regFromSelect = dest.iSdst; assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -89150,7 +90104,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); + SQLITE_CONSTRAINT_NOTNULL, onError, regData+i); zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); @@ -89190,7 +90144,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( }else{ zConsName = 0; } - sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zConsName, P4_DYNAMIC); } sqlite3VdbeResolveLabel(v, allOk); } @@ -89221,8 +90176,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); break; } case OE_Replace: { @@ -89349,7 +90304,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3StrAccumAppend(&errMsg, pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, 0); sqlite3DbFree(errMsg.db, zErr); break; } @@ -89648,7 +90604,7 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } @@ -89757,8 +90713,8 @@ static int xferOptimization( if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); }else if( pDest->pIndex==0 ){ @@ -90215,6 +91171,20 @@ struct sqlite3_api_routines { int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); int (*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); }; /* @@ -90418,6 +91388,20 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen sqlite3_api->blob_reopen #define sqlite3_vtab_config sqlite3_api->vtab_config #define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; @@ -90787,6 +91771,19 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_blob_reopen, sqlite3_vtab_config, sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2 }; /* @@ -91253,6 +92250,9 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ { "sql_trace", SQLITE_SqlTrace }, { "vdbe_listing", SQLITE_VdbeListing }, { "vdbe_trace", SQLITE_VdbeTrace }, + { "vdbe_addoptrace", SQLITE_VdbeAddopTrace}, + { "vdbe_debug", SQLITE_SqlTrace | SQLITE_VdbeListing + | SQLITE_VdbeTrace }, #endif #ifndef SQLITE_OMIT_CHECK { "ignore_check_constraints", SQLITE_IgnoreChecks }, @@ -91426,6 +92426,7 @@ SQLITE_PRIVATE void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; + db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ @@ -92016,11 +93017,14 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - int i; + int i, k; int nHidden = 0; Column *pCol; + Index *pPk; + for(pPk=pTab->pIndex; pPk && pPk->autoIndex!=2; pPk=pPk->pNext){} sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); @@ -92043,7 +93047,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -92059,6 +93070,7 @@ SQLITE_PRIVATE void sqlite3Pragma( pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); @@ -92085,6 +93097,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); @@ -92148,6 +93161,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); @@ -92181,6 +93195,122 @@ SQLITE_PRIVATE void sqlite3Pragma( }else #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + if( sqlite3StrICmp(zLeft, "foreign_key_check")==0 ){ + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + if( pParent==0 ) break; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + } + }else{ + k = 0; + break; + } + } + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + assert( pParent!=0 ); + pIdx = 0; + aiCols = 0; + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + addrOk = sqlite3VdbeMakeLabel(v); + if( pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKeynCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; jnCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[0].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regRow, pFK->nCol, regKey); + sqlite3VdbeChangeP4(v, -1, + sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); + sqlite3VdbeJumpHere(v, addrTop); + } + }else +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ @@ -92302,7 +93432,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp2(v, OP_Move, 2, 4); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); @@ -92605,6 +93735,22 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_db_release_memory(db); }else + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + if( sqlite3StrICmp(zLeft, "busy_timeout")==0 ){ + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "timeout", db->busyTimeout); + }else + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -92620,13 +93766,12 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; - Pager *pPager; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; - if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ @@ -92663,7 +93808,7 @@ SQLITE_PRIVATE void sqlite3Pragma( }else #endif #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ + if( sqlite3StrICmp(zLeft, "activate_extensions")==0 && zRight ){ #ifdef SQLITE_HAS_CODEC if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ sqlite3_activate_see(&zRight[4]); @@ -92835,7 +93980,9 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ int rc; int i; +#ifndef SQLITE_OMIT_DEPRECATED int size; +#endif Table *pTab; Db *pDb; char const *azArg[4]; @@ -92878,7 +94025,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -92958,11 +94105,15 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 u8 encoding; /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; ENC(db) = encoding; +#else + ENC(db) = SQLITE_UTF8; +#endif }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ @@ -93099,7 +94250,7 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ @@ -93122,7 +94273,7 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ @@ -93620,7 +94771,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -93644,7 +94795,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->selFlags = selFlags; pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; @@ -94090,6 +95241,19 @@ static int checkForMultiColumnSelectError( } #endif +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -94106,7 +95270,7 @@ static void selectInnerLoop( int srcTab, /* Pull data from this table */ int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int distinct, /* If >=0, make sure results are distinct */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -94122,7 +95286,7 @@ static void selectInnerLoop( assert( v ); if( NEVER(v==0) ) return; assert( pEList!=0 ); - hasDistinct = distinct>=0; + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ codeOffset(v, p, iContinue); } @@ -94162,7 +95326,55 @@ static void selectInnerLoop( if( hasDistinct ){ assert( pEList!=0 ); assert( pEList->nExpr==nColumn ); - codeDistinct(pParse, distinct, iContinue, nColumn, regResult); + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nColumn; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + + iJump = sqlite3VdbeCurrentAddr(v) + nColumn; + for(i=0; ia[i].pExpr); + if( iaddrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + break; + } + } if( pOrderBy==0 ){ codeOffset(v, p, iContinue); } @@ -94220,7 +95432,8 @@ static void selectInnerLoop( */ case SRT_Set: { assert( nColumn==1 ); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -94229,7 +95442,7 @@ static void selectInnerLoop( pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regResult, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -94496,7 +95709,8 @@ static void generateSortTail( #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); sqlite3ExprCacheAffinityChange(pParse, regRow, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; @@ -94811,7 +96025,7 @@ static void generateColumnNames( static int selectColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ - int *pnCol, /* Write the number of columns here */ + i16 *pnCol, /* Write the number of columns here */ Column **paCol /* Write the new column list here */ ){ sqlite3 *db = pParse->db; /* Database connection */ @@ -94837,9 +96051,7 @@ static int selectColumnsFromExprList( for(i=0, pCol=aCol; ia[i].pExpr; - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); + p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ zName = sqlite3DbStrDup(db, zName); @@ -94877,6 +96089,9 @@ static int selectColumnsFromExprList( for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} + if( zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -95208,6 +96423,8 @@ static int multiSelect( int addr = 0; int nLimit; assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); @@ -95333,7 +96550,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -95411,7 +96628,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); sqlite3VdbeResolveLabel(v, iBreak); @@ -95457,6 +96674,7 @@ static int multiSelect( *apColl = db->pDfltColl; } } + pKeyInfo->aSortOrder = (u8*)apColl; for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ for(i=0; i<2; i++){ @@ -95530,7 +96748,7 @@ static int generateOutputSubroutine( (char*)pKeyInfo, p4type); sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iSdst, regPrev+1, pIn->nSdst); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; @@ -95565,10 +96783,10 @@ static int generateOutputSubroutine( case SRT_Set: { int r1; assert( pIn->nSdst==1 ); - p->affinity = + pDest->affSdst = sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &p->affinity, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -95834,12 +97052,13 @@ static int multiSelectOrderBy( for(i=0; ia[i].pExpr; - if( pTerm->flags & EP_ExpCollate ){ - pColl = pTerm->pColl; + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); }else{ pColl = multiSelectCollSeq(pParse, p, aPermute[i]); - pTerm->flags |= EP_ExpCollate; - pTerm->pColl = pColl; + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); } pKeyMerge->aColl[i] = pColl; pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; @@ -95863,7 +97082,8 @@ static int multiSelectOrderBy( }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite3DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); @@ -96042,14 +97262,9 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite3VdbeResolveLabel(v, labelEnd); @@ -96109,9 +97324,6 @@ static Expr *substExpr( assert( pEList!=0 && pExpr->iColumnnExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - if( pNew && pExpr->pColl ){ - pNew->pColl = pExpr->pColl; - } sqlite3ExprDelete(db, pExpr); pExpr = pNew; } @@ -96310,7 +97522,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; @@ -96464,12 +97676,15 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; + p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -96599,10 +97814,9 @@ static int flattenSubquery( pList = pParent->pEList; for(i=0; inExpr; i++){ if( pList->a[i].zName==0 ){ - const char *zSpan = pList->a[i].zSpan; - if( ALWAYS(zSpan) ){ - pList->a[i].zName = sqlite3DbStrDup(db, zSpan); - } + char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); + sqlite3Dequote(zName); + pList->a[i].zName = zName; } } substExprList(db, pParent->pEList, iParent, pSub->pEList); @@ -96663,34 +97877,43 @@ static int flattenSubquery( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: ** -** 1. There is a single object in the FROM clause. +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. +** +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } } - return WHERE_ORDERBY_NORMAL; + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; } /* @@ -96785,14 +98008,16 @@ static int selectExpander(Walker *pWalker, Select *p){ ExprList *pEList; struct SrcList_item *pFrom; sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } - p->selFlags |= SF_Expanded; pTabList = p->pSrc; pEList = p->pEList; @@ -96833,9 +98058,14 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ @@ -96871,7 +98101,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** that need expanding. */ for(k=0; knExpr; k++){ - Expr *pE = pEList->a[k].pExpr; + pE = pEList->a[k].pExpr; if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); @@ -96889,10 +98119,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; knExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -96907,32 +98145,43 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This expression is a "*" or a "TABLE.*" and needs to be ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ + char *zTName = 0; /* text of name of TABLE */ if( pE->op==TK_DOT ){ assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; - }else{ - zTName = 0; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } - tableSeen = 1; for(j=0; jnCol; j++){ - Expr *pExpr, *pRight; char *zName = pTab->aCol[j].zName; char *zColname; /* The computed column name */ char *zToFree; /* Malloced string that needs to be freed */ Token sColname; /* Computed column name as a token */ + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + /* If a column is marked as 'hidden' (currently only possible ** for virtual tables), do not include it in the expanded ** result-set list. @@ -96941,6 +98190,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert(IsVirtual(pTab)); continue; } + tableSeen = 1; if( i>0 && zTName==0 ){ if( (pFrom->jointype & JT_NATURAL)!=0 @@ -96963,6 +98213,10 @@ static int selectExpander(Walker *pWalker, Select *p){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); zToFree = zColname; @@ -96974,6 +98228,18 @@ static int selectExpander(Walker *pWalker, Select *p){ sColname.z = zColname; sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } sqlite3DbFree(db, zToFree); } } @@ -97112,6 +98378,7 @@ SQLITE_PRIVATE void sqlite3SelectPrep( sqlite3 *db; if( NEVER(p==0) ) return; db = pParse->db; + if( db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); if( pParse->nErr || db->mallocFailed ) return; @@ -97350,11 +98617,9 @@ SQLITE_PRIVATE int sqlite3Select( ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ - int isDistinct; /* True if the DISTINCT keyword is present */ - int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ - int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -97414,8 +98679,17 @@ SQLITE_PRIVATE int sqlite3Select( int isAggSub; if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ if( pItem->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } continue; } @@ -97436,6 +98710,44 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags |= SF_Aggregate; } i = -1; + }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop; + int addrEof; + pItem->regReturn = ++pParse->nMem; + addrEof = ++pParse->nMem; + /* Before coding the OP_Goto to jump to the start of the main routine, + ** ensure that the jump to the verify-schema routine has already + ** been coded. Otherwise, the verify-schema would likely be coded as + ** part of the co-routine. If the main routine then accessed the + ** database before invoking the co-routine for the first time (for + ** example to initialize a LIMIT register from a sub-select), it would + ** be doing so without having verified the schema version and obtained + ** the required db locks. See ticket d6b36be38. */ + sqlite3CodeVerifySchema(pParse, -1); + sqlite3VdbeAddOp0(v, OP_Goto); + addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor); + sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, "coroutine for %s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof); + sqlite3VdbeChangeP5(v, 1); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->viaCoroutine = 1; + sqlite3VdbeChangeP2(v, addrTop, dest.iSdst); + sqlite3VdbeChangeP3(v, addrTop, dest.nSdst); + sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof); + sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); }else{ /* Generate a subroutine that will fill an ephemeral table with ** the content of this subquery. pItem->addrFillSub will point @@ -97480,7 +98792,7 @@ SQLITE_PRIVATE int sqlite3Select( pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; - isDistinct = (p->selFlags & SF_Distinct)!=0; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #ifndef SQLITE_OMIT_COMPOUND_SELECT /* If there is are a sequence of queries, do the earlier ones first. @@ -97515,7 +98827,7 @@ SQLITE_PRIVATE int sqlite3Select( ** to disable this optimization for testing purposes. */ if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ + && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ pOrderBy = 0; } @@ -97541,6 +98853,10 @@ SQLITE_PRIVATE int sqlite3Select( p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; pOrderBy = 0; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); } /* If there is an ORDER BY clause, then this sorting @@ -97581,24 +98897,27 @@ SQLITE_PRIVATE int sqlite3Select( /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ - KeyInfo *pKeyInfo; - distinct = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList), + P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ - distinct = addrDistinctIndex = -1; + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; } - /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - ExprList *pDist = (isDistinct ? p->pEList : 0); + /* No aggregate functions and no GROUP BY clause */ + ExprList *pDist = (sDistinct.isTnct ? p->pEList : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; + if( pWInfo->eDistinct ) sDistinct.eTnctType = pWInfo->eDistinct; + if( pOrderBy && pWInfo->nOBSat==pOrderBy->nExpr ) pOrderBy = 0; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -97609,59 +98928,16 @@ SQLITE_PRIVATE int sqlite3Select( p->addrOpenEphm[2] = -1; } - if( pWInfo->eDistinct ){ - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - - assert( addrDistinctIndex>=0 ); - pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); - - assert( isDistinct ); - assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED - || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE - ); - distinct = -1; - if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ - int iJump; - int iExpr; - int iFlag = ++pParse->nMem; - int iBase = pParse->nMem+1; - int iBase2 = iBase + pEList->nExpr; - pParse->nMem += (pEList->nExpr*2); - - /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The - ** OP_Integer initializes the "first row" flag. */ - pOp->opcode = OP_Integer; - pOp->p1 = 1; - pOp->p2 = iFlag; - - sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); - iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; - sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); - for(iExpr=0; iExprnExpr; iExpr++){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); - sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); - sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); - - sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); - assert( sqlite3VdbeCurrentAddr(v)==iJump ); - sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); - }else{ - pOp->opcode = OP_Noop; - } - } - /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. */ sqlite3WhereEnd(pWInfo); }else{ - /* This is the processing for aggregate queries */ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -97769,14 +99045,13 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ + if( pWInfo->nOBSat==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ - pGroupBy = p->pGroupBy; groupBySort = 0; }else{ /* Rows are coming out in undetermined order. We have to push @@ -97790,7 +99065,8 @@ SQLITE_PRIVATE int sqlite3Select( int nGroupBy; explainTempTable(pParse, - isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -97922,7 +99198,7 @@ SQLITE_PRIVATE int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, pDest, + &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -98010,7 +99286,7 @@ SQLITE_PRIVATE int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value @@ -98022,10 +99298,17 @@ SQLITE_PRIVATE int sqlite3Select( ** Refer to code and comments in where.c for details. */ ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; @@ -98038,13 +99321,14 @@ SQLITE_PRIVATE int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ + assert( pMinMax==0 || pMinMax->nExpr==1 ); + if( pWInfo->nOBSat>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -98055,7 +99339,7 @@ SQLITE_PRIVATE int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, + selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -98063,7 +99347,7 @@ SQLITE_PRIVATE int sqlite3Select( } /* endif aggregate query */ - if( distinct>=0 ){ + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ explainTempTable(pParse, "DISTINCT"); } @@ -98180,7 +99464,10 @@ SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ sqlite3ExplainPrintf(pVdbe, "(null-select)"); return; } - while( p->pPrior ) p = p->pPrior; + while( p->pPrior ){ + p->pPrior->pNext = p; + p = p->pPrior; + } sqlite3ExplainPush(pVdbe); while( p ){ explainOneSelect(pVdbe, p); @@ -99128,6 +100415,15 @@ static int codeTriggerProgram( */ pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; + /* Clear the cookieGoto flag. When coding triggers, the cookieGoto + ** variable is used as a flag to indicate to sqlite3ExprCodeConstants() + ** that it is not safe to refactor constants (this happens after the + ** start of the first loop in the SQL statement is coded - at that + ** point code may be conditionally executed, so it is no longer safe to + ** initialize constant register values). */ + assert( pParse->cookieGoto==0 || pParse->cookieGoto==-1 ); + pParse->cookieGoto = 0; + switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, @@ -99983,7 +101279,7 @@ SQLITE_PRIVATE void sqlite3Update( /* The row-trigger may have deleted the row being updated. In this ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated + ** required. This behavior - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the ** documentation. */ @@ -100284,6 +101580,7 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ Vdbe *v = sqlite3GetVdbe(pParse); if( v ){ sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); + sqlite3VdbeUsesBtree(v, 0); } return; } @@ -100812,7 +102109,7 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ if( p->azModuleArg ){ int i; for(i=0; inModuleArg; i++){ - sqlite3DbFree(db, p->azModuleArg[i]); + if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); } sqlite3DbFree(db, p->azModuleArg); } @@ -100872,7 +102169,7 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); + addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); @@ -101029,6 +102326,7 @@ static int vtabCallConstructor( int nArg = pTab->nModuleArg; char *zErr = 0; char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + int iDb; if( !zModuleName ){ return SQLITE_NOMEM; @@ -101042,6 +102340,9 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->azModuleArg[1] = db->aDb[iDb].zName; + /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); @@ -101076,7 +102377,7 @@ static int vtabCallConstructor( /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". - ** If so, set the Column.isHidden flag and remove the token from + ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from ** the type string. */ pVTable->pNext = pTab->pVTable; pTab->pVTable = pVTable; @@ -101107,7 +102408,7 @@ static int vtabCallConstructor( assert(zType[i-1]==' '); zType[i-1] = '\0'; } - pTab->aCol[iCol].isHidden = 1; + pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; } } } @@ -101668,9 +102969,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace = 0; #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) # define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X #else # define WHERETRACE(X) @@ -101742,8 +103044,8 @@ struct WhereTerm { int leftCursor; /* Cursor number of X in "X " */ union { int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ @@ -101784,7 +103086,6 @@ struct WhereTerm { struct WhereClause { Parse *pParse; /* The parser context */ WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ @@ -101871,6 +103172,7 @@ struct WhereCost { #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ @@ -101897,18 +103199,55 @@ struct WhereCost { #define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ #define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ #define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ +#define WHERE_IN_ABLE 0x080f1000 /* Able to support an IN operator */ #define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xrCostrCost ) return 1; + if( pProbe->rCost>pBaseline->rCost ) return 0; + if( pProbe->plan.nOBSat>pBaseline->plan.nOBSat ) return 1; + if( pProbe->plan.nRowplan.nRow ) return 1; + return 0; +} /* ** Initialize a preallocated WhereClause structure. @@ -101925,7 +103264,6 @@ static void whereClauseInit( pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; pWC->wctrlFlags = wctrlFlags; } @@ -102012,7 +103350,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } pTerm = &pWC->a[idx = pWC->nTerm++]; - pTerm->pExpr = p; + pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; @@ -102172,23 +103510,32 @@ static int allowedOp(int op){ ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If a collation sequence is associated with either the left or right +** If left/right precedence rules come into play when determining the +** collating ** side of the comparison, it remains associated with the same side after ** the commutation. So "Y collate NOCASE op X" becomes -** "X collate NOCASE op Y". This is because any collation sequence on +** "X op Y". This is because any collation sequence on ** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_ExpCollate flag +** attached to the right. For the same reason the EP_Collate flag ** is not commuted. */ static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); - u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); - pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; - pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } + } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ assert( TK_LT==TK_GT+2 ); @@ -102229,6 +103576,23 @@ static u16 operatorMask(int op){ ** where X is a reference to the iColumn of table iCur and is one of ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ static WhereTerm *findTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -102238,45 +103602,85 @@ static WhereTerm *findTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - WhereTerm *pTerm; - int k; + WhereTerm *pTerm; /* Term being examined as possible result */ + WhereTerm *pResult = 0; /* The answer to return */ + WhereClause *pWCOrig = pWC; /* Original pWC value */ + int j, k; /* Loop counters */ + Expr *pX; /* Pointer to an expression */ + Parse *pParse; /* Parsing context */ + int iOrigCol = iColumn; /* Original value of iColumn */ + int nEquiv = 2; /* Number of entires in aEquiv[] */ + int iEquiv = 2; /* Number of entries of aEquiv[] processed so far */ + int aEquiv[22]; /* iCur,iColumn and up to 10 other equivalents */ + assert( iCur>=0 ); - op &= WO_ALL; - for(; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - assert(pColl || pParse->nErr); - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; + aEquiv[0] = iCur; + aEquiv[1] = iColumn; + for(;;){ + for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){ + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + ){ + if( (pTerm->prereqRight & notReady)==0 + && (pTerm->eOperator & op & WO_ALL)!=0 + ){ + if( iOrigCol>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + char idxaff; + + pX = pTerm->pExpr; + pParse = pWC->pParse; + idxaff = pIdx->pTable->aCol[iOrigCol].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ){ + continue; + } + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + + for(j=0; pIdx->aiColumn[j]!=iOrigCol; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ){ + continue; + } + } + if( pTerm->prereqRight==0 ){ + pResult = pTerm; + goto findTerm_success; + }else if( pResult==0 ){ + pResult = pTerm; + } + } + if( (pTerm->eOperator & WO_EQUIV)!=0 + && nEquivpExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jiTable && aEquiv[j+1]==pX->iColumn ) break; + } + if( j==nEquiv ){ + aEquiv[j] = pX->iTable; + aEquiv[j+1] = pX->iColumn; + nEquiv += 2; + } } - if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; } - return pTerm; } } + if( iEquiv>=nEquiv ) break; + iCur = aEquiv[iEquiv++]; + iColumn = aEquiv[iEquiv++]; } - return 0; +findTerm_success: + return pResult; } /* Forward reference */ @@ -102462,7 +103866,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: @@ -102550,11 +103954,10 @@ static void exprAnalyzeOrTerm( ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); @@ -102593,7 +103996,7 @@ static void exprAnalyzeOrTerm( b |= getMask(pMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; @@ -102644,7 +104047,7 @@ static void exprAnalyzeOrTerm( for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and @@ -102670,7 +104073,7 @@ static void exprAnalyzeOrTerm( /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); + assert( IsPowerOfTwo(chngToIN) ); assert( chngToIN==getMask(pMaskSet, iCursor) ); break; } @@ -102680,7 +104083,7 @@ static void exprAnalyzeOrTerm( ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ @@ -102716,7 +104119,7 @@ static void exprAnalyzeOrTerm( for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); @@ -102746,7 +104149,6 @@ static void exprAnalyzeOrTerm( } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -102789,6 +104191,7 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pMaskSet = pWC->pMaskSet; pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ @@ -102814,17 +104217,19 @@ static void exprAnalyze( pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; + if( allowedOp(op) ){ + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pLeft->op==TK_COLUMN ){ pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + pTerm->eOperator = operatorMask(op) & opMask; } if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -102839,18 +104244,25 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = pDup->pLeft; + pLeft = sqlite3ExprSkipCollate(pDup->pLeft); pNew->leftCursor = pLeft->iTable; pNew->u.leftColumn = pLeft->iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } } @@ -102923,7 +104335,7 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; - CollSeq *pColl; /* Collating sequence to use */ + Token sCollSeqName; /* Name of collating sequence */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); @@ -102945,16 +104357,19 @@ static void exprAnalyze( } *pC = c + 1; } - pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; + sCollSeqName.n = 6; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr1, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), + pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr2, 0); + sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), + pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); @@ -103051,25 +104466,6 @@ static void exprAnalyze( pTerm->prereqRight |= extraRight; } -/* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. -*/ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ -){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirstnExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ - return 1; - } - } - return 0; -} - /* ** This function searches the expression list passed as the second argument ** for an expression of type TK_COLUMN that refers to the same column and @@ -103091,12 +104487,12 @@ static int findIndexCol( const char *zColl = pIdx->azColl[iCol]; for(i=0; inExpr; i++){ - Expr *p = pList->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); if( p->op==TK_COLUMN && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ return i; } @@ -103129,7 +104525,8 @@ static int isDistinctIndex( Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ int i; /* Iterator variable */ - if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + assert( pDistinct!=0 ); + if( pIdx->zName==0 || pDistinct->nExpr>=BMS ) return 0; testcase( pDistinct->nExpr==BMS-1 ); /* Loop through all the expressions in the distinct list. If any of them @@ -103142,7 +104539,7 @@ static int isDistinctIndex( */ for(i=0; inExpr; i++){ WhereTerm *pTerm; - Expr *p = pDistinct->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); if( p->op!=TK_COLUMN ) return 0; pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); if( pTerm ){ @@ -103194,7 +104591,7 @@ static int isDistinctRedundant( ** current SELECT is a correlated sub-query. */ for(i=0; inExpr; i++){ - Expr *p = pDistinct->a[i].pExpr; + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } @@ -103231,165 +104628,6 @@ static int isDistinctRedundant( return 0; } -/* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. -** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. -** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. -** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. -*/ -static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int wsFlags, /* Index usages flags */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ -){ - int i, j; /* Loop counters */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ - sqlite3 *db = pParse->db; - - if( !pOrderBy ) return 0; - if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->bUnordered ) return 0; - - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=j=0, pTerm=pOrderBy->a; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ - - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return 0; - } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqCol ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - return 0; - } - }else{ - sortOrder = termSortOrder; - } - j++; - pTerm++; - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; - } - } - - *pbRev = sortOrder!=0; - if( j>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index so - ** this index can be used for sorting. */ - return 1; - } - if( pIdx->onError!=OE_None && i==pIdx->nColumn - && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) - ){ - Column *aCol = pIdx->pTable->aCol; - - /* All terms of this index match some prefix of the ORDER BY clause, - ** the index is UNIQUE, and no terms on the tail of the ORDER BY - ** refer to other tables in a join. So, assuming that the index entries - ** visited contain no NULL values, then this index delivers rows in - ** the required order. - ** - ** It is not possible for any of the first nEqCol index fields to be - ** NULL (since the corresponding "=" operator in the WHERE clause would - ** not be true). So if all remaining index columns have NOT NULL - ** constaints attached to them, we can be confident that the visited - ** index entries are free of NULLs. */ - for(i=nEqCol; inColumn; i++){ - if( aCol[pIdx->aiColumn[i]].notNull==0 ) break; - } - return (i==pIdx->nColumn); - } - return 0; -} - /* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating @@ -103454,9 +104692,7 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ /* ** Required because bestIndex() is called by bestOrClauseIndex() */ -static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, - Bitmask, Bitmask, ExprList*, WhereCost*); +static void bestIndex(WhereBestIdx*); /* ** This routine attempts to find an scanning strategy that can be used @@ -103465,20 +104701,14 @@ static void bestIndex( ** The table associated with FROM clause term pSrc may be either a ** regular B-Tree table or a virtual table. */ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestOrClauseIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + const int iCur = pSrc->iCursor; /* The cursor of the table */ const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ /* The OR-clause optimization is disallowed if the INDEXED BY or ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ @@ -103491,8 +104721,8 @@ static void bestOrClauseIndex( /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTermeOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 + if( (pTerm->eOperator & WO_OR)!=0 + && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; @@ -103502,15 +104732,19 @@ static void bestOrClauseIndex( double rTotal = 0; double nRow = 0; Bitmask used = 0; + WhereBestIdx sBOI; + sBOI = *p; + sBOI.pOrderBy = 0; + sBOI.pDistinct = 0; + sBOI.ppIdxInfo = 0; for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) )); - if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); + if( (pOrTerm->eOperator& WO_AND)!=0 ){ + sBOI.pWC = &pOrTerm->u.pAndInfo->wc; + bestIndex(&sBOI); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; @@ -103520,19 +104754,20 @@ static void bestOrClauseIndex( tempWC.a = pOrTerm; tempWC.wctrlFlags = 0; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); + sBOI.pWC = &tempWC; + bestIndex(&sBOI); }else{ continue; } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; + rTotal += sBOI.cost.rCost; + nRow += sBOI.cost.plan.nRow; + used |= sBOI.cost.used; + if( rTotal>=p->cost.rCost ) break; } /* If there is an ORDER BY clause, increase the scan cost to account ** for the cost of the sort. */ - if( pOrderBy!=0 ){ + if( p->pOrderBy!=0 ){ WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", rTotal, rTotal+nRow*estLog(nRow))); rTotal += nRow*estLog(nRow); @@ -103542,12 +104777,13 @@ static void bestOrClauseIndex( ** less than the current cost stored in pCost, replace the contents ** of pCost. */ WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotalrCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; + if( rTotalcost.rCost ){ + p->cost.rCost = rTotal; + p->cost.used = used; + p->cost.plan.nRow = nRow; + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; + p->cost.plan.wsFlags = flags; + p->cost.plan.u.pTerm = pTerm; } } } @@ -103567,7 +104803,7 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; @@ -103584,15 +104820,12 @@ static int termCanDriveIndex( ** is taken into account, then alter the query plan to use the ** transient index. */ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ +static void bestAutomaticIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + double nTableRow; /* Rows in the input table */ + double logN; /* log(nTableRow) */ double costTempIdx; /* per-query cost of the transient index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ @@ -103606,10 +104839,16 @@ static void bestAutomaticIndex( /* Automatic indices are disabled at run-time */ return; } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ + if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 + && (p->cost.plan.wsFlags & WHERE_COVER_SCAN)==0 + ){ /* We already have some kind of index in use for this query. */ return; } + if( pSrc->viaCoroutine ){ + /* Cannot index a co-routine */ + return; + } if( pSrc->notIndexed ){ /* The NOT INDEXED clause appears in the SQL. */ return; @@ -103624,7 +104863,7 @@ static void bestAutomaticIndex( nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ + if( costTempIdx>=p->cost.rCost ){ /* The cost of creating the transient table would be greater than ** doing the full table scan */ return; @@ -103633,19 +104872,19 @@ static void bestAutomaticIndex( /* Search for any equality comparison term */ pWCEnd = &pWC->a[pWC->nTerm]; for(pTerm=pWC->a; pTermnotReady) ){ WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", - pCost->rCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->used = pTerm->prereqRight; + p->cost.rCost, costTempIdx)); + p->cost.rCost = costTempIdx; + p->cost.plan.nRow = logN + 1; + p->cost.plan.wsFlags = WHERE_TEMP_INDEX; + p->cost.used = pTerm->prereqRight; break; } } } #else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ +# define bestAutomaticIndex(A) /* no-op */ #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -103806,12 +105045,11 @@ static void constructAutomaticIndex( ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to sqlite3_free(). */ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy -){ +static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ + Parse *pParse = p->pParse; + WhereClause *pWC = p->pWC; + struct SrcList_item *pSrc = p->pSrc; + ExprList *pOrderBy = p->pOrderBy; int i, j; int nTerm; struct sqlite3_index_constraint *pIdxCons; @@ -103827,10 +105065,10 @@ static sqlite3_index_info *allocateIndexInfo( ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } @@ -103841,12 +105079,13 @@ static sqlite3_index_info *allocateIndexInfo( */ nOrderBy = 0; if( pOrderBy ){ - for(i=0; inExpr; i++){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; + if( i==n){ + nOrderBy = n; } } @@ -103877,15 +105116,18 @@ static sqlite3_index_info *allocateIndexInfo( pUsage; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The ** following asserts verify this fact. */ @@ -103895,7 +105137,7 @@ static sqlite3_index_info *allocateIndexInfo( assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); j++; } for(i=0; ipParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ Table *pTab = pSrc->pTab; sqlite3_index_info *pIdxInfo; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage; WhereTerm *pTerm; - int i, j; + int i, j, k; int nOrderBy; + int sortOrder; /* Sort order for IN clauses */ + int bAllowIN; /* Allow IN optimizations */ double rCost; /* Make sure wsFlags is initialized to some sane value. Otherwise, if the ** malloc in allocateIndexInfo() fails and this function returns leaving ** wsFlags in an uninitialized state, the caller may behave unpredictably. */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.plan.wsFlags = WHERE_VIRTUALTABLE; /* If the sqlite3_index_info structure has not been previously ** allocated and initialized, then allocate and initialize it now. */ - pIdxInfo = *ppIdxInfo; + pIdxInfo = *p->ppIdxInfo; if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); + *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(p); } if( pIdxInfo==0 ){ return; @@ -104021,65 +105259,102 @@ static void bestVirtualIndex( assert( pTab->azModuleArg && pTab->azModuleArg[0] ); assert( sqlite3GetVTable(pParse->db, pTab) ); - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. + /* Try once or twice. On the first attempt, allow IN optimizations. + ** If an IN optimization is accepted by the virtual table xBestIndex + ** method, but the pInfo->aConstrainUsage.omit flag is not set, then + ** the query will not work because it might allow duplicate rows in + ** output. In that case, run the xBestIndex method a second time + ** without the IN constraints. Usually this loop only runs once. + ** The loop will exit using a "break" statement. */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } + for(bAllowIN=1; 1; bAllowIN--){ + assert( bAllowIN==0 || bAllowIN==1 ); - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } - - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; + /* Set the aConstraint[].usable fields and initialize all + ** output variables to zero. + ** + ** aConstraint[].usable is true for constraints where the right-hand + ** side contains only references to tables to the left of the current + ** table. In other words, if the constraint is of the form: + ** + ** column = expr + ** + ** and we are evaluating a join, then the constraint on column is + ** only valid if all tables referenced in expr occur to the left + ** of the table containing column. + ** + ** The aConstraints[] array contains entries for all constraints + ** on the current table. That way we only have to compute it once + ** even though we might try to pick the best index multiple times. + ** For each attempt at picking an index, the order of tables in the + ** join might be different so we have to recompute the usable flag + ** each time. + */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pUsage = pIdxInfo->aConstraintUsage; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + if( (pTerm->prereqRight&p->notReady)==0 + && (bAllowIN || (pTerm->eOperator & WO_IN)==0) + ){ + pIdxCons->usable = 1; + }else{ + pIdxCons->usable = 0; + } } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + } + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + nOrderBy = pIdxInfo->nOrderBy; + if( !p->pOrderBy ){ + pIdxInfo->nOrderBy = 0; + } + + if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ + return; + } + + sortOrder = SQLITE_SO_ASC; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + if( pUsage[i].argvIndex>0 ){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + p->cost.used |= pTerm->prereqRight; + if( (pTerm->eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + for(k=0; knOrderBy; k++){ + if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){ + sortOrder = pIdxInfo->aOrderBy[k].desc; + break; + } + } + } + } + } + if( i>=pIdxInfo->nConstraint ) break; } - + /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This ** matches the processing for non-virtual tables in bestBtreeIndex(). */ rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ + if( p->pOrderBy && pIdxInfo->orderByConsumed==0 ){ rCost += estLog(rCost)*rCost; } @@ -104091,21 +105366,25 @@ static void bestVirtualIndex( ** is defined. */ if( (SQLITE_BIG_DBL/((double)2))rCost = (SQLITE_BIG_DBL/((double)2)); + p->cost.rCost = (SQLITE_BIG_DBL/((double)2)); }else{ - pCost->rCost = rCost; + p->cost.rCost = rCost; } - pCost->plan.u.pVtabIdx = pIdxInfo; + p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; + assert( sortOrder==0 || sortOrder==1 ); + p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE; + p->cost.plan.nOBSat = nOrderBy; + }else{ + p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; } - pCost->plan.nEq = 0; + p->cost.plan.nEq = 0; pIdxInfo->nOrderBy = nOrderBy; /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestOrClauseIndex(p); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -104193,10 +105472,8 @@ static int whereKeyStats( pColl = db->pDfltColl; assert( pColl->enc==SQLITE_UTF8 ); }else{ - pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); + pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl); if( pColl==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", - *pIdx->azColl); return SQLITE_ERROR; } z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); @@ -104367,24 +105644,24 @@ static int whereRangeScanEst( if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK ){ iLower = a[0]; - if( pLower->eOperator==WO_GT ) iLower += a[1]; + if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK ){ iUpper = a[0]; - if( pUpper->eOperator==WO_LE ) iUpper += a[1]; + if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } sqlite3ValueFree(pRangeVal); } @@ -104504,11 +105781,295 @@ static int whereInScanEst( } #endif /* defined(SQLITE_ENABLE_STAT3) */ +/* +** Check to see if column iCol of the table with cursor iTab will appear +** in sorted order according to the current query plan. +** +** Return values: +** +** 0 iCol is not ordered +** 1 iCol has only a single value +** 2 iCol is in ASC order +** 3 iCol is in DESC order +*/ +static int isOrderedColumn( + WhereBestIdx *p, + int iTab, + int iCol +){ + int i, j; + WhereLevel *pLevel = &p->aLevel[p->i-1]; + Index *pIdx; + u8 sortOrder; + for(i=p->i-1; i>=0; i--, pLevel--){ + if( pLevel->iTabCur!=iTab ) continue; + if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + return 1; + } + assert( (pLevel->plan.wsFlags & WHERE_ORDERED)!=0 ); + if( (pIdx = pLevel->plan.u.pIdx)!=0 ){ + if( iCol<0 ){ + sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + }else{ + int n = pIdx->nColumn; + for(j=0; jaiColumn[j] ) break; + } + if( j>=n ) return 0; + sortOrder = pIdx->aSortOrder[j]; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + } + }else{ + if( iCol!=(-1) ) return 0; + sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + } + if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ){ + assert( sortOrder==0 || sortOrder==1 ); + testcase( sortOrder==1 ); + sortOrder = 1 - sortOrder; + } + return sortOrder+2; + } + return 0; +} + +/* +** This routine decides if pIdx can be used to satisfy the ORDER BY +** clause, either in whole or in part. The return value is the +** cumulative number of terms in the ORDER BY clause that are satisfied +** by the index pIdx and other indices in outer loops. +** +** The table being queried has a cursor number of "base". pIdx is the +** index that is postulated for use to access the table. +** +** The *pbRev value is set to 0 order 1 depending on whether or not +** pIdx should be run in the forward order or in reverse order. +*/ +static int isSortingIndex( + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + int *pbRev, /* Set to 1 for reverse-order scan of pIdx */ + int *pbObUnique /* ORDER BY column values will different in every row */ +){ + int i; /* Number of pIdx terms used */ + int j; /* Number of ORDER BY terms satisfied */ + int sortOrder = 2; /* 0: forward. 1: backward. 2: unknown */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pOBItem;/* A term of the ORDER BY clause */ + Table *pTab = pIdx->pTable; /* Table that owns index pIdx */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Parse *pParse = p->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ + int outerObUnique; /* Outer loops generate different values in + ** every row for the ORDER BY columns */ + + if( p->i==0 ){ + nPriorSat = 0; + outerObUnique = 1; + }else{ + u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags; + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + if( (wsFlags & WHERE_ORDERED)==0 ){ + /* This loop cannot be ordered unless the next outer loop is + ** also ordered */ + return nPriorSat; + } + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ){ + /* Only look at the outer-most loop if the OrderByIdxJoin + ** optimization is disabled */ + return nPriorSat; + } + testcase( wsFlags & WHERE_OB_UNIQUE ); + testcase( wsFlags & WHERE_ALL_UNIQUE ); + outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0; + } + pOrderBy = p->pOrderBy; + assert( pOrderBy!=0 ); + if( pIdx->bUnordered ){ + /* Hash indices (indicated by the "unordered" tag on sqlite_stat1) cannot + ** be used for sorting */ + return nPriorSat; + } + nTerm = pOrderBy->nExpr; + uniqueNotNull = pIdx->onError!=OE_None; + assert( nTerm>0 ); + + /* Argument pIdx must either point to a 'real' named index structure, + ** or an index structure allocated on the stack by bestBtreeIndex() to + ** represent the rowid index that is part of every table. */ + assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + + /* Match terms of the ORDER BY clause against columns of + ** the index. + ** + ** Note that indices have pIdx->nColumn regular columns plus + ** one additional column containing the rowid. The rowid column + ** of the index is also allowed to match against the ORDER BY + ** clause. + */ + j = nPriorSat; + for(i=0,pOBItem=&pOrderBy->a[j]; jnColumn; i++){ + Expr *pOBExpr; /* The expression of the ORDER BY pOBItem */ + CollSeq *pColl; /* The collating sequence of pOBExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + int isEq; /* Subject to an == or IS NULL constraint */ + int isMatch; /* ORDER BY term matches the index term */ + const char *zColl; /* Name of collating sequence for i-th index term */ + WhereTerm *pConstraint; /* A constraint in the WHERE clause */ + + /* If the next term of the ORDER BY clause refers to anything other than + ** a column in the "base" table, then this index will not be of any + ** further use in handling the ORDER BY. */ + pOBExpr = sqlite3ExprSkipCollate(pOBItem->pExpr); + if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){ + break; + } + + /* Find column number and collating sequence for the next entry + ** in the index */ + if( pIdx->zName && inColumn ){ + iColumn = pIdx->aiColumn[i]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = -1; + } + iSortOrder = pIdx->aSortOrder[i]; + zColl = pIdx->azColl[i]; + assert( zColl!=0 ); + }else{ + iColumn = -1; + iSortOrder = 0; + zColl = 0; + } + + /* Check to see if the column number and collating sequence of the + ** index match the column number and collating sequence of the ORDER BY + ** clause entry. Set isMatch to 1 if they both match. */ + if( pOBExpr->iColumn==iColumn ){ + if( zColl ){ + pColl = sqlite3ExprCollSeq(pParse, pOBItem->pExpr); + if( !pColl ) pColl = db->pDfltColl; + isMatch = sqlite3StrICmp(pColl->zName, zColl)==0; + }else{ + isMatch = 1; + } + }else{ + isMatch = 0; + } + + /* termSortOrder is 0 or 1 for whether or not the access loop should + ** run forward or backwards (respectively) in order to satisfy this + ** term of the ORDER BY clause. */ + assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = iSortOrder ^ pOBItem->sortOrder; + + /* If X is the column in the index and ORDER BY clause, check to see + ** if there are any X= or X IS NULL constraints in the WHERE clause. */ + pConstraint = findTerm(p->pWC, base, iColumn, p->notReady, + WO_EQ|WO_ISNULL|WO_IN, pIdx); + if( pConstraint==0 ){ + isEq = 0; + }else if( (pConstraint->eOperator & WO_IN)!=0 ){ + isEq = 0; + }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ + uniqueNotNull = 0; + isEq = 1; /* "X IS NULL" means X has only a single value */ + }else if( pConstraint->prereqRight==0 ){ + isEq = 1; /* Constraint "X=constant" means X has only a single value */ + }else{ + Expr *pRight = pConstraint->pExpr->pRight; + if( pRight->op==TK_COLUMN ){ + WHERETRACE((" .. isOrderedColumn(tab=%d,col=%d)", + pRight->iTable, pRight->iColumn)); + isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn); + WHERETRACE((" -> isEq=%d\n", isEq)); + + /* If the constraint is of the form X=Y where Y is an ordered value + ** in an outer loop, then make sure the sort order of Y matches the + ** sort order required for X. */ + if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){ + testcase( isEq==2 ); + testcase( isEq==3 ); + break; + } + }else{ + isEq = 0; /* "X=expr" places no ordering constraints on X */ + } + } + if( !isMatch ){ + if( isEq==0 ){ + break; + }else{ + continue; + } + }else if( isEq!=1 ){ + if( sortOrder==2 ){ + sortOrder = termSortOrder; + }else if( termSortOrder!=sortOrder ){ + break; + } + } + j++; + pOBItem++; + if( iColumn<0 ){ + seenRowid = 1; + break; + }else if( pTab->aCol[iColumn].notNull==0 && isEq!=1 ){ + testcase( isEq==0 ); + testcase( isEq==2 ); + testcase( isEq==3 ); + uniqueNotNull = 0; + } + } + if( seenRowid ){ + uniqueNotNull = 1; + }else if( uniqueNotNull==0 || inColumn ){ + uniqueNotNull = 0; + } + + /* If we have not found at least one ORDER BY term that matches the + ** index, then show no progress. */ + if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; + + /* Either the outer queries must generate rows where there are no two + ** rows with the same values in all ORDER BY columns, or else this + ** loop must generate just a single row of output. Example: Suppose + ** the outer loops generate A=1 and A=1, and this loop generates B=3 + ** and B=4. Then without the following test, ORDER BY A,B would + ** generate the wrong order output: 1,3 1,4 1,3 1,4 + */ + if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat; + *pbObUnique = uniqueNotNull; + + /* Return the necessary scan order back to the caller */ + *pbRev = sortOrder & 1; + + /* If there was an "ORDER BY rowid" term that matched, or it is only + ** possible for a single row from this table to match, then skip over + ** any additional ORDER BY terms dealing with this table. + */ + if( uniqueNotNull ){ + /* Advance j over additional ORDER BY terms associated with base */ + WhereMaskSet *pMS = p->pWC->pMaskSet; + Bitmask m = ~getMask(pMS, base); + while( ja[j].pExpr)&m)==0 ){ + j++; + } + } + return j; +} /* ** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. +** best query plan and its cost into the p->cost. ** ** The lowest cost plan wins. The cost is an estimate of the amount of ** CPU and disk I/O needed to process the requested result. @@ -104528,21 +106089,15 @@ static int whereInScanEst( ** SQLITE_BIG_DBL. If a plan is found that uses the named index, ** then the cost is calculated in the usual way. ** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table +** If a NOT INDEXED clause was attached to the table ** in the SELECT statement, then no indexes are considered. However, the ** selected plan may still take advantage of the built-in rowid primary key ** index. */ -static void bestBtreeIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - ExprList *pDistinct, /* The select-list if query is DISTINCT */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestBtreeIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ Index *pProbe; /* An index we are evaluating */ Index *pIdx; /* Copy of pProbe, or zero for IPK index */ @@ -104551,11 +106106,16 @@ static void bestBtreeIndex( Index sPk; /* A fake index object for the primary key */ tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ + int wsFlagMask; /* Allowed flags in p->cost.plan.wsFlag */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int nOrderBy; /* Number of ORDER BY terms */ + char bSortInit; /* Initializer for bSort in inner loop */ + char bDistInit; /* Initializer for bDist in inner loop */ + /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE_BIG_DBL; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.rCost = SQLITE_BIG_DBL; /* If the pSrc table is the right table of a LEFT JOIN then we may not ** use an index to satisfy IS NULL constraints on that table. This is @@ -104601,22 +106161,29 @@ static void bestBtreeIndex( pIdx = 0; } + nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; + if( p->i ){ + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + bSortInit = nPriorSat0; + bDistInit = p->pDistinct!=0; + } + /* Loop over all indices looking for the best one to use */ for(; pProbe; pIdx=pProbe=pProbe->pNext){ const tRowcnt * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ + WhereCost pc; /* Cost of using pProbe */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - int rev; /* True to scan in reverse order */ - int wsFlags = 0; - Bitmask used = 0; /* The following variables are populated based on the properties of ** index being evaluated. They are then used to determine the expected ** cost and number of rows returned. ** - ** nEq: + ** pc.plan.nEq: ** Number of equality terms that can be implemented using the index. ** In other words, the number of initial fields in the index that ** are used in == or IN or NOT NULL constraints of the WHERE clause. @@ -104659,6 +106226,10 @@ static void bestBtreeIndex( ** external sort (i.e. scanning the index being evaluated will not ** correctly order records). ** + ** bDist: + ** Boolean. True if there is a DISTINCT clause that will require an + ** external btree. + ** ** bLookup: ** Boolean. True if a table lookup is required for each index entry ** visited. In other words, true if this is not a covering index. @@ -104674,29 +106245,35 @@ static void bestBtreeIndex( ** SELECT a, b FROM tbl WHERE a = 1; ** SELECT a, b, c FROM tbl WHERE a = 1; */ - int nEq; /* Number of == or IN terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = !!pOrderBy; /* True if external sort required */ - int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ - int bLookup = 0; /* True if not a covering index */ + char bSort = bSortInit; /* True if external sort required */ + char bDist = bDistInit; /* True if index cannot help with DISTINCT */ + char bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif - /* Determine the values of nEq and nInMul */ - for(nEq=0; nEqnColumn; nEq++){ - int j = pProbe->aiColumn[nEq]; - pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); + WHERETRACE(( + " %s(%s):\n", + pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk") + )); + memset(&pc, 0, sizeof(pc)); + pc.plan.nOBSat = nPriorSat; + + /* Determine the values of pc.plan.nEq and nInMul */ + for(pc.plan.nEq=0; pc.plan.nEqnColumn; pc.plan.nEq++){ + int j = pProbe->aiColumn[pc.plan.nEq]; + pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; - wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); + pc.plan.wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); testcase( pTerm->pWC!=pWC ); if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; + pc.plan.wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ nInMul *= 25; @@ -104706,12 +106283,12 @@ static void bestBtreeIndex( nInMul *= pExpr->x.pList->nExpr; } }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; + pc.plan.wsFlags |= WHERE_COLUMN_NULL; } #ifdef SQLITE_ENABLE_STAT3 - if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; + if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; #endif - used |= pTerm->prereqRight; + pc.used |= pTerm->prereqRight; } /* If the index being considered is UNIQUE, and there is an equality @@ -104720,65 +106297,82 @@ static void bestBtreeIndex( ** indicate this to the caller. ** ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (nEq+1) that can be - ** optimized using the index. + ** there is a range constraint on indexed column (pc.plan.nEq+1) that + ** can be optimized using the index. */ - if( nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; + if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ + testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); + testcase( pc.plan.wsFlags & WHERE_COLUMN_NULL ); + if( (pc.plan.wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ + pc.plan.wsFlags |= WHERE_UNIQUE; + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + pc.plan.wsFlags |= WHERE_ALL_UNIQUE; + } } }else if( pProbe->bUnordered==0 ){ - int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); + int j; + j = (pc.plan.nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[pc.plan.nEq]); + if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ + WhereTerm *pTop, *pBtm; + pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx); + pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx); + whereRangeScanEst(pParse, pProbe, pc.plan.nEq, pBtm, pTop, &rangeDiv); if( pTop ){ nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; + pc.plan.wsFlags |= WHERE_TOP_LIMIT; + pc.used |= pTop->prereqRight; testcase( pTop->pWC!=pWC ); } if( pBtm ){ nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; + pc.plan.wsFlags |= WHERE_BTM_LIMIT; + pc.used |= pBtm->prereqRight; testcase( pBtm->pWC!=pWC ); } - wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); + pc.plan.wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); } } /* If there is an ORDER BY clause and the index being considered will ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ - if( isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) - ){ - bSort = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); + ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but + ** the index will scan rows in a different order, set the bSort + ** variable. */ + if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ + int bRev = 2; + int bObUnique = 0; + WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat)); + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); + WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", + bRev, bObUnique, pc.plan.nOBSat)); + if( nPriorSatpDistinct, pc.plan.nEq) + && (pc.plan.wsFlags & WHERE_COLUMN_IN)==0 ){ bDist = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; + pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK ** index), determine if all required column data may be obtained without ** using the main table (i.e. if the index is a covering ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. */ - if( pIdx && wsFlags ){ + ** pc.plan.wsFlags. Otherwise, set the bLookup variable to true. */ + if( pIdx ){ Bitmask m = pSrc->colUsed; int j; for(j=0; jnColumn; j++){ @@ -104788,7 +106382,7 @@ static void bestBtreeIndex( } } if( m==0 ){ - wsFlags |= WHERE_IDX_ONLY; + pc.plan.wsFlags |= WHERE_IDX_ONLY; }else{ bLookup = 1; } @@ -104798,10 +106392,10 @@ static void bestBtreeIndex( ** Estimate the number of rows of output. For an "x IN (SELECT...)" ** constraint, do not let the estimate exceed half the rows in the table. */ - nRow = (double)(aiRowEst[nEq] * nInMul); - if( bInEst && nRow*2>aiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); + pc.plan.nRow = (double)(aiRowEst[pc.plan.nEq] * nInMul); + if( bInEst && pc.plan.nRow*2>aiRowEst[0] ){ + pc.plan.nRow = aiRowEst[0]/2; + nInMul = (int)(pc.plan.nRow / aiRowEst[pc.plan.nEq]); } #ifdef SQLITE_ENABLE_STAT3 @@ -104811,15 +106405,19 @@ static void bestBtreeIndex( ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ + if( pc.plan.nRow>(double)1 && pc.plan.nEq==1 + && pFirstTerm!=0 && aiRowEst[1]>1 ){ assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); + testcase( pFirstTerm->eOperator & WO_EQ ); + testcase( pFirstTerm->eOperator & WO_EQUIV ); + testcase( pFirstTerm->eOperator & WO_ISNULL ); + whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, + &pc.plan.nRow); }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator==WO_IN ); - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); + assert( pFirstTerm->eOperator & WO_IN ); + whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, + &pc.plan.nRow); } } #endif /* SQLITE_ENABLE_STAT3 */ @@ -104827,8 +106425,8 @@ static void bestBtreeIndex( /* Adjust the number of output rows and downward to reflect rows ** that are excluded by range constraints. */ - nRow = nRow/rangeDiv; - if( nRow<1 ) nRow = 1; + pc.plan.nRow = pc.plan.nRow/rangeDiv; + if( pc.plan.nRow<1 ) pc.plan.nRow = 1; /* Experiments run on real SQLite databases show that the time needed ** to do a binary search to locate a row in a table or index is roughly @@ -104843,7 +106441,19 @@ static void bestBtreeIndex( ** So this computation assumes table records are about twice as big ** as index records */ - if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE)) + ==WHERE_IDX_ONLY + && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) + ){ + /* This index is not useful for indexing, but it is a covering index. + ** A full-scan of the index might be a little faster than a full-scan + ** of the table, so give this case a cost slightly less than a table + ** scan. */ + pc.rCost = aiRowEst[0]*3 + pProbe->nColumn; + pc.plan.wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE; + }else if( (pc.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ /* The cost of a full table scan is a number of move operations equal ** to the number of rows in the table. ** @@ -104853,10 +106463,15 @@ static void bestBtreeIndex( ** decision and one which we expect to revisit in the future. But ** it seems to be working well enough at the moment. */ - cost = aiRowEst[0]*4; + pc.rCost = aiRowEst[0]*4; + pc.plan.wsFlags &= ~WHERE_IDX_ONLY; + if( pIdx ){ + pc.plan.wsFlags &= ~WHERE_ORDERED; + pc.plan.nOBSat = nPriorSat; + } }else{ log10N = estLog(aiRowEst[0]); - cost = nRow; + pc.rCost = pc.plan.nRow; if( pIdx ){ if( bLookup ){ /* For an index lookup followed by a table lookup: @@ -104864,20 +106479,20 @@ static void bestBtreeIndex( ** + nRow steps through the index ** + nRow table searches to lookup the table entry using the rowid */ - cost += (nInMul + nRow)*log10N; + pc.rCost += (nInMul + pc.plan.nRow)*log10N; }else{ /* For a covering index: ** nInMul index searches to find the initial entry ** + nRow steps through the index */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } }else{ /* For a rowid primary key lookup: ** nInMult table searches to find the initial entry for each range ** + nRow steps through the table */ - cost += nInMul*log10N; + pc.rCost += nInMul*log10N; } } @@ -104888,10 +106503,12 @@ static void bestBtreeIndex( ** difference and select C of 3.0. */ if( bSort ){ - cost += nRow*estLog(nRow)*3; + double m = estLog(pc.plan.nRow*(nOrderBy - pc.plan.nOBSat)/nOrderBy); + m *= (double)(pc.plan.nOBSat ? 2 : 3); + pc.rCost += pc.plan.nRow*m; } if( bDist ){ - cost += nRow*estLog(nRow)*3; + pc.rCost += pc.plan.nRow*estLog(pc.plan.nRow)*3; } /**** Cost of using this index has now been computed ****/ @@ -104912,25 +106529,25 @@ static void bestBtreeIndex( ** might be selected even when there exists an optimal index that has ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost ){ + if( pc.plan.nRow>2 && pc.rCost<=p->cost.rCost ){ int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ + int nSkipEq = pc.plan.nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ Bitmask thisTab; /* Bitmap for pSrc */ thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ + for(pTerm=pWC->a, k=pWC->nTerm; pc.plan.nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; + if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index + /* Ignore the first pc.plan.nEq equality matches since the index ** has already accounted for these */ nSkipEq--; }else{ /* Assume each additional equality match reduces the result ** set size by a factor of 10 */ - nRow /= 10; + pc.plan.nRow /= 10; } }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ if( nSkipRange ){ @@ -104944,37 +106561,33 @@ static void bestBtreeIndex( ** more selective intentionally because of the subjective ** observation that indexed range constraints really are more ** selective in practice, on average. */ - nRow /= 3; + pc.plan.nRow /= 3; } - }else if( pTerm->eOperator!=WO_NOOP ){ + }else if( (pTerm->eOperator & WO_NOOP)==0 ){ /* Any other expression lowers the output row count by half */ - nRow /= 2; + pc.plan.nRow /= 2; } } - if( nRow<2 ) nRow = 2; + if( pc.plan.nRow<2 ) pc.plan.nRow = 2; } WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - notReady, log10N, nRow, cost, used + " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" + " used=0x%llx nOBSat=%d\n", + pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags, + p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used, + pc.plan.nOBSat )); /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. + ** index and its cost in the p->cost structure. */ - if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) - ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = (wsFlags&wsFlagMask); - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pIdx; + if( (!pIdx || pc.plan.wsFlags) && compareCost(&pc, &p->cost) ){ + p->cost = pc; + p->cost.plan.wsFlags &= wsFlagMask; + p->cost.plan.u.pIdx = pIdx; } /* If there was an INDEXED BY clause, then only that one index is @@ -104989,27 +106602,26 @@ static void bestBtreeIndex( /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag ** is set, then reverse the order that the index will be scanned ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that + ** where application behavior depends on the (undefined) order that ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; + if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ + p->cost.plan.wsFlags |= WHERE_REVERSE; } - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); + assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERED)==0 ); + assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 ); assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex + || p->cost.plan.u.pIdx==0 + || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") - )); + WHERETRACE((" best index is %s cost=%.1f\n", + p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk", + p->cost.rCost)); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; + bestOrClauseIndex(p); + bestAutomaticIndex(p); + p->cost.plan.wsFlags |= eqTermMask; } /* @@ -105017,28 +106629,28 @@ static void bestBtreeIndex( ** best query plan and its cost into the WhereCost object supplied ** as the last parameter. This function may calculate the cost of ** both real and virtual table scans. +** +** This function does not take ORDER BY or DISTINCT into account. Nor +** does it remember the virtual table query plan. All it does is compute +** the cost while determining if an OR optimization is applicable. The +** details will be reconsidered later if the optimization is found to be +** applicable. */ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); - if( p->needToFreeIdxStr ){ - sqlite3_free(p->idxStr); + if( IsVirtual(p->pSrc->pTab) ){ + sqlite3_index_info *pIdxInfo = 0; + p->ppIdxInfo = &pIdxInfo; + bestVirtualIndex(p); + assert( pIdxInfo!=0 || p->pParse->db->mallocFailed ); + if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); } - sqlite3DbFree(pParse->db, p); + sqlite3DbFree(p->pParse->db, pIdxInfo); }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); + bestBtreeIndex(p); } } @@ -105137,7 +106749,8 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ static int codeEqualityTerm( Parse *pParse, /* The parsing context */ WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; @@ -105155,12 +106768,26 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; + u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 + && pLevel->plan.u.pIdx->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 ); + testcase( iEq>0 && iEq+1plan.u.pIdx->nColumn ); + testcase( bRev ); + bRev = !bRev; + } assert( pX->op==TK_IN ); iReg = iTarget; eType = sqlite3FindInIndex(pParse, pX, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); @@ -105178,6 +106805,7 @@ static int codeEqualityTerm( }else{ pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; sqlite3VdbeAddOp1(v, OP_IsNull, iReg); }else{ pLevel->u.in.nIn = 0; @@ -105272,7 +106900,7 @@ static int codeAllEqualityTerms( ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); @@ -105516,12 +107144,23 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "init LEFT JOIN no-match flag")); } + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield); + pLevel->p2 = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName)); + sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk); + pLevel->op = OP_Goto; + }else + #ifndef SQLITE_OMIT_VIRTUALTABLE if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ /* Case 0: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; int nConstraint = pVtabIdx->nConstraint; struct sqlite3_index_constraint_usage *aUsage = @@ -105531,11 +107170,18 @@ static Bitmask codeOneLoopStart( sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; for(j=1; j<=nConstraint; j++){ for(k=0; ka[iTerm].pExpr->pRight, iReg+j+1); + int iTarget = iReg+j+1; + pTerm = &pWC->a[aConstraint[k].iTermOffset]; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, k, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } break; } } @@ -105543,7 +107189,7 @@ static Bitmask codeOneLoopStart( } sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pVtabIdx->idxStr, pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); pVtabIdx->needToFreeIdxStr = 0; for(j=0; jpExpr!=0 ); - assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); pLevel->op = OP_Noop; @@ -105734,7 +107380,7 @@ static Bitmask codeOneLoopStart( ** this requires some special handling. */ if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) + && (pLevel->plan.wsFlags&WHERE_ORDERED) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ @@ -105897,6 +107543,11 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; + if( pLevel->plan.wsFlags & WHERE_COVER_SCAN ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION @@ -105956,7 +107607,7 @@ static Bitmask codeOneLoopStart( pTerm = pLevel->plan.u.pTerm; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; @@ -106029,7 +107680,7 @@ static Bitmask codeOneLoopStart( for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; if( pAndExpr ){ @@ -106291,42 +107942,46 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** ORDER BY CLAUSE PROCESSING ** -** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, +** pOrderBy is a pointer to the ORDER BY clause of a SELECT statement, ** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. ** ** If an index can be used so that the natural output order of the table ** scan is correct for the ORDER BY clause, then that index is used and -** *ppOrderBy is set to NULL. This is an optimization that prevents an -** unnecessary sort of the result set if an index appropriate for the -** ORDER BY clause already exists. +** the returned WhereInfo.nOBSat field is set to pOrderBy->nExpr. This +** is an optimization that prevents an unnecessary sort of the result set +** if an index appropriate for the ORDER BY clause already exists. ** ** If the where clause loops cannot be arranged to provide the correct -** output order, then the *ppOrderBy is unchanged. +** output order, then WhereInfo.nOBSat is 0. */ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ - int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ + WhereBestIdx sWBI; /* Best index search context */ WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct SrcList_item *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in the pWInfo list */ - int iFrom; /* First unused FROM clause element */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + int iFrom; /* First unused FROM clause element */ int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ + + /* Variable initialization */ + memset(&sWBI, 0, sizeof(sWBI)); + sWBI.pParse = pParse; + /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ @@ -106366,22 +108021,23 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; + pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; + sWBI.aLevel = pWInfo->a; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ) pDistinct = 0; /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(sWBI.pWC, pParse, pMaskSet, wctrlFlags); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(sWBI.pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -106402,30 +108058,19 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; inSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); - } -#endif + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); } #ifndef NDEBUG { Bitmask toTheLeft = 0; - for(i=0; inSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); assert( (m-1)==toTheLeft ); toTheLeft |= m; } @@ -106437,7 +108082,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, pWC); + exprAnalyzeAll(pTabList, sWBI.pWC); if( db->mallocFailed ){ goto whereBeginError; } @@ -106446,7 +108091,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse, pTabList, sWBI.pWC, pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -106466,22 +108111,26 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** This loop also figures out the nesting order of tables in the FROM ** clause. */ - notReady = ~(Bitmask)0; + sWBI.notValid = ~(Bitmask)0; + sWBI.pOrderBy = pOrderBy; + sWBI.n = nTabList; + sWBI.pDistinct = pDistinct; andFlags = ~0; WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; ia; sWBI.i=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); - if( (m & notReady)==0 ){ + + /* The optimal scan check only occurs if there are two or more tables + ** available to be reordered */ + if( iFrom==nTabList-1 ){ + ckOptimal = 0; /* Common case of just one table in the FROM clause */ + }else{ + ckOptimal = -1; + for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; jiCursor); + if( (m & sWBI.notValid)==0 ){ if( j==iFrom ) iFrom++; continue; } - mask = (isOptimal ? m : notReady); - pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); - pDist = (i==0 ? pDistinct : 0); - if( pTabItem->pIndex==0 ) nUnconstrained++; + if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ) break; + if( ++ckOptimal ) break; + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; + } + } + assert( ckOptimal==0 || ckOptimal==1 ); + + for(isOptimal=ckOptimal; isOptimal>=0 && bestJ<0; isOptimal--){ + for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; jiFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ){ + /* This break and one like it in the ckOptimal computation loop + ** above prevent table reordering across LEFT and CROSS JOINs. + ** The LEFT JOIN case is necessary for correctness. The prohibition + ** against reordering across a CROSS JOIN is an SQLite feature that + ** allows the developer to control table reordering */ + break; + } + m = getMask(pMaskSet, sWBI.pSrc->iCursor); + if( (m & sWBI.notValid)==0 ){ + assert( j>iFrom ); + continue; + } + sWBI.notReady = (isOptimal ? m : sWBI.notValid); + if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); - assert( pTabItem->pTab ); + WHERETRACE((" === trying table %d (%s) with isOptimal=%d ===\n", + j, sWBI.pSrc->pTab->zName, isOptimal)); + assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost, pp); + if( IsVirtual(sWBI.pSrc->pTab) ){ + sWBI.ppIdxInfo = &pWInfo->a[j].pIdxInfo; + bestVirtualIndex(&sWBI); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - pDist, &sCost); + bestBtreeIndex(&sWBI); } - assert( isOptimal || (sCost.used¬Ready)==0 ); + assert( isOptimal || (sWBI.cost.used&sWBI.notValid)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); + assert( sWBI.pSrc->pIndex==0 + || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sWBI.cost.plan.u.pIdx==sWBI.pSrc->pIndex ); - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + if( isOptimal && (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ notIndexed |= m; } + if( isOptimal ){ + pWInfo->a[j].rOptCost = sWBI.cost.rCost; + }else if( ckOptimal ){ + /* If two or more tables have nearly the same outer loop cost, but + ** very different inner loop (optimal) cost, we want to choose + ** for the outer loop that table which benefits the least from + ** being in the inner loop. The following code scales the + ** outer loop cost estimate to accomplish that. */ + WHERETRACE((" scaling cost from %.1f to %.1f\n", + sWBI.cost.rCost, + sWBI.cost.rCost/pWInfo->a[j].rOptCost)); + sWBI.cost.rCost /= pWInfo->a[j].rOptCost; + } /* Conditions under which this table becomes the best so far: ** ** (1) The table must not depend on other tables that have not - ** yet run. + ** yet run. (In other words, it must not depend on tables + ** in inner loops.) ** - ** (2) A full-table-scan plan cannot supercede indexed plan unless - ** the full-table-scan is an "optimal" plan as defined above. + ** (2) (This rule was removed on 2012-11-09. The scaling of the + ** cost using the optimal scan cost made this rule obsolete.) ** ** (3) All tables have an INDEXED BY clause or this table lacks an ** INDEXED BY clause or this table uses the specific @@ -106591,43 +108268,47 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** The NEVER() comes about because rule (2) above prevents ** An indexable full-table-scan from reaching rule (3). ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. + ** (4) The plan cost must be lower than prior plans, where "cost" + ** is defined by the compareCost() function above. */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ - || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCostpIndex==0 /* (3) */ + || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || compareCost(&sWBI.cost, &bestPlan)) /* (4) */ ){ - WHERETRACE(("=== table %d is best so far" - " with cost=%g and nRow=%g\n", - j, sCost.rCost, sCost.plan.nRow)); - bestPlan = sCost; + WHERETRACE((" === table %d (%s) is best so far\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=%08x\n", + j, sWBI.pSrc->pTab->zName, + sWBI.cost.rCost, sWBI.cost.plan.nRow, + sWBI.cost.plan.nOBSat, sWBI.cost.plan.wsFlags)); + bestPlan = sWBI.cost; bestJ = j; } - if( doNotReorder ) break; + + /* In a join like "w JOIN x LEFT JOIN y JOIN z" make sure that + ** table y (and not table z) is always the next inner loop inside + ** of table x. */ + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; } } assert( bestJ>=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - /* The ALWAYS() that follows was added to hush up clang scan-build */ - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 && ALWAYS(ppOrderBy) ){ - *ppOrderBy = 0; - } + assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); + assert( bestJ==iFrom || (pTabList->a[iFrom].jointype & JT_LEFT)==0 ); + testcase( bestJ>iFrom && (pTabList->a[iFrom].jointype & JT_CROSS)!=0 ); + testcase( bestJ>iFrom && bestJa[bestJ+1].jointype & JT_LEFT)!=0 ); + WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", + bestJ, pTabList->a[bestJ].pTab->zName, + pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, + bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ assert( pWInfo->eDistinct==0 ); pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; + pLevel->iTabCur = pTabList->a[bestJ].iCursor; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ @@ -106641,7 +108322,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( }else{ pLevel->iIdxCur = -1; } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); + sWBI.notValid &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); pLevel->iFrom = (u8)bestJ; if( bestPlan.plan.nRow>=(double)1 ){ pParse->nQueryLoop *= bestPlan.plan.nRow; @@ -106669,12 +108350,19 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pParse->nErr || db->mallocFailed ){ goto whereBeginError; } + if( nTabList ){ + pLevel--; + pWInfo->nOBSat = pLevel->plan.nOBSat; + }else{ + pWInfo->nOBSat = 0; + } /* If the total query only selects a single row, then the ORDER BY ** clause is irrelevant. */ - if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ - *ppOrderBy = 0; + if( (andFlags & WHERE_UNIQUE)!=0 && pOrderBy ){ + assert( nTabList==0 || (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ); + pWInfo->nOBSat = pOrderBy->nExpr; } /* If the caller is an UPDATE or DELETE statement that is requesting @@ -106694,13 +108382,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; pWInfo->nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; ia; iia[pLevel->iFrom]; pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ @@ -106711,6 +108399,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); int iCur = pTabItem->iCursor; sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ }else #endif if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 @@ -106732,7 +108422,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); + constructAutomaticIndex(pParse, sWBI.pWC, pTabItem, notReady, pLevel); }else #endif if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ @@ -106746,7 +108436,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( VdbeComment((v, "%s", pIx->zName)); } sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(sWBI.pWC->pMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -106756,10 +108446,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** program. */ notReady = ~(Bitmask)0; - for(i=0; ia[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + for(ii=0; iia[ii]; + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + notReady = codeOneLoopStart(pWInfo, ii, wctrlFlags, notReady); pWInfo->iContinue = pLevel->addrCont; } @@ -106770,16 +108460,20 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** the index is listed as "{}". If the primary key is used the ** index name is '*'. */ - for(i=0; ia[i]; + int w; + struct SrcList_item *pTabItem; + + pLevel = &pWInfo->a[ii]; + w = pLevel->plan.wsFlags; pTabItem = &pTabList->a[pLevel->iFrom]; z = pTabItem->zAlias; if( z==0 ) z = pTabItem->pTab->zName; n = sqlite3Strlen30(z); if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ - if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ + if( (w & WHERE_IDX_ONLY)!=0 && (w & WHERE_COVER_SCAN)==0 ){ memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); nQPlan += 2; }else{ @@ -106788,12 +108482,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } sqlite3_query_plan[nQPlan++] = ' '; } - testcase( pLevel->plan.wsFlags & WHERE_ROWID_EQ ); - testcase( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ); - if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + testcase( w & WHERE_ROWID_EQ ); + testcase( w & WHERE_ROWID_RANGE ); + if( w & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); nQPlan += 2; - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + }else if( (w & WHERE_INDEXED)!=0 && (w & WHERE_COVER_SCAN)==0 ){ n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); @@ -106854,7 +108548,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); @@ -107162,6 +108856,7 @@ typedef union { IdList* yy180; struct {int value; int mask;} yy207; u8 yy258; + u16 yy305; struct LikeOp yy318; TriggerStep* yy327; ExprSpan yy342; @@ -109112,8 +110807,6 @@ static void yy_reduce( case 86: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==86); case 98: /* defer_subclause_opt ::= */ yytestcase(yyruleno==98); case 109: /* ifexists ::= */ yytestcase(yyruleno==109); - case 120: /* distinct ::= ALL */ yytestcase(yyruleno==120); - case 121: /* distinct ::= */ yytestcase(yyruleno==121); case 221: /* between_op ::= BETWEEN */ yytestcase(yyruleno==221); case 224: /* in_op ::= IN */ yytestcase(yyruleno==224); {yygotominor.yy392 = 0;} @@ -109123,7 +110816,6 @@ static void yy_reduce( case 70: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==70); case 85: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==85); case 108: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==108); - case 119: /* distinct ::= DISTINCT */ yytestcase(yyruleno==119); case 222: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==222); case 225: /* in_op ::= NOT IN */ yytestcase(yyruleno==225); {yygotominor.yy392 = 1;} @@ -109363,9 +111055,16 @@ static void yy_reduce( break; case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yygotominor.yy159 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy442,yymsp[-5].minor.yy347,yymsp[-4].minor.yy122,yymsp[-3].minor.yy442,yymsp[-2].minor.yy122,yymsp[-1].minor.yy442,yymsp[-7].minor.yy392,yymsp[0].minor.yy64.pLimit,yymsp[0].minor.yy64.pOffset); + yygotominor.yy159 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy442,yymsp[-5].minor.yy347,yymsp[-4].minor.yy122,yymsp[-3].minor.yy442,yymsp[-2].minor.yy122,yymsp[-1].minor.yy442,yymsp[-7].minor.yy305,yymsp[0].minor.yy64.pLimit,yymsp[0].minor.yy64.pOffset); } break; + case 119: /* distinct ::= DISTINCT */ +{yygotominor.yy305 = SF_Distinct;} + break; + case 120: /* distinct ::= ALL */ + case 121: /* distinct ::= */ yytestcase(yyruleno==121); +{yygotominor.yy305 = 0;} + break; case 122: /* sclp ::= selcollist COMMA */ case 246: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==246); {yygotominor.yy442 = yymsp[-1].minor.yy442;} @@ -109434,10 +111133,20 @@ static void yy_reduce( { if( yymsp[-6].minor.yy347==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy122==0 && yymsp[0].minor.yy180==0 ){ yygotominor.yy347 = yymsp[-4].minor.yy347; + }else if( yymsp[-4].minor.yy347->nSrc==1 ){ + yygotominor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + if( yygotominor.yy347 ){ + struct SrcList_item *pNew = &yygotominor.yy347->a[yygotominor.yy347->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy347->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pOld->zName = pOld->zDatabase = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy347); }else{ Select *pSubquery; sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy347); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy347,0,0,0,0,0,0,0); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy347,0,0,0,0,SF_NestedFrom,0,0); yygotominor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); } } @@ -109652,7 +111361,7 @@ static void yy_reduce( break; case 194: /* expr ::= expr COLLATE ids */ { - yygotominor.yy342.pExpr = sqlite3ExprSetCollByToken(pParse, yymsp[-2].minor.yy342.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy342.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy342.pExpr, &yymsp[0].minor.yy0); yygotominor.yy342.zStart = yymsp[-2].minor.yy342.zStart; yygotominor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } @@ -109670,7 +111379,7 @@ static void yy_reduce( } yygotominor.yy342.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy442, &yymsp[-4].minor.yy0); spanSet(&yygotominor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy392 && yygotominor.yy342.pExpr ){ + if( yymsp[-2].minor.yy305 && yygotominor.yy342.pExpr ){ yygotominor.yy342.pExpr->flags |= EP_Distinct; } } @@ -109911,11 +111620,7 @@ static void yy_reduce( break; case 247: /* idxlist ::= idxlist COMMA nm collate sortorder */ { - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, p); sqlite3ExprListSetName(pParse,yygotominor.yy442,&yymsp[-2].minor.yy0,1); sqlite3ExprListCheckLength(pParse, yygotominor.yy442, "index"); @@ -109924,11 +111629,7 @@ static void yy_reduce( break; case 248: /* idxlist ::= nm collate sortorder */ { - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0); yygotominor.yy442 = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, yygotominor.yy442, &yymsp[-2].minor.yy0, 1); sqlite3ExprListCheckLength(pParse, yygotominor.yy442, "index"); @@ -110177,6 +111878,7 @@ static void yy_reduce( /* (326) anylist ::= anylist ANY */ yytestcase(yyruleno==326); break; }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; @@ -111742,6 +113444,13 @@ SQLITE_API int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, @@ -112085,6 +113794,20 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + default: { rc = SQLITE_ERROR; break; @@ -112424,6 +114147,13 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_BUSY; } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + /* Convert the connection into a zombie and then close it. */ db->magic = SQLITE_MAGIC_ZOMBIE; @@ -112467,7 +114197,7 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* If we reach this point, it means that the database connection has ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** pased to sqlite3_close (meaning that it is a zombie). Therefore, + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ @@ -112584,7 +114314,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( db->flags&SQLITE_InternChanges ){ + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } @@ -112726,6 +114456,7 @@ SQLITE_API int sqlite3_busy_handler( db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; db->busyHandler.nBusy = 0; + db->busyTimeout = 0; sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -112763,8 +114494,8 @@ SQLITE_API void sqlite3_progress_handler( */ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ if( ms>0 ){ - db->busyTimeout = ms; sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; }else{ sqlite3_busy_handler(db, 0, 0); } @@ -113377,6 +115108,15 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ return db->errCode; } +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -114046,6 +115786,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif return sqlite3ApiExit(0, rc); } @@ -114351,7 +116098,7 @@ SQLITE_API int sqlite3_table_column_metadata( zDataType = pCol->zType; zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; - primarykey = pCol->isPrimKey!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; }else{ zDataType = "INTEGER"; @@ -114614,8 +116361,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } @@ -115503,7 +117249,7 @@ struct sqlite3_tokenizer_module { ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialised by + ** sqlite3_tokenizer.pModule variable should not be initialized by ** this callback. The caller will do so. */ int (*xCreate)( @@ -115608,7 +117354,7 @@ int fts3_term_cnt(int iTerm, int iCol); ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** @@ -117515,7 +119261,7 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ }else{ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occured, then the %_content + /* If no row was found and no error has occurred, then the %_content ** table is missing a row that is present in the full-text index. ** The data structures are corrupt. */ rc = FTS_CORRUPT_VTAB; @@ -118755,7 +120501,7 @@ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ } /* -** This function retreives the doclist for the specified term (or term +** This function retrieves the doclist for the specified term (or term ** prefix) from the database. */ static int fts3TermSelect( @@ -119506,7 +121252,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const #endif /* -** Initialise the fts3 extension. If this extension is built as part +** Initialize the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. @@ -119540,7 +121286,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); - /* Allocate and initialise the hash-table used to store tokenizers. */ + /* Allocate and initialize the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; @@ -120687,35 +122433,39 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; + if( nTmp==0 ){ res = 0; }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } } + + sqlite3_free(aTmp); } - - sqlite3_free(aTmp); } return res; @@ -121135,7 +122885,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( ** of the current row. ** ** More specifically, the returned buffer contains 1 varint for each -** occurence of the phrase in the column, stored using the normal (delta+2) +** occurrence of the phrase in the column, stored using the normal (delta+2) ** compression and is terminated by either an 0x01 or 0x00 byte. For example, ** if the requested column contains "a b X c d X X" and the position-list ** for 'X' is requested, the buffer returned may contain: @@ -121874,7 +123624,7 @@ struct ParseContext { ** This function is equivalent to the standard isspace() function. ** ** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behaviour when passed +** is defined to accept an argument of type int, its behavior when passed ** an integer that falls outside of the range of the unsigned char type ** is undefined (and sometimes, "undefined" means segfault). This wrapper ** is defined to accept an argument of type char, and always returns 0 for @@ -121953,7 +123703,7 @@ static int getNextToken( rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; - int nToken, iStart, iEnd, iPosition; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; int nByte; /* total space to allocate */ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); @@ -122068,7 +123818,7 @@ static int getNextString( int ii; for(ii=0; rc==SQLITE_OK; ii++){ const char *zByte; - int nByte, iBegin, iEnd, iPos; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ Fts3PhraseToken *pToken; @@ -124065,10 +125815,10 @@ static void testFunc( const char *azArg[64]; const char *zToken; - int nToken; - int iStart; - int iEnd; - int iPos; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; int i; Tcl_Obj *pRet; @@ -124242,7 +125992,7 @@ static void intTestFunc( /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must -** been initialised to use string keys, and to take a private copy +** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); @@ -125315,16 +127065,16 @@ static int fts3PendingTermsAdd( int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ ){ int rc; - int iStart; - int iEnd; - int iPos; + int iStart = 0; + int iEnd = 0; + int iPos = 0; int nWord = 0; char const *zToken; - int nToken; + int nToken = 0; sqlite3_tokenizer *pTokenizer = p->pTokenizer; sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; @@ -125379,7 +127129,7 @@ static int fts3PendingTermsAdd( } pModule->xClose(pCsr); - *pnWord = nWord; + *pnWord += nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } @@ -125583,11 +127333,13 @@ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz /* Sizes of deleted document written here */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ ){ int rc; sqlite3_stmt *pSelect; + assert( *pbFound==0 ); if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ @@ -125605,6 +127357,7 @@ static void fts3DeleteTerms( *pRC = rc; return; } + *pbFound = 1; } rc = sqlite3_reset(pSelect); }else{ @@ -126018,6 +127771,7 @@ static int fts3SegReaderNextDocid( *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } + /* List may have been edited in place by fts3EvalNearTrim() */ while( papSegment, nMerge, j, xCmp); + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + if( pMsr->iColFilter>=0 ){ - fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); } if( nList>0 ){ - if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); - if( rc!=SQLITE_OK ) return rc; - *paPoslist = pMsr->aBuffer; - assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); - }else{ - *paPoslist = pList; - } + *paPoslist = pList; *piDocid = iDocid; *pnPoslist = nList; break; @@ -127392,7 +129153,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } if( isColFilter ){ - fts3ColumnFilter(pFilter->iCol, &pList, &nList); + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ @@ -127829,7 +129590,7 @@ static int fts3DoRebuild(Fts3Table *p){ int iCol; int iLangid = langidFromSelect(p, pStmt); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); - aSz[p->nColumn] = 0; + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); @@ -129473,9 +131234,9 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ - int nToken; /* Number of bytes in token */ - int iDum1, iDum2; /* Dummy variables */ - int iPos; /* Position of token in zText */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); if( rc==SQLITE_OK ){ @@ -129642,9 +131403,9 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ - int nToken; /* Number of bytes in token */ - int iDum1, iDum2; /* Dummy variables */ - int iPos; /* Position of token in zText */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ @@ -129733,28 +131494,32 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( static int fts3DeleteByRowid( Fts3Table *p, sqlite3_value *pRowid, - int *pnDoc, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ - int isEmpty = 0; - int rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p, 1); - *pnDoc = *pnDoc - 1; - }else{ - fts3DeleteTerms(&rc, p, pRowid, aSzDel); - if( p->zContentTbl==0 ){ - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); }else{ - *pnDoc = *pnDoc - 1; - } - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } } } } @@ -129785,7 +131550,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ - u32 *aSzDel; /* Sizes of deleted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; @@ -129813,13 +131578,13 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ){ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; } - aSzDel = &aSzIns[p->nColumn+1]; - memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); /* If this is an INSERT operation, or an UPDATE that modifies the rowid ** value, then this operation requires constraint handling. @@ -129904,7 +131669,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } update_out: - sqlite3_free(aSzIns); + sqlite3_free(aSzDel); sqlite3Fts3SegmentsClose(p); return rc; } @@ -130325,9 +132090,9 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ ** is the snippet with the highest score, where scores are calculated ** by adding: ** -** (a) +1 point for each occurence of a matchable phrase in the snippet. +** (a) +1 point for each occurrence of a matchable phrase in the snippet. ** -** (b) +1000 points for the first occurence of each matchable phrase in +** (b) +1000 points for the first occurrence of each matchable phrase in ** the snippet for which the corresponding mCovered bit is not set. ** ** The selected snippet parameters are stored in structure *pFragment before @@ -130512,7 +132277,7 @@ static int fts3SnippetShift( return rc; } while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ - const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); } pMod->xClose(pC); @@ -130556,8 +132321,6 @@ static int fts3SnippetText( int iCol = pFragment->iCol+1; /* Query column to extract text from */ sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ - const char *ZDUMMY; /* Dummy argument used with tokenizer */ - int DUMMY1; /* Dummy argument used with tokenizer */ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); if( zDoc==0 ){ @@ -130576,10 +132339,23 @@ static int fts3SnippetText( } while( rc==SQLITE_OK ){ - int iBegin; /* Offset in zDoc of start of token */ - int iFin; /* Offset in zDoc of end of token */ - int isHighlight; /* True for highlighted terms */ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ @@ -131269,8 +133045,6 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; - const char *ZDUMMY; /* Dummy argument used with xNext() */ - int NDUMMY; /* Dummy argument used with xNext() */ int rc; /* Return Code */ int nToken; /* Number of tokens in query */ int iCol; /* Column currently being processed */ @@ -131303,9 +133077,11 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( */ for(iCol=0; iColnColumn; iCol++){ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ - int iStart; - int iEnd; - int iCurrent; + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; const char *zDoc; int nDoc; @@ -131571,7 +133347,7 @@ static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ ** ** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() ** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. -** It is not possible to change the behaviour of the tokenizer with respect +** It is not possible to change the behavior of the tokenizer with respect ** to these codepoints. */ static int unicodeAddExceptions( @@ -134869,12 +136645,12 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ */ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ int rc; /* Return code */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ int iCell; /* Index of iDelete cell in pLeaf */ RtreeNode *pRoot; /* Root node of rtree structure */ - /* Obtain a reference to the root node to initialise Rtree.iDepth */ + /* Obtain a reference to the root node to initialize Rtree.iDepth */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); /* Obtain a reference to the leaf node that contains the entry @@ -135072,7 +136848,7 @@ static int rtreeUpdate( */ if( rc==SQLITE_OK && nData>1 ){ /* Insert the new record into the r-tree */ - RtreeNode *pLeaf; + RtreeNode *pLeaf = 0; /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ @@ -135258,7 +137034,8 @@ static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ static int getNodeSize( sqlite3 *db, /* Database handle */ Rtree *pRtree, /* Rtree handle */ - int isCreate /* True for xCreate, false for xConnect */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ ){ int rc; char *zSql; @@ -135271,6 +137048,8 @@ static int getNodeSize( if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } }else{ zSql = sqlite3_mprintf( @@ -135278,6 +137057,9 @@ static int getNodeSize( pRtree->zDb, pRtree->zName ); rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } } sqlite3_free(zSql); @@ -135341,7 +137123,7 @@ static int rtreeInit( memcpy(pRtree->zName, argv[2], nName); /* Figure out the node size to use. */ - rc = getNodeSize(db, pRtree, isCreate); + rc = getNodeSize(db, pRtree, isCreate, pzErr); /* Create/Connect to the underlying relational database schema. If ** that is successful, call sqlite3_declare_vtab() to configure @@ -136188,7 +137970,7 @@ static int icuOpen( nChar = nInput+1; pCsr = (IcuCursor *)sqlite3_malloc( sizeof(IcuCursor) + /* IcuCursor */ - nChar * sizeof(UChar) + /* IcuCursor.aChar[] */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ ); if( !pCsr ){ @@ -136196,7 +137978,7 @@ static int icuOpen( } memset(pCsr, 0, sizeof(IcuCursor)); pCsr->aChar = (UChar *)&pCsr[1]; - pCsr->aOffset = (int *)&pCsr->aChar[nChar]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; pCsr->aOffset[iOut] = iInput; U8_NEXT(zInput, iInput, nInput, c); diff --git a/code/game/sqlite3.h b/code/game/sqlite3.h index 5a1f9d4..69b4586 100644 --- a/code/game/sqlite3.h +++ b/code/game/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.14.1" -#define SQLITE_VERSION_NUMBER 3007014 -#define SQLITE_SOURCE_ID "2012-10-04 19:37:12 091570e46d04e84b67228e0bdbcd6e1fb60c6bdb" +#define SQLITE_VERSION "3.7.16.2" +#define SQLITE_VERSION_NUMBER 3007016 +#define SQLITE_SOURCE_ID "2013-04-12 11:52:43 cbea02d93865ce0e06789db95fd9168ebac970c7" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -288,7 +288,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated ** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has +** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or ** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], @@ -474,14 +474,26 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -855,6 +867,26 @@ struct sqlite3_io_methods { ** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] ** file control occurs at the beginning of pragma statement analysis and so ** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^This file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connections busy-handler callback. The argument is of type (void **) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connections +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Application can invoke this file-control to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -871,6 +903,8 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 /* ** CAPI3REF: Mutex Handle @@ -1567,11 +1601,39 @@ struct sqlite3_mem_methods { ** disabled. The default value may be changed by compiling with the ** [SQLITE_USE_URI] symbol defined. ** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    This option takes a single integer argument which is interpreted as +** a boolean in order to enable or disable the use of covering indices for +** full table scans in the query optimizer. The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** malfunction when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** ** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. ** +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1592,6 +1654,8 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_URI 17 /* int */ #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2600,7 +2664,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an error)^. ** ^If "ro" is specified, then the database is opened for read-only ** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the -** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** third argument to sqlite3_open_v2(). ^If the mode option is set to ** "rw", then the database is opened for read-write (but not create) ** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had ** been set. ^Value "rwc" is equivalent to setting both @@ -2616,7 +2680,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** ** @@ -2752,6 +2816,11 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -2770,6 +2839,7 @@ SQLITE_API int sqlite3_errcode(sqlite3 *db); SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); /* ** CAPI3REF: SQL Statement Object @@ -3928,7 +3998,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4008,14 +4079,17 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -4732,6 +4806,9 @@ SQLITE_API void *sqlite3_update_hook( ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** ** See Also: [SQLite Shared-Cache Mode] */ SQLITE_API int sqlite3_enable_shared_cache(int); @@ -6306,7 +6383,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** **
  • -**
    createFlag Behaviour when page is not already in cache +**
    createFlag Behavior when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL.