diff --git a/browser/base/content/test/browser_tabMatchesInAwesomebar.js b/browser/base/content/test/browser_tabMatchesInAwesomebar.js index 337cefce63b..c679916898a 100644 --- a/browser/base/content/test/browser_tabMatchesInAwesomebar.js +++ b/browser/base/content/test/browser_tabMatchesInAwesomebar.js @@ -221,10 +221,9 @@ function ensure_opentabs_match_db() { try { var stmt = db.createStatement( - "SELECT t.url, open_count, IFNULL(p_t.id, p.id) " + + "SELECT t.url, open_count, p.id " + "FROM moz_openpages_temp t " + - "LEFT JOIN moz_places p ON p.url = t.url " + - "LEFT JOIN moz_places_temp p_t ON p_t.url = t.url"); + "LEFT JOIN moz_places p ON p.url = t.url "); } catch (e) { ok(false, "error creating db statement: " + e); return; diff --git a/browser/components/nsBrowserGlue.js b/browser/components/nsBrowserGlue.js index ee4cb6995bc..90759311b37 100644 --- a/browser/components/nsBrowserGlue.js +++ b/browser/components/nsBrowserGlue.js @@ -417,10 +417,7 @@ BrowserGlue.prototype = { } // Load the "more info" page for a locked places.sqlite - // This property is set earlier in the startup process: - // nsPlacesDBFlush loads after profile-after-change and initializes - // the history service, which sends out places-database-locked - // which sets this property. + // This property is set earlier by places-database-locked topic. if (this._isPlacesDatabaseLocked) { this._showPlacesLockedNotificationBox(); } diff --git a/browser/components/places/tests/unit/test_clearHistory_shutdown.js b/browser/components/places/tests/unit/test_clearHistory_shutdown.js index ad5359f9826..6ce1804886b 100644 --- a/browser/components/places/tests/unit/test_clearHistory_shutdown.js +++ b/browser/components/places/tests/unit/test_clearHistory_shutdown.js @@ -53,9 +53,7 @@ let EXPECTED_NOTIFICATIONS = [ "places-shutdown" , "places-will-close-connection" , "places-connection-closing" -, "places-sync-finished" , "places-expiration-finished" -, "places-sync-finished" , "places-connection-closed" ]; diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index b69ab55def6..95ef5a347c4 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -345,7 +345,6 @@ @BINPATH@/components/toolkitplaces.manifest @BINPATH@/components/nsLivemarkService.js @BINPATH@/components/nsTaggingService.js -@BINPATH@/components/nsPlacesDBFlush.js @BINPATH@/components/nsPlacesAutoComplete.manifest @BINPATH@/components/nsPlacesAutoComplete.js @BINPATH@/components/nsPlacesExpiration.js diff --git a/browser/installer/removed-files.in b/browser/installer/removed-files.in index edd016e2fc2..5732c73d884 100644 --- a/browser/installer/removed-files.in +++ b/browser/installer/removed-files.in @@ -1089,3 +1089,4 @@ extensions/inspector@mozilla.org/components/@DLL_PREFIX@inspector@DLL_SUFFIX@ extensions/inspector@mozilla.org/chrome/icons/default/winInspectorMain.ico components/nsPlacesTransactionsService.js components/browserplaces.xpt +components/nsPlacesDBFlush.js diff --git a/configure.in b/configure.in index eee361dd56c..aa99ad59c60 100644 --- a/configure.in +++ b/configure.in @@ -130,7 +130,7 @@ GCONF_VERSION=1.2.1 GIO_VERSION=2.0 STARTUP_NOTIFICATION_VERSION=0.8 DBUS_VERSION=0.60 -SQLITE_VERSION=3.7.1 +SQLITE_VERSION=3.7.3 LIBNOTIFY_VERSION=0.4 MSMANIFEST_TOOL= diff --git a/db/sqlite3/README.MOZILLA b/db/sqlite3/README.MOZILLA index 633df7d9291..4ccdefab228 100644 --- a/db/sqlite3/README.MOZILLA +++ b/db/sqlite3/README.MOZILLA @@ -1,6 +1,6 @@ -This is sqlite 3.7.1 +This is sqlite 3.7.3 --- Shawn Wilsher , 08/2010 +-- Shawn Wilsher , 10/2010 See http://www.sqlite.org/ for more info. diff --git a/db/sqlite3/src/sqlite3.c b/db/sqlite3/src/sqlite3.c index be7e5f126db..6e9e7ea289f 100644 --- a/db/sqlite3/src/sqlite3.c +++ b/db/sqlite3/src/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.1. By combining all the individual C code files into this +** version 3.7.3. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a one translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -354,15 +354,21 @@ #endif /* -** The SQLITE_THREADSAFE macro must be defined as either 0 or 1. +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the libary is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** ** Older versions of SQLite used an optional THREADSAFE macro. -** We support that for legacy +** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) #if defined(THREADSAFE) # define SQLITE_THREADSAFE THREADSAFE #else -# define SQLITE_THREADSAFE 1 +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ #endif #endif @@ -644,9 +650,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.1" -#define SQLITE_VERSION_NUMBER 3007001 -#define SQLITE_SOURCE_ID "2010-08-21 16:01:46 3613b0695a5e990905ab146fadcab34dd04d5874" +#define SQLITE_VERSION "3.7.3" +#define SQLITE_VERSION_NUMBER 3007003 +#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1294,15 +1300,19 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** @@ -1313,7 +1323,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** ** +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application @@ -1343,10 +1354,11 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the @@ -1355,7 +1367,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that @@ -1365,13 +1377,13 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is @@ -1385,10 +1397,10 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current @@ -1785,7 +1797,7 @@ struct sqlite3_mem_methods { **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] -**
    • [sqlite3_soft_heap_limit()] +**
    • [sqlite3_soft_heap_limit64()] **
    • [sqlite3_status()] **
    )^ ** ^Memory allocation statistics are enabled by default unless SQLite is @@ -1799,15 +1811,14 @@ struct sqlite3_mem_methods { ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed. ** **
    SQLITE_CONFIG_PAGECACHE
    @@ -1827,8 +1838,7 @@ struct sqlite3_mem_methods { ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined. ** @@ -1957,8 +1967,14 @@ struct sqlite3_mem_methods { ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE] +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^ ** ** */ @@ -2263,6 +2279,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries ** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. @@ -2283,7 +2302,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** ** 
    @@ -2307,7 +2326,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
 **        azResult[5] = "28";
 **        azResult[6] = "Cindy";
 **        azResult[7] = "21";
-**
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 @@ -2315,19 +2334,19 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ @@ -2479,7 +2498,9 @@ SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in @@ -2737,17 +2758,28 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. @@ -2806,7 +2838,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2931,17 +2963,22 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a @@ -2970,7 +3007,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ @@ -2988,7 +3025,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -3001,8 +3040,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [GLOB] operators.)^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ @@ -3074,12 +3112,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
    • ** **
  • @@ -3092,11 +3125,16 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
    • ** **
  • -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
    • ** */ @@ -3163,7 +3201,7 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the @@ -3358,6 +3396,8 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); @@ -3548,8 +3588,14 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); @@ -3629,18 +3675,26 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object @@ -3685,10 +3739,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** **
      @@ -3701,22 +3755,22 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); **
    • The initial content is UTF-16 text and sqlite3_column_bytes() or ** sqlite3_column_text() is called. The content must be converted ** to UTF-8.
      • -**
    )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** **
      **
    • sqlite3_column_text() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_blob() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_text16() followed by sqlite3_column_bytes16()
      • -**
    )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result @@ -3754,17 +3808,26 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); @@ -3800,23 +3863,25 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or @@ -3826,10 +3891,10 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. @@ -3841,13 +3906,21 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is invoked when the function is deleted, either by being +** overloaded or when the database connection closes. +** ^When the destructure callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the pointer which was +** the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -3863,11 +3936,6 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not @@ -3894,6 +3962,17 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -4240,46 +4319,70 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. ** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. +** ^(The third argument (eTextRep) must be one of the constants: +**
      +**
    • [SQLITE_UTF8], +**
    • [SQLITE_UTF16LE], +**
    • [SQLITE_UTF16BE], +**
    • [SQLITE_UTF16], or +**
    • [SQLITE_UTF16_ALIGNED]. +**
    )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. ** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it any more). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. ** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ @@ -4287,14 +4390,14 @@ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); @@ -4302,7 +4405,7 @@ SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); @@ -4391,16 +4494,19 @@ SQLITE_API void sqlite3_activate_cerod( /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); @@ -4622,40 +4728,73 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. ** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. ** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. +** ^If the argument N is zero then the soft heap limit is disabled. ** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: ** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. +**
      +**
    • The soft heap limit is set to zero. +**
    • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
    • An alternative page cache implementation is specifed using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). +**
    • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
    )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. */ -SQLITE_API void sqlite3_soft_heap_limit(int); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table @@ -4779,34 +4918,47 @@ SQLITE_API int sqlite3_load_extension( SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions +** CAPI3REF: Automatically Load Statically Linked Extensions ** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. ** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: ** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +** 
    +**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); @@ -5445,7 +5597,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as @@ -5642,7 +5794,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status @@ -5661,7 +5814,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be @@ -5711,7 +5864,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to @@ -5734,7 +5887,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the @@ -5783,6 +5936,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. ** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5909,32 +6065,42 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. @@ -5942,47 +6108,50 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache @@ -5991,36 +6160,35 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** @@ -6498,6 +6666,62 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -7068,6 +7292,7 @@ typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; +typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; @@ -7174,12 +7399,11 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** NOTE: These values must match the corresponding PAGER_ values in ** pager.h. */ -#define BTREE_OMIT_JOURNAL 1 /* Do not use journal. No argument */ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ #define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ -#define BTREE_MEMORY 4 /* In-memory DB. No argument */ -#define BTREE_READONLY 8 /* Open the database in read-only mode */ -#define BTREE_READWRITE 16 /* Open for both reading and writing */ -#define BTREE_CREATE 32 /* Create the database if it does not exist */ +#define BTREE_MEMORY 4 /* This is an in-memory DB */ +#define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); @@ -7215,11 +7439,17 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); /* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR -** of the following flags: +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) */ #define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ -#define BTREE_ZERODATA 2 /* Table has keys only - no data */ -#define BTREE_LEAFDATA 4 /* Data stored in leaves only. Implies INTKEY */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); @@ -7840,6 +8070,7 @@ typedef struct PgHdr DbPage; */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ +#define PAGER_MEMORY 0x0004 /* In-memory database */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). @@ -8474,8 +8705,8 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); #define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK #define sqlite3_mutex_leave(X) -#define sqlite3_mutex_held(X) 1 -#define sqlite3_mutex_notheld(X) 1 +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() @@ -8797,6 +9028,27 @@ struct FuncDef { void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ +}; + +/* +** This structure encapsulates a user-function destructor callback (as +** configured using create_function_v2()) and a reference counter. When +** create_function_v2() is called to create a function with a destructor, +** a single object of this type is allocated. FuncDestructor.nRef is set to +** the number of FuncDef objects created (either 1 or 3, depending on whether +** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor +** member of each of the new FuncDef objects is set to point to the allocated +** FuncDestructor. +** +** Thereafter, when one of the FuncDef objects is deleted, the reference +** count on this object is decremented. When it reaches 0, the destructor +** is invoked and the FuncDestructor structure freed. +*/ +struct FuncDestructor { + int nRef; + void (*xDestroy)(void *); + void *pUserData; }; /* @@ -8837,15 +9089,15 @@ struct FuncDef { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - pArg, 0, xFunc, 0, 0, #zName, 0} + pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ - {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0} + {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0} + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at @@ -9065,6 +9317,7 @@ struct Table { Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ int tnum; /* Root BTree node for this table (see note above) */ + unsigned nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ u16 nRef; /* Number of pointers to this Table */ u8 tabFlags; /* Mask of TF_* values */ @@ -10319,7 +10572,6 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); ** Internal function prototypes */ SQLITE_PRIVATE int sqlite3StrICmp(const char *, const char *); -SQLITE_PRIVATE int sqlite3IsNumber(const char*, int*, u8); SQLITE_PRIVATE int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -10343,7 +10595,7 @@ SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); -SQLITE_PRIVATE int sqlite3MemoryAlarm(void (*)(void*, sqlite3_int64, int), void*, sqlite3_int64); +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* ** On systems with ample stack space and that support alloca(), make @@ -10514,7 +10766,6 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*, int); SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse*,int,int); SQLITE_PRIVATE int sqlite3ExprCode(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); @@ -10634,17 +10885,14 @@ SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int) #endif SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); -SQLITE_PRIVATE int sqlite3BtreeFactory(sqlite3 *db, const char *zFilename, - int omitJournal, int nCache, int flags, Btree **ppBtree); SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); -SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *, int); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); @@ -10690,7 +10938,7 @@ SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *, Table *); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); -SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*); +SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -10761,7 +11009,9 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *, Index *); SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*)); + void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + FuncDestructor *pDestructor +); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); @@ -11681,6 +11931,7 @@ struct VdbeCursor { Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ + Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ Btree *pBt; /* Separate file holding temporary table */ int pseudoTableReg; /* Register holding pseudotable content. */ @@ -11775,6 +12026,10 @@ struct Mem { u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ +#ifdef SQLITE_DEBUG + Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ + void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ +#endif void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ }; @@ -11801,6 +12056,7 @@ struct Mem { #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_RowSet 0x0020 /* Value is a RowSet object */ #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +#define MEM_Invalid 0x0080 /* Value is undefined */ #define MEM_TypeMask 0x00ff /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -11814,19 +12070,25 @@ struct Mem { #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ #define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ - #ifdef SQLITE_OMIT_INCRBLOB #undef MEM_Zero #define MEM_Zero 0x0000 #endif - /* ** Clear any existing type flags from a Mem and replace them with f */ #define MemSetTypeFlag(p, f) \ ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) +/* +** Return true if a memory cell is not marked as invalid. This macro +** is for use inside assert() statements only. +*/ +#ifdef SQLITE_DEBUG +#define memIsValid(M) ((M)->flags & MEM_Invalid)==0 +#endif + /* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains ** additional information about auxiliary information bound to arguments @@ -12015,6 +12277,10 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*); +#endif + #ifndef SQLITE_OMIT_FOREIGN_KEY SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); #else @@ -12371,12 +12637,6 @@ end_getDigits: return cnt; } -/* -** Read text from z[] and convert into a floating point number. Return -** the number of digits converted. -*/ -#define getValue sqlite3AtoF - /* ** Parse a timezone extension on the end of a date-time. ** The extension is of the form: @@ -12578,7 +12838,7 @@ static int parseDateOrTime( const char *zDate, DateTime *p ){ - int isRealNum; /* Return from sqlite3IsNumber(). Not used */ + double r; if( parseYyyyMmDd(zDate,p)==0 ){ return 0; }else if( parseHhMmSs(zDate, p)==0 ){ @@ -12586,9 +12846,7 @@ static int parseDateOrTime( }else if( sqlite3StrICmp(zDate,"now")==0){ setDateTimeToCurrent(context, p); return 0; - }else if( sqlite3IsNumber(zDate, &isRealNum, SQLITE_UTF8) ){ - double r; - getValue(zDate, &r); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; @@ -12809,8 +13067,9 @@ static int parseModifier(const char *zMod, DateTime *p){ ** weekday N where 0==Sunday, 1==Monday, and so forth. If the ** date is already on the appropriate weekday, this is a no-op. */ - if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + if( strncmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; @@ -12865,8 +13124,11 @@ static int parseModifier(const char *zMod, DateTime *p){ case '8': case '9': { double rRounder; - n = getValue(z, &r); - assert( n>=1 ); + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + rc = 1; + break; + } if( z[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds @@ -13520,6 +13782,12 @@ SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ } SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); }else{ @@ -13903,7 +14171,7 @@ static int sqlite3MemSize(void *pPrior){ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); - nByte = ROUND8(nByte); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ p--; p = realloc(p, nByte+8 ); if( p ){ @@ -14309,6 +14577,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ struct MemBlockHdr *pOldHdr; void *pNew; assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ pOldHdr = sqlite3MemsysGetHeader(pPrior); pNew = sqlite3MemMalloc(nByte); if( pNew ){ @@ -15578,7 +15847,7 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ int nOld; void *p; assert( pPrior!=0 ); - assert( (nBytes&(nBytes-1))==0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ assert( nBytes>=0 ); if( nBytes==0 ){ return 0; @@ -17102,6 +17371,66 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** Memory allocation functions used throughout sqlite. */ +/* +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. +*/ +SQLITE_API int sqlite3_release_memory(int n){ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + return sqlite3PcacheReleaseMemory(n); +#else + /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine + ** is a no-op returning zero if SQLite is not compiled with + ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ + UNUSED_PARAMETER(n); + return 0; +#endif +} + +/* +** An instance of the following object records the location of +** each unused scratch buffer. +*/ +typedef struct ScratchFreeslot { + struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ +} ScratchFreeslot; + +/* +** State information local to the memory allocation subsystem. +*/ +static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + + /* + ** The alarm callback and its arguments. The mem0.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; + + /* + ** Pointers to the end of sqlite3GlobalConfig.pScratch memory + ** (so that a range test can be used to determine if an allocation + ** being freed came from pScratch) and a pointer to the list of + ** unused scratch allocations. + */ + void *pScratchEnd; + ScratchFreeslot *pScratchFree; + u32 nScratchFree; + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +#define mem0 GLOBAL(struct Mem0Global, mem0) + /* ** This routine runs when the memory allocator sees that the ** total memory allocation is about to exceed the soft heap @@ -17116,79 +17445,67 @@ static void softHeapLimitEnforcer( sqlite3_release_memory(allocSize); } +/* +** Change the alarm callback +*/ +static int sqlite3MemoryAlarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + int nUsed; + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; + mem0.alarmThreshold = iThreshold; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3MemoryAlarm. +*/ +SQLITE_API int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); +} +#endif + /* ** Set the soft heap-size limit for the library. Passing a zero or ** negative value indicates no limit. */ -SQLITE_API void sqlite3_soft_heap_limit(int n){ - sqlite3_uint64 iLimit; - int overage; - if( n<0 ){ - iLimit = 0; - }else{ - iLimit = n; - } +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; + sqlite3_int64 excess; #ifndef SQLITE_OMIT_AUTOINIT sqlite3_initialize(); #endif - if( iLimit>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, iLimit); + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.alarmThreshold; + sqlite3_mutex_leave(mem0.mutex); + if( n<0 ) return priorLimit; + if( n>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); }else{ sqlite3MemoryAlarm(0, 0, 0); } - overage = (int)(sqlite3_memory_used() - (i64)n); - if( overage>0 ){ - sqlite3_release_memory(overage); - } + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; } - -/* -** Attempt to release up to n bytes of non-essential memory currently -** held by SQLite. An example of non-essential memory is memory used to -** cache database pages that are not currently in use. -*/ -SQLITE_API int sqlite3_release_memory(int n){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - int nRet = 0; - nRet += sqlite3PcacheReleaseMemory(n-nRet); - return nRet; -#else - UNUSED_PARAMETER(n); - return SQLITE_OK; -#endif +SQLITE_API void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); } -/* -** State information local to the memory allocation subsystem. -*/ -static SQLITE_WSD struct Mem0Global { - /* Number of free pages for scratch and page-cache memory */ - u32 nScratchFree; - u32 nPageFree; - - sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; - - /* - ** Pointers to the end of sqlite3GlobalConfig.pScratch and - ** sqlite3GlobalConfig.pPage to a block of memory that records - ** which pages are available. - */ - u32 *aScratchFree; - u32 *aPageFree; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; - -#define mem0 GLOBAL(struct Mem0Global, mem0) - /* ** Initialize the memory allocation subsystem. */ @@ -17201,36 +17518,45 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); } if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 - && sqlite3GlobalConfig.nScratch>=0 ){ - int i; - sqlite3GlobalConfig.szScratch = ROUNDDOWN8(sqlite3GlobalConfig.szScratch-4); - mem0.aScratchFree = (u32*)&((char*)sqlite3GlobalConfig.pScratch) - [sqlite3GlobalConfig.szScratch*sqlite3GlobalConfig.nScratch]; - for(i=0; i0 ){ + int i, n, sz; + ScratchFreeslot *pSlot; + sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); + sqlite3GlobalConfig.szScratch = sz; + pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; + n = sqlite3GlobalConfig.nScratch; + mem0.pScratchFree = pSlot; + mem0.nScratchFree = n; + for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); + pSlot = pSlot->pNext; + } + pSlot->pNext = 0; + mem0.pScratchEnd = (void*)&pSlot[1]; }else{ + mem0.pScratchEnd = 0; sqlite3GlobalConfig.pScratch = 0; sqlite3GlobalConfig.szScratch = 0; + sqlite3GlobalConfig.nScratch = 0; } - if( sqlite3GlobalConfig.pPage && sqlite3GlobalConfig.szPage>=512 - && sqlite3GlobalConfig.nPage>=1 ){ - int i; - int overhead; - int sz = ROUNDDOWN8(sqlite3GlobalConfig.szPage); - int n = sqlite3GlobalConfig.nPage; - overhead = (4*n + sz - 1)/sz; - sqlite3GlobalConfig.nPage -= overhead; - mem0.aPageFree = (u32*)&((char*)sqlite3GlobalConfig.pPage) - [sqlite3GlobalConfig.szPage*sqlite3GlobalConfig.nPage]; - for(i=0; i= mem0.alarmThreshold ){ + mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; } } p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( p==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } +#endif if( p ){ nFull = sqlite3MallocSize(p); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); @@ -17350,7 +17651,9 @@ static int mallocWithAlarm(int n, void **pp){ */ SQLITE_PRIVATE void *sqlite3Malloc(int n){ void *p; - if( n<=0 || n>=0x7fffff00 ){ + if( n<=0 /* IMP: R-65312-04917 */ + || n>=0x7fffff00 + ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving @@ -17364,6 +17667,7 @@ SQLITE_PRIVATE void *sqlite3Malloc(int n){ }else{ p = sqlite3GlobalConfig.m.xMalloc(n); } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ return p; } @@ -17402,59 +17706,65 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); + sqlite3_mutex_enter(mem0.mutex); + if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ + p = mem0.pScratchFree; + mem0.pScratchFree = mem0.pScratchFree->pNext; + mem0.nScratchFree--; + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + n = mallocWithAlarm(n, &p); + if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3GlobalConfig.m.xMalloc(n); + } + sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); + } + assert( sqlite3_mutex_notheld(mem0.mutex) ); + + #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocation per thread - ** is outstanding at one time. (This is only checked in the + /* Verify that no more than two scratch allocations per thread + ** are outstanding at one time. (This is only checked in the ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut<=1 ); -#endif - - if( sqlite3GlobalConfig.szScratch=(void*)mem0.aScratchFree ){ + +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) + /* Verify that no more than two scratch allocation per thread + ** is outstanding at one time. (This is only checked in the + ** single-threaded case since checking in the multi-threaded case + ** would be much more complicated.) */ + assert( scratchAllocOut>=1 && scratchAllocOut<=2 ); + scratchAllocOut--; +#endif + + if( p>=sqlite3GlobalConfig.pScratch && ppNext = mem0.pScratchFree; + mem0.pScratchFree = pSlot; + mem0.nScratchFree++; + assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch ); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); + sqlite3_mutex_leave(mem0.mutex); + }else{ + /* Release memory back to the heap */ assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); @@ -17469,26 +17779,6 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ }else{ sqlite3GlobalConfig.m.xFree(p); } - }else{ - int i; - i = (int)((u8*)p - (u8*)sqlite3GlobalConfig.pScratch); - i /= sqlite3GlobalConfig.szScratch; - assert( i>=0 && i=1 && scratchAllocOut<=2 ); - scratchAllocOut = 0; -#endif - } } } @@ -17529,7 +17819,7 @@ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ SQLITE_API void sqlite3_free(void *p){ - if( p==0 ) return; + if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ @@ -17576,10 +17866,10 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ int nOld, nNew; void *pNew; if( pOld==0 ){ - return sqlite3Malloc(nBytes); + return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ } if( nBytes<=0 ){ - sqlite3_free(pOld); + sqlite3_free(pOld); /* IMP: R-31593-10574 */ return 0; } if( nBytes>=0x7fffff00 ){ @@ -17587,6 +17877,9 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ return 0; } nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); if( nOld==nNew ){ pNew = pOld; @@ -17612,6 +17905,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ return pNew; } @@ -19775,6 +20069,12 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ /* ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows +** applications and extensions to compare the contents of two buffers +** containing UTF-8 strings in a case-independent fashion, using the same +** definition of case independence that SQLite uses internally when +** comparing identifiers. */ SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; @@ -19792,121 +20092,111 @@ SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ } /* -** Return TRUE if z is a pure numeric string. Return FALSE and leave -** *realnum unchanged if the string contains any character which is not -** part of a number. +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. ** -** If the string is pure numeric, set *realnum to TRUE if the string -** contains the '.' character or an "E+000" style exponentiation suffix. -** Otherwise set *realnum to FALSE. Note that just becaue *realnum is -** false does not mean that the number can be successfully converted into -** an integer - it might be too big. +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. ** -** An empty string is considered non-numeric. +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. Valid numbers +** are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. */ -SQLITE_PRIVATE int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +#ifndef SQLITE_OMIT_FLOATING_POINT int incr = (enc==SQLITE_UTF8?1:2); - if( enc==SQLITE_UTF16BE ) z++; - if( *z=='-' || *z=='+' ) z += incr; - if( !sqlite3Isdigit(*z) ){ - return 0; - } - z += incr; - *realnum = 0; - while( sqlite3Isdigit(*z) ){ z += incr; } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( *z=='.' ){ - z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z += incr; - if( *z=='+' || *z=='-' ) z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } -#endif - return *z==0; -} - -/* -** The string z[] is an ASCII representation of a real number. -** Convert this string to a double. -** -** This routine assumes that z[] really is a valid number. If it -** is not, the result is undefined. -** -** This routine is used instead of the library atof() function because -** the library atof() might want to use "," as the decimal point instead -** of "." depending on how locale is set. But that would cause problems -** for SQL. So this routine always uses "." regardless of locale. -*/ -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ -#ifndef SQLITE_OMIT_FLOATING_POINT - const char *zBegin = z; + const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + *pResult = 0.0; /* Default return value, in case of an error */ + + if( enc==SQLITE_UTF16BE ) z++; + /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; + while( z=zEnd ) return 0; + /* get sign of significand */ if( *z=='-' ){ sign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } + /* skip leading zeroes */ - while( z[0]=='0' ) z++, nDigits++; + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z++; + z+=incr; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + while( z=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z++; + z+=incr; + eValid = 0; + if( z>=zEnd ) goto do_atof_calc; /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } /* copy digits to exponent */ - while( sqlite3Isdigit(*z) ){ + while( z=zEnd && nDigits>0 && eValid; #else - return sqlite3Atoi64(z, pResult); + return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } @@ -19976,20 +20266,26 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ ** Compare the 19-character string zNum against the text representation ** value 2^63: 9223372036854775808. Return negative, zero, or positive ** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. ** ** Unlike memcmp() this routine is guaranteed to return the difference ** in the values of the last digit if the only difference is in the ** last digit. So, for example, ** -** compare2pow63("9223372036854775800") +** compare2pow63("9223372036854775800", 1) ** ** will return -8. */ -static int compare2pow63(const char *zNum){ - int c; - c = memcmp(zNum,"922337203685477580",18)*10; +static int compare2pow63(const char *zNum, int incr){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i*incr]-pow63[i])*10; + } if( c==0 ){ - c = zNum[18] - '8'; + c = zNum[18*incr] - '8'; testcase( c==(-1) ); testcase( c==0 ); testcase( c==(+1) ); @@ -19999,94 +20295,60 @@ static int compare2pow63(const char *zNum){ /* -** Return TRUE if zNum is a 64-bit signed integer and write -** the value of the integer into *pNum. If zNum is not an integer -** or is an integer that is too large to be expressed with 64 bits, -** then return false. +** Convert zNum to a 64-bit signed integer and write +** the value of the integer into *pNum. +** If zNum is exactly 9223372036854665808, return 2. +** This is a special case as the context will determine +** if it is too big (used as a negative). +** If zNum is not an integer or is an integer that +** is too large to be expressed with 64 bits, +** then return 1. Otherwise return 0. ** -** When this routine was originally written it dealt with only -** 32-bit numbers. At that time, it was much faster than the -** atoi() library routine in RedHat 7.2. +** length is the number of bytes in the string (bytes, not characters). +** The string is not necessarily zero-terminated. The encoding is +** given by enc. */ -SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum){ +SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ + int incr = (enc==SQLITE_UTF8?1:2); i64 v = 0; - int neg; - int i, c; + int neg = 0; /* assume positive */ + int i; + int c = 0; const char *zStart; - while( sqlite3Isspace(*zNum) ) zNum++; + const char *zEnd = zNum + length; + if( enc==SQLITE_UTF16BE ) zNum++; + while( zNum=zEnd ) goto do_atoi_calc; if( *zNum=='-' ){ neg = 1; - zNum++; + zNum+=incr; }else if( *zNum=='+' ){ - neg = 0; - zNum++; - }else{ - neg = 0; + zNum+=incr; } +do_atoi_calc: zStart = zNum; - while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */ - for(i=0; (c=zNum[i])>='0' && c<='9'; i++){ + while( zNum='0' && c<='9'; i+=incr){ v = v*10 + c - '0'; } *pNum = neg ? -v : v; testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ + if( (c!=0 && &zNum[i]19*incr ){ /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranting that it is too large) */ - return 0; - }else if( i<19 ){ - /* Less than 19 digits, so we know that it fits in 64 bits */ + ** than 19 digits (thus guaranteeing that it is too large) */ return 1; + }else if( i<19*incr ){ + /* Less than 19 digits, so we know that it fits in 64 bits */ + return 0; }else{ /* 19-digit numbers must be no larger than 9223372036854775807 if positive ** or 9223372036854775808 if negative. Note that 9223372036854665808 - ** is 2^63. */ - return compare2pow63(zNum)='0' && zNum[0]<='9' ); /* zNum is an unsigned number */ - - if( negFlag ) neg = 1-neg; - while( *zNum=='0' ){ - zNum++; /* Skip leading zeros. Ticket #2454 */ - } - for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( i<19 ){ - /* Guaranteed to fit if less than 19 digits */ - return 1; - }else if( i>19 ){ - /* Guaranteed to be too big if greater than 19 digits */ - return 0; - }else{ - /* Compare against 2^63. */ - return compare2pow63(zNum)apRegion = apNew; while(pShmNode->nRegion<=iRegion){ void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, iRegion*szRegion + MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ rc = SQLITE_IOERR; @@ -26697,11 +26959,21 @@ static int fillInUnixFile( */ UNUSED_PARAMETER(isDelete); + /* Usually the path zFilename should not be a relative pathname. The + ** exception is when opening the proxy "conch" file in builds that + ** include the special Apple locking styles. + */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + assert( zFilename==0 || zFilename[0]=='/' + || pVfs->pAppData==(void*)&autolockIoFinder ); +#else + assert( zFilename==0 || zFilename[0]=='/' ); +#endif + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); pNew->h = h; pNew->dirfd = dirfd; pNew->fileFlags = 0; - assert( zFilename==0 || zFilename[0]=='/' ); /* Never a relative pathname */ pNew->zPath = zFilename; #if OS_VXWORKS @@ -28002,6 +28274,8 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); +#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ + && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 if( gethostuuid(pHostID, &timeout) ){ int err = errno; if( pError ){ @@ -28009,6 +28283,7 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ } return SQLITE_IOERR; } +#endif #ifdef SQLITE_TEST /* simulate multiple hosts by creating unique hostid file paths */ if( sqlite3_hostid_num != 0){ @@ -30341,6 +30616,14 @@ static int winDeviceCharacteristics(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +SYSTEM_INFO winSysInfo; + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -30509,6 +30792,7 @@ static int winDelete(sqlite3_vfs *,const char*,int); static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode **pp; winShmNode *p; + BOOL bRc; assert( winShmMutexHeld() ); pp = &winShmNodeList; while( (p = *pp)!=0 ){ @@ -30516,8 +30800,14 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ int i; if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ - UnmapViewOfFile(p->aRegion[i].pMap); - CloseHandle(p->aRegion[i].hMap); + bRc = UnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); + bRc = CloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); } if( p->hFile.h != INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); @@ -30594,10 +30884,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc = SQLITE_NOMEM; goto shm_open_err; } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ 0); if( SQLITE_OK!=rc ){ rc = SQLITE_CANTOPEN_BKPT; @@ -30905,10 +31196,18 @@ static int winShmMap( hMap = CreateFileMapping(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); + OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, - 0, 0, nByte + 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); + OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, + pMap ? "ok" : "failed")); } if( !pMap ){ pShmNode->lastErrno = GetLastError(); @@ -30925,8 +31224,10 @@ static int winShmMap( shmpage_out: if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; char *p = (char *)pShmNode->aRegion[iRegion].pMap; - *pp = (void *)&p[iRegion*szRegion]; + *pp = (void *)&p[iOffsetShift]; }else{ *pp = 0; } @@ -31153,9 +31454,60 @@ static int winOpen( int isTemp = 0; #endif winFile *pFile = (winFile*)id; - void *zConverted; /* Filename in OS encoding */ - const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ - char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); +#ifndef NDEBUG + int isReadonly = (flags & SQLITE_OPEN_READONLY); +#endif + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); assert( id!=0 ); UNUSED_PARAMETER(pVfs); @@ -31166,7 +31518,8 @@ static int winOpen( ** temporary file name to use */ if( !zUtf8Name ){ - int rc = getTempname(MAX_PATH+1, zTmpname); + assert(isDelete && !isOpenJournal); + rc = getTempname(MAX_PATH+1, zTmpname); if( rc!=SQLITE_OK ){ return rc; } @@ -31179,29 +31532,31 @@ static int winOpen( return SQLITE_NOMEM; } - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; }else{ dwDesiredAccess = GENERIC_READ; } + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is ** created. SQLite doesn't use it to indicate "exclusive access" ** as it is usually understood. */ - assert(!(flags & SQLITE_OPEN_EXCLUSIVE) || (flags & SQLITE_OPEN_CREATE)); - if( flags & SQLITE_OPEN_EXCLUSIVE ){ + if( isExclusive ){ /* Creates a new file, only if it does not already exist. */ /* If the file exists, it fails. */ dwCreationDisposition = CREATE_NEW; - }else if( flags & SQLITE_OPEN_CREATE ){ + }else if( isCreate ){ /* Open existing file, or create if it doesn't exist */ dwCreationDisposition = OPEN_ALWAYS; }else{ /* Opens a file, only if it exists. */ dwCreationDisposition = OPEN_EXISTING; } + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + + if( isDelete ){ #if SQLITE_OS_WINCE dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; isTemp = 1; @@ -31218,6 +31573,7 @@ static int winOpen( #if SQLITE_OS_WINCE dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; #endif + if( isNT() ){ h = CreateFileW((WCHAR*)zConverted, dwDesiredAccess, @@ -31243,26 +31599,30 @@ static int winOpen( ); #endif } + OSTRACE(("OPEN %d %s 0x%lx %s\n", h, zName, dwDesiredAccess, h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = GetLastError(); free(zConverted); - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~SQLITE_OPEN_READWRITE), pOutFlags); + ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } } + if( pOutFlags ){ - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ *pOutFlags = SQLITE_OPEN_READWRITE; }else{ *pOutFlags = SQLITE_OPEN_READONLY; } } + memset(pFile, 0, sizeof(*pFile)); pFile->pMethod = &winIoMethod; pFile->h = h; @@ -31271,9 +31631,9 @@ static int winOpen( pFile->pShm = 0; pFile->zPath = zName; pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); + #if SQLITE_OS_WINCE - if( (flags & (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB)) == - (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB) + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && !winceCreateLock(zName, pFile) ){ CloseHandle(h); @@ -31287,8 +31647,9 @@ static int winOpen( { free(zConverted); } + OpenCounter(+1); - return SQLITE_OK; + return rc; } /* @@ -31807,6 +32168,13 @@ SQLITE_API int sqlite3_os_init(void){ winCurrentTimeInt64, /* xCurrentTimeInt64 */ }; +#ifndef SQLITE_OMIT_WAL + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&winSysInfo); + assert(winSysInfo.dwAllocationGranularity > 0); +#endif + sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; } @@ -32371,12 +32739,16 @@ static void pcacheUnpin(PgHdr *p){ */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ if( sqlite3GlobalConfig.pcache.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ sqlite3PCacheSetDefault(); } return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); } SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ if( sqlite3GlobalConfig.pcache.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); } } @@ -32837,8 +33209,13 @@ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; -/* Pointers to structures of this type are cast and returned as -** opaque sqlite3_pcache* handles +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable @@ -32898,6 +33275,9 @@ static SQLITE_WSD struct PCacheGlobal { /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */ int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nFreeSlot; /* Number of unused pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ void *pStart, *pEnd; /* Bounds of pagecache malloc range */ PgFreeslot *pFree; /* Free page blocks */ int isInit; /* True if initialized */ @@ -32945,6 +33325,8 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ PgFreeslot *p; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; while( n-- ){ @@ -32971,6 +33353,8 @@ static void *pcache1Alloc(int nByte){ assert( pcache1.isInit ); p = (PgHdr1 *)pcache1.pFree; pcache1.pFree = pcache1.pFree->pNext; + pcache1.nFreeSlot--; + assert( pcache1.nFreeSlot>=0 ); sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); }else{ @@ -33004,6 +33388,8 @@ static void pcache1Free(void *p){ pSlot = (PgFreeslot*)p; pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + assert( pcache1.nFreeSlot<=pcache1.nSlot ); }else{ int iSize; assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); @@ -33016,7 +33402,7 @@ static void pcache1Free(void *p){ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* -** Return the size of a pache allocation +** Return the size of a pcache allocation */ static int pcache1MemSize(void *p){ assert( sqlite3_mutex_held(pcache1.mutex) ); @@ -33089,6 +33475,32 @@ SQLITE_PRIVATE void sqlite3PageFree(void *p){ pcache1LeaveMutex(); } + +/* +** Return true if it desirable to avoid allocating a new page cache +** entry. +** +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory put the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. +*/ +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){ + return pcache1.nFreeSlot=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage) || nPinned>=(pCache->nMax * 9 / 10) + || pcache1UnderMemoryPressure(pCache) )){ goto fetch_out; } @@ -33390,7 +33808,9 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ /* Step 4. Try to recycle a page buffer if appropriate. */ if( pCache->bPurgeable && pcache1.pLruTail && ( - (pCache->nPage+1>=pCache->nMax) || pcache1.nCurrentPage>=pcache1.nMaxPage + (pCache->nPage+1>=pCache->nMax) + || pcache1.nCurrentPage>=pcache1.nMaxPage + || pcache1UnderMemoryPressure(pCache) )){ pPage = pcache1.pLruTail; pcache1RemoveFromHash(pPage); @@ -33533,6 +33953,7 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ */ static void pcache1Destroy(sqlite3_pcache *p){ PCache1 *pCache = (PCache1 *)p; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(); pcache1TruncateUnsafe(pCache, 0); pcache1.nMaxPage -= pCache->nMax; @@ -33580,7 +34001,7 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ if( pcache1.pStart==0 ){ PgHdr1 *p; pcache1EnterMutex(); - while( (nReq<0 || nFreepPager; - assert( !pPg->pageHash || pPager->errCode - || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else #define pager_datahash(X,Y) 0 #define pager_pagehash(X) 0 +#define pager_set_pagehash(X) #define CHECK_PAGE(x) #endif /* SQLITE_CHECK_PAGES */ @@ -36088,11 +36510,19 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } + } #ifdef SQLITE_CHECK_PAGES - sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); -#endif + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnref(p); + } } +#endif + sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; @@ -36439,9 +36869,8 @@ static int pager_playback_one_page( assert( !pagerUseWal(pPager) ); sqlite3PcacheMakeClean(pPg); } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); + /* If this was page 1, then restore the value of Pager.dbFileVers. ** Do this before any decoding. */ if( pgno==1 ){ @@ -37080,6 +37509,14 @@ static int pagerWalFrames( sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } } + +#ifdef SQLITE_CHECK_PAGES + { + PgHdr *p; + for(p=pList; p; p=p->pDirty) pager_set_pagehash(p); + } +#endif + return rc; } @@ -37106,12 +37543,13 @@ static int pagerBeginReadTransaction(Pager *pPager){ sqlite3WalEndReadTransaction(pPager->pWal); rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc==SQLITE_OK && changed ){ + if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); } return rc; } +#endif /* ** This function is called as part of the transition from PAGER_OPEN @@ -37168,7 +37606,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ return SQLITE_OK; } - +#ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager ** exists if the database is not empy, or verify that the *-wal file does @@ -38098,9 +38536,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } -#ifdef SQLITE_CHECK_PAGES - pList->pageHash = pager_pagehash(pList); -#endif + pager_set_pagehash(pList); pList = pList->pDirty; } @@ -38363,6 +38799,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + zFilename = 0; + } +#endif + /* Compute and store the full pathname in an allocated buffer pointed ** to by zPathname, length nPathname. Or, if this is a temporary file, ** leave both nPathname and zPathname set to 0. @@ -38373,17 +38816,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( if( zPathname==0 ){ return SQLITE_NOMEM; } -#ifndef SQLITE_OMIT_MEMORYDB - if( strcmp(zFilename,":memory:")==0 ){ - memDb = 1; - zPathname[0] = 0; - }else -#endif - { - zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ - rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); - } - + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); nPathname = sqlite3Strlen30(zPathname); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by @@ -38438,19 +38872,15 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ if( zPathname ){ + assert( nPathname>0 ); pPager->zJournal = (char*)(pPtr += nPathname + 1); memcpy(pPager->zFilename, zPathname, nPathname); memcpy(pPager->zJournal, zPathname, nPathname); memcpy(&pPager->zJournal[nPathname], "-journal", 8); - if( pPager->zFilename[0]==0 ){ - pPager->zJournal[0] = 0; - } #ifndef SQLITE_OMIT_WAL - else{ - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal", 4); - } + pPager->zWal = &pPager->zJournal[nPathname+8+1]; + memcpy(pPager->zWal, zPathname, nPathname); + memcpy(&pPager->zWal[nPathname], "-wal", 4); #endif sqlite3_free(zPathname); } @@ -38459,9 +38889,10 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Open the pager file. */ - if( zFilename && zFilename[0] && !memDb ){ + if( zFilename && zFilename[0] ){ int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); readOnly = (fout&SQLITE_OPEN_READONLY); /* If the file was successfully opened for read/write access, @@ -38915,7 +39346,9 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif } if( pagerUseWal(pPager) ){ @@ -39088,9 +39521,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( goto pager_acquire_err; } } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } return SQLITE_OK; @@ -39346,29 +39777,29 @@ static int pager_write(PgHdr *pPg){ CHECK_PAGE(pPg); + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); + /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ assert( !pagerUseWal(pPager) ); - assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); }else{ - - /* If we get this far, it means that the page needs to be - ** written to the transaction journal or the checkpoint journal - ** or both. - ** - ** Higher level routines have already obtained the necessary locks - ** to begin the write-transaction, but the rollback journal might not - ** yet be open. Open it now if this is the case. - */ - if( pPager->eState==PAGER_WRITER_LOCKED ){ - rc = pager_open_journal(pPager); - if( rc!=SQLITE_OK ) return rc; - } - assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); - assert( assert_pager_state(pPager) ); /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to @@ -39588,9 +40019,7 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } } @@ -43172,7 +43601,7 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ ** ** SQLITE_OK is returned if no error is encountered (regardless of whether ** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned -** if some error +** if an error occurs. */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; @@ -43205,6 +43634,8 @@ static int walRestartLog(Wal *pWal){ for(i=1; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; } } walUnlockShared(pWal, WAL_READ_LOCK(0)); @@ -43284,7 +43715,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( return rc; } } - assert( pWal->szPage==szPage ); + assert( (int)pWal->szPage==szPage ); /* Write the log file. */ for(p=pList; p; p=p->pDirty){ @@ -43944,6 +44375,7 @@ struct BtShared { u8 pageSizeFixed; /* True if the page size can no longer be changed */ u8 secureDelete; /* True if secure_delete is enabled */ u8 initiallyEmpty; /* Database is empty at start of transaction */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ @@ -46194,11 +46626,20 @@ static int btreeInvokeBusyHandler(void *pArg){ ** Open a database file. ** ** zFilename is the name of the database file. If zFilename is NULL -** a new database with a random name is created. This randomly named -** database file will be deleted when sqlite3BtreeClose() is called. +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** ** If zFilename is ":memory:" then an in-memory database is created ** that is automatically destroyed when it is closed. ** +** The "flags" parameter is a bitmask that might contain bits +** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK +** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. +** These flags are passed through into sqlite3PagerOpen() and must +** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. +** ** If the database is already opened in the same database connection ** and we are in shared cache mode, then the open will fail with an ** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared @@ -46220,6 +46661,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( u8 nReserve; /* Byte of unused space on each page */ unsigned char zDbHeader[100]; /* Database header content */ + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; + /* Set the variable isMemdb to true for an in-memory database, or ** false for a file-based database. This symbol is only required if ** either of the shared-data or autovacuum features are compiled @@ -46229,13 +46673,30 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #ifdef SQLITE_OMIT_MEMORYDB const int isMemdb = 0; #else - const int isMemdb = zFilename && !strcmp(zFilename, ":memory:"); + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)); #endif #endif assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( db->flags & SQLITE_NoReadlock ){ + flags |= BTREE_NO_READLOCK; + } + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } pVfs = db->pVfs; p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ @@ -46253,7 +46714,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** If this Btree is a candidate for shared cache, try to find an ** existing BtShared object that we can share with */ - if( isMemdb==0 && zFilename && zFilename[0] ){ + if( isMemdb==0 && isTempDb==0 ){ if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ int nFullPathname = pVfs->mxPathname+1; char *zFullPathname = sqlite3Malloc(nFullPathname); @@ -46328,6 +46789,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( rc!=SQLITE_OK ){ goto btree_open_out; } + pBt->openFlags = (u8)flags; pBt->db = db; sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); p->pBt = pBt; @@ -46432,6 +46894,14 @@ btree_open_out: sqlite3_free(pBt); sqlite3_free(p); *ppBtree = 0; + }else{ + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); + } } if( mutexOpen ){ assert( sqlite3_mutex_held(mutexOpen) ); @@ -49329,6 +49799,10 @@ static int allocateBtreePage( if( !pPrevTrunk ){ memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); } }else{ @@ -51378,11 +51852,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ ** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys ** BTREE_ZERODATA Used for SQL indices */ -static int btreeCreateTable(Btree *p, int *piTable, int flags){ +static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ BtShared *pBt = p->pBt; MemPage *pRoot; Pgno pgnoRoot; int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -51501,8 +51976,14 @@ static int btreeCreateTable(Btree *p, int *piTable, int flags){ } #endif assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - zeroPage(pRoot, flags | PTF_LEAF); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); *piTable = (int)pgnoRoot; return SQLITE_OK; } @@ -52762,7 +53243,10 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ); p = 0; }else { - /* Allocate space for a new sqlite3_backup object */ + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); if( !p ){ sqlite3Error(pDestDb, SQLITE_NOMEM, 0); @@ -53145,6 +53629,9 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ } sqlite3BtreeLeave(p->pSrc); if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ sqlite3_free(p); } sqlite3_mutex_leave(mutex); @@ -53396,6 +53883,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; pMem->flags |= MEM_Term; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif } return SQLITE_OK; @@ -53516,7 +54006,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ctx.s.db = pMem->db; ctx.pMem = pMem; ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); sqlite3DbFree(pMem->db, pMem->zMalloc); memcpy(pMem, &ctx.s, sizeof(ctx.s)); @@ -53629,13 +54119,9 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ return doubleToInt64(pMem->r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - return 0; - } - assert( pMem->z ); - sqlite3Atoi64(pMem->z, &value); + assert( pMem->z || pMem->n==0 ); + testcase( pMem->z==0 ); + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ return 0; @@ -53658,14 +54144,7 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ double val = (double)0; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - return (double)0; - } - assert( pMem->z ); - sqlite3AtoF(pMem->z, &val); + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); return val; }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ @@ -53738,21 +54217,19 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - int rc; - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ); - assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8); - if( rc ) return rc; - rc = sqlite3VdbeMemNulTerminate(pMem); - if( rc ) return rc; - if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){ - MemSetTypeFlag(pMem, MEM_Int); - }else{ - pMem->r = sqlite3VdbeRealValue(pMem); - MemSetTypeFlag(pMem, MEM_Real); - sqlite3VdbeIntegerAffinity(pMem); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + MemSetTypeFlag(pMem, MEM_Int); + }else{ + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); + } } + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob); return SQLITE_OK; } @@ -53857,6 +54334,28 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ return 0; } +#ifdef SQLITE_DEBUG +/* +** This routine prepares a memory cell for modication by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to make sure shallow +** copies are not misused. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + pX->flags |= MEM_Invalid; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ + /* ** Size of struct Mem not including the Mem.zMalloc member. */ @@ -54225,7 +54724,7 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return 0; } } - sqlite3VdbeMemNulTerminate(pVal); + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ }else{ assert( (pVal->flags&MEM_Blob)==0 ); sqlite3VdbeMemStringify(pVal, enc); @@ -54273,6 +54772,8 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( int op; char *zVal = 0; sqlite3_value *pVal = 0; + int negInt = 1; + const char *zNeg = ""; if( !pExpr ){ *ppVal = 0; @@ -54290,13 +54791,24 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; #endif + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. + */ + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ pVal = sqlite3ValueNew(db); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ - sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue); + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3DbStrDup(db, pExpr->u.zToken); + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; @@ -54306,14 +54818,18 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } + if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; if( enc!=SQLITE_UTF8 ){ sqlite3VdbeChangeEncoding(pVal, enc); } }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ + sqlite3VdbeMemNumerify(pVal); pVal->u.i = -1 * pVal->u.i; /* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */ pVal->r = (double)-1 * pVal->r; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -56140,9 +56656,10 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ char const *zFile = sqlite3BtreeGetJournalname(pBt); - if( zFile==0 || zFile[0]==0 ){ + if( zFile==0 ){ continue; /* Ignore TEMP and :memory: databases */ } + assert( zFile[0]!=0 ); if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ needSync = 1; } @@ -57606,6 +58123,8 @@ static int vdbeSafetyNotNull(Vdbe *p){ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; @@ -57682,7 +58201,7 @@ SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ sqlite3VdbeMemExpandBlob(p); p->flags &= ~MEM_Str; p->flags |= MEM_Blob; - return p->z; + return p->n ? p->z : 0; }else{ return sqlite3_value_text(pVal); } @@ -58036,6 +58555,12 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ /* ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. +** +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ assert( p && p->pFunc ); @@ -58245,8 +58770,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** -** But not for sqlite3_column_blob(), which never calls malloc(). +** sqiite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { @@ -58514,6 +59038,12 @@ static int vdbeUnbind(Vdbe *p, int i){ /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. */ if( p->isPrepareV2 && ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) @@ -59011,6 +59541,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( ** commenting and indentation practices when changing or adding code. */ +/* +** Invoke this macro on memory cells just prior to changing the +** value of the cell. This macro verifies that shallow copies are +** not misused. +*/ +#ifdef SQLITE_DEBUG +# define memAboutToChange(P,M) sqlite3VdbeMemPrepareToChange(P,M) +#else +# define memAboutToChange(P,M) +#endif + /* ** The following global variable is incremented every time a cursor ** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test @@ -59203,31 +59744,17 @@ static VdbeCursor *allocateCursor( */ static void applyNumericAffinity(Mem *pRec){ if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - int realnum; + double rValue; + i64 iValue; u8 enc = pRec->enc; - sqlite3VdbeMemNulTerminate(pRec); - if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){ - i64 value; - char *zUtf8 = pRec->z; -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - assert( pRec->db ); - zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc); - if( !zUtf8 ) return; - } -#endif - if( !realnum && sqlite3Atoi64(zUtf8, &value) ){ - pRec->u.i = value; - MemSetTypeFlag(pRec, MEM_Int); - }else{ - sqlite3AtoF(zUtf8, &pRec->r); - MemSetTypeFlag(pRec, MEM_Real); - } -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - sqlite3DbFree(pRec->db, zUtf8); - } -#endif + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; } } } @@ -60120,6 +60647,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); sqlite3VdbeMemReleaseExternal(pOut); pOut->flags = MEM_Int; } @@ -60129,25 +60657,30 @@ SQLITE_PRIVATE int sqlite3VdbeExec( if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); assert( pOp->p1<=p->nMem ); + assert( memIsValid(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + assert( memIsValid(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + assert( memIsValid(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p3]); } #endif @@ -60209,6 +60742,7 @@ case OP_Goto: { /* jump */ case OP_Gosub: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); + memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; pIn1->u.i = pc; REGISTER_TRACE(pOp->p1, pIn1); @@ -60416,11 +60950,7 @@ case OP_Null: { /* out2-prerelease */ /* Opcode: Blob P1 P2 * P4 ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. This instruction is not coded directly -** by the compiler. Instead, the compiler layer specifies -** an OP_HexBlob opcode, with the hex string representation of -** the blob as P4. This opcode is transformed to an OP_Blob -** the first time it is executed. +** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); @@ -60478,6 +61008,8 @@ case OP_Move: { while( u.ac.n-- ){ assert( pOut<=&aMem[p->nMem] ); assert( pIn1<=&aMem[p->nMem] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); u.ac.zMalloc = pOut->zMalloc; pOut->zMalloc = 0; sqlite3VdbeMemMove(pOut, pIn1); @@ -60523,6 +61055,9 @@ case OP_SCopy: { /* in1, out2 */ pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; +#endif REGISTER_TRACE(pOp->p2, pOut); break; } @@ -60583,6 +61118,10 @@ case OP_ResultRow: { */ u.ad.pMem = p->pResultSet = &aMem[pOp->p1]; for(u.ad.i=0; u.ad.ip2; u.ad.i++){ + assert( memIsValid(&u.ad.pMem[u.ad.i]) ); + Deephemeralize(&u.ad.pMem[u.ad.i]); + assert( (u.ad.pMem[u.ad.i].flags & MEM_Ephem)==0 + || (u.ad.pMem[u.ad.i].flags & (MEM_Str|MEM_Blob))==0 ); sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]); sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]); REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]); @@ -60814,12 +61353,17 @@ case OP_Function: { u.ag.n = pOp->p5; u.ag.apVal = p->apArg; assert( u.ag.apVal || u.ag.n==0 ); + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+u.ag.n ); u.ag.pArg = &aMem[pOp->p2]; for(u.ag.i=0; u.ag.ip2+u.ag.i, u.ag.pArg); } @@ -60833,8 +61377,6 @@ case OP_Function: { u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc; } - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pOut = &aMem[pOp->p3]; u.ag.ctx.s.flags = MEM_Null; u.ag.ctx.s.db = db; u.ag.ctx.s.xDel = 0; @@ -60854,7 +61396,7 @@ case OP_Function: { assert( pOp[-1].opcode==OP_CollSeq ); u.ag.ctx.pColl = pOp[-1].p4.pColl; } - (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); + (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ if( db->mallocFailed ){ /* Even though a malloc() has failed, the implementation of the ** user function may have called an sqlite3_result_XXX() function @@ -60906,7 +61448,7 @@ case OP_Function: { /* Opcode: ShiftLeft P1 P2 P3 * * ** ** Shift the integer value in register P2 to the left by the -** number of bits specified by the integer in regiser P1. +** number of bits specified by the integer in register P1. ** Store the result in register P3. ** If either input is NULL, the result is NULL. */ @@ -60956,6 +61498,7 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ */ case OP_AddImm: { /* in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); sqlite3VdbeMemIntegerify(pIn1); pIn1->u.i += pOp->p2; break; @@ -60970,6 +61513,7 @@ case OP_AddImm: { /* in1 */ */ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); if( (pIn1->flags & MEM_Int)==0 ){ if( pOp->p2==0 ){ @@ -61015,6 +61559,7 @@ case OP_RealAffinity: { /* in1 */ */ case OP_ToText: { /* same as TK_TO_TEXT, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( pIn1->flags & MEM_Null ) break; assert( MEM_Str==(MEM_Blob>>3) ); pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; @@ -61061,16 +61606,14 @@ case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ */ case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){ - sqlite3VdbeMemNumerify(pIn1); - } + sqlite3VdbeMemNumerify(pIn1); break; } #endif /* SQLITE_OMIT_CAST */ /* Opcode: ToInt P1 * * * * ** -** Force the value in register P1 be an integer. If +** Force the value in register P1 to be an integer. If ** The value is currently a real number, drop its fractional part. ** If the value is text or blob, try to convert it to an integer using the ** equivalent of atoi() and store 0 if no such conversion is possible. @@ -61097,6 +61640,7 @@ case OP_ToInt: { /* same as TK_TO_INT, in1 */ */ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( (pIn1->flags & MEM_Null)==0 ){ sqlite3VdbeMemRealify(pIn1); } @@ -61111,7 +61655,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** ** If the SQLITE_JUMPIFNULL bit of P5 is set and either reg(P1) or ** reg(P3) is NULL then take the jump. If the SQLITE_JUMPIFNULL -** bit is clear then fall thru if either operand is NULL. +** bit is clear then fall through if either operand is NULL. ** ** The SQLITE_AFF_MASK portion of P5 must be an affinity character - ** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made @@ -61241,6 +61785,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = u.ai.res; REGISTER_TRACE(pOp->p2, pOut); @@ -61272,8 +61817,8 @@ case OP_Permutation: { /* Opcode: Compare P1 P2 P3 P4 * ** -** Compare to vectors of registers in reg(P1)..reg(P1+P3-1) (all this -** one "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of ** the comparison for use by the next OP_Jump instruct. ** ** P4 is a KeyInfo structure that defines collating sequences and sort @@ -61315,6 +61860,8 @@ case OP_Compare: { #endif /* SQLITE_DEBUG */ for(u.aj.i=0; u.aj.inField ); @@ -61546,6 +62093,7 @@ case OP_Column: { assert( u.am.p1nCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.am.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.am.pDest); MemSetTypeFlag(u.am.pDest, MEM_Null); u.am.zRec = 0; @@ -61593,6 +62141,7 @@ case OP_Column: { }else if( u.am.pC->pseudoTableReg>0 ){ u.am.pReg = &aMem[u.am.pC->pseudoTableReg]; assert( u.am.pReg->flags & MEM_Blob ); + assert( memIsValid(u.am.pReg) ); u.am.payloadSize = u.am.pReg->n; u.am.zRec = u.am.pReg->z; u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; @@ -61817,6 +62366,7 @@ case OP_Affinity: { pIn1 = &aMem[pOp->p1]; while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){ assert( pIn1 <= &p->aMem[p->nMem] ); + assert( memIsValid(pIn1) ); ExpandBlob(pIn1); applyAffinity(pIn1, u.an.cAff, encoding); pIn1++; @@ -61826,12 +62376,9 @@ case OP_Affinity: { /* Opcode: MakeRecord P1 P2 P3 P4 * ** -** Convert P2 registers beginning with P1 into a single entry -** suitable for use as a data record in a database table or as a key -** in an index. The details of the format are irrelevant as long as -** the OP_Column opcode can decode the record later. -** Refer to source code comments for the details of the record -** format. +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. ** ** P4 may be a string that is P2 characters long. The nth character of the ** string indicates the column affinity that should be used for the nth @@ -61888,10 +62435,16 @@ case OP_MakeRecord: { u.ao.pLast = &u.ao.pData0[u.ao.nField-1]; u.ao.file_format = p->minWriteFileFormat; + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + /* Loop through the elements that will make up the record to figure ** out how much space is required for the new record. */ for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ + assert( memIsValid(u.ao.pRec) ); if( u.ao.zAffinity ){ applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding); } @@ -61926,8 +62479,6 @@ case OP_MakeRecord: { ** be one of the input registers (because the following call to ** sqlite3VdbeMemGrow() could clobber the value before it is used). */ - assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); - pOut = &aMem[pOp->p3]; if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){ goto no_mem; } @@ -62100,6 +62651,7 @@ case OP_Savepoint: { if( u.aq.p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetInternalSchema(db, 0); + db->flags = (db->flags | SQLITE_InternChanges); } } @@ -62490,6 +63042,8 @@ case OP_OpenWrite: { assert( u.aw.p2>0 ); assert( u.aw.p2<=p->nMem ); pIn2 = &aMem[u.aw.p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); sqlite3VdbeMemIntegerify(pIn2); u.aw.p2 = (int)pIn2->u.i; /* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and @@ -62512,6 +63066,7 @@ case OP_OpenWrite: { u.aw.pCur = allocateCursor(p, pOp->p1, u.aw.nField, u.aw.iDb, 1); if( u.aw.pCur==0 ) goto no_mem; u.aw.pCur->nullRow = 1; + u.aw.pCur->isOrdered = 1; rc = sqlite3BtreeCursor(u.aw.pX, u.aw.p2, u.aw.wrFlag, u.aw.pKeyInfo, u.aw.pCur->pCursor); u.aw.pCur->pKeyInfo = u.aw.pKeyInfo; @@ -62564,7 +63119,7 @@ case OP_OpenEphemeral: { #if 0 /* local variables moved into u.ax */ VdbeCursor *pCx; #endif /* local variables moved into u.ax */ - static const int openFlags = + static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | @@ -62575,21 +63130,21 @@ case OP_OpenEphemeral: { u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( u.ax.pCx==0 ) goto no_mem; u.ax.pCx->nullRow = 1; - rc = sqlite3BtreeFactory(db, 0, 1, SQLITE_DEFAULT_TEMP_CACHE_SIZE, openFlags, - &u.ax.pCx->pBt); + rc = sqlite3BtreeOpen(0, db, &u.ax.pCx->pBt, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); } if( rc==SQLITE_OK ){ /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_ZERODATA flag before + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an INTKEY table). + ** automatically created table with root-page 1 (an BLOB_INTKEY table). */ if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_ZERODATA); + rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1, @@ -62603,6 +63158,7 @@ case OP_OpenEphemeral: { u.ax.pCx->isTable = 1; } } + u.ax.pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); u.ax.pCx->isIndex = !u.ax.pCx->isTable; break; } @@ -62722,6 +63278,7 @@ case OP_SeekGt: { /* jump, in3 */ assert( OP_SeekLe == OP_SeekLt+1 ); assert( OP_SeekGe == OP_SeekLt+2 ); assert( OP_SeekGt == OP_SeekLt+3 ); + assert( u.az.pC->isOrdered ); if( u.az.pC->pCursor!=0 ){ u.az.oc = pOp->opcode; u.az.pC->nullRow = 0; @@ -62804,6 +63361,9 @@ case OP_SeekGt: { /* jump, in3 */ assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 ); u.az.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ipCursor, &u.az.r, 0, 0, &u.az.res); if( rc!=SQLITE_OK ){ @@ -62932,11 +63492,14 @@ case OP_Found: { /* jump, in3 */ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo; u.bb.r.nField = (u16)pOp->p4.i; u.bb.r.aMem = pIn3; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iflags & MEM_Blob ); - ExpandBlob(pIn3); + assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z, u.bb.aTempRec, sizeof(u.bb.aTempRec)); if( u.bb.pIdxKey==0 ){ @@ -63031,6 +63594,9 @@ case OP_IsUnique: { /* jump, in3 */ u.bc.r.nField = u.bc.nField + 1; u.bc.r.flags = UNPACKED_PREFIX_SEARCH; u.bc.r.aMem = u.bc.aMx; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ip3<=p->nMem ); u.be.pMem = &aMem[pOp->p3]; + memAboutToChange(p, u.be.pMem); } + assert( memIsValid(u.be.pMem) ); REGISTER_TRACE(pOp->p3, u.be.pMem); sqlite3VdbeMemIntegerify(u.be.pMem); @@ -63230,29 +63798,36 @@ case OP_NewRowid: { /* out2-prerelease */ sqlite3BtreeSetCachedRowid(u.be.pC->pCursor, u.be.vuseRandomRowid ){ - /* IMPLEMENTATION-OF: R-48598-02938 If the largest ROWID is equal to the + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the ** largest possible integer (9223372036854775807) then the database - ** engine starts picking candidate ROWIDs at random until it finds one - ** that is not previously used. - */ + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ + /* on the first attempt, simply do one more than previous */ u.be.v = db->lastRowid; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + u.be.v++; /* ensure non-zero */ u.be.cnt = 0; - do{ - if( u.be.cnt==0 && (u.be.v&0xffffff)==u.be.v ){ - u.be.v++; + while( ((rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, + 0, &u.be.res))==SQLITE_OK) + && (u.be.res==0) + && (++u.be.cnt<100)){ + /* collision - try another random rowid */ + sqlite3_randomness(sizeof(u.be.v), &u.be.v); + if( u.be.cnt<5 ){ + /* try "small" random rowids for the initial attempts */ + u.be.v &= 0xffffff; }else{ - sqlite3_randomness(sizeof(u.be.v), &u.be.v); - if( u.be.cnt<5 ) u.be.v &= 0xffffff; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ } - rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, 0, &u.be.res); - u.be.cnt++; - }while( u.be.cnt<100 && rc==SQLITE_OK && u.be.res==0 ); + u.be.v++; /* ensure non-zero */ + } if( rc==SQLITE_OK && u.be.res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } + assert( u.be.v>0 ); /* EV: R-40812-03570 */ } u.be.pC->rowidIsValid = 0; u.be.pC->deferredMoveto = 0; @@ -63322,6 +63897,7 @@ case OP_InsertInt: { u.bf.pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(u.bf.pData) ); u.bf.pC = p->apCsr[pOp->p1]; assert( u.bf.pC!=0 ); assert( u.bf.pC->pCursor!=0 ); @@ -63332,6 +63908,7 @@ case OP_InsertInt: { if( pOp->opcode==OP_Insert ){ u.bf.pKey = &aMem[pOp->p3]; assert( u.bf.pKey->flags & MEM_Int ); + assert( memIsValid(u.bf.pKey) ); REGISTER_TRACE(pOp->p3, u.bf.pKey); u.bf.iKey = u.bf.pKey->u.i; }else{ @@ -63483,6 +64060,7 @@ case OP_RowData: { #endif /* local variables moved into u.bh */ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -63825,6 +64403,9 @@ case OP_IdxDelete: { u.bo.r.nField = (u16)pOp->p3; u.bo.r.flags = 0; u.bo.r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ip1>=0 && pOp->p1nCursor ); u.bq.pC = p->apCsr[pOp->p1]; assert( u.bq.pC!=0 ); + assert( u.bq.pC->isOrdered ); if( ALWAYS(u.bq.pC->pCursor!=0) ){ assert( u.bq.pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); @@ -63921,6 +64503,9 @@ case OP_IdxGE: { /* jump */ u.bq.r.flags = UNPACKED_IGNORE_ROWID; } u.bq.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iopcode==OP_IdxLT ){ u.bq.res = -u.bq.res; @@ -64024,6 +64609,8 @@ case OP_Clear: { if( pOp->p3 ){ p->nChange += u.bs.nChange; if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); aMem[pOp->p3].u.i += u.bs.nChange; } } @@ -64067,9 +64654,9 @@ case OP_CreateTable: { /* out2-prerelease */ assert( u.bt.pDb->pBt!=0 ); if( pOp->opcode==OP_CreateTable ){ /* u.bt.flags = BTREE_INTKEY; */ - u.bt.flags = BTREE_LEAFDATA|BTREE_INTKEY; + u.bt.flags = BTREE_INTKEY; }else{ - u.bt.flags = BTREE_ZERODATA; + u.bt.flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags); pOut->u.i = u.bt.pgno; @@ -64398,6 +64985,7 @@ case OP_Program: { /* jump */ u.by.pProgram = pOp->p4.pProgram; u.by.pRt = &aMem[pOp->p3]; + assert( memIsValid(u.by.pRt) ); assert( u.by.pProgram->nOp>0 ); /* If the p5 flag is clear, then recursive invocation of triggers is @@ -64571,6 +65159,7 @@ case OP_MemMax: { /* in2 */ }else{ u.ca.pIn1 = &aMem[pOp->p1]; } + assert( memIsValid(u.ca.pIn1) ); sqlite3VdbeMemIntegerify(u.ca.pIn1); pIn2 = &aMem[pOp->p2]; sqlite3VdbeMemIntegerify(pIn2); @@ -64657,7 +65246,9 @@ case OP_AggStep: { u.cb.apVal = p->apArg; assert( u.cb.apVal || u.cb.n==0 ); for(u.cb.i=0; u.cb.ip4.pFunc; @@ -64677,7 +65268,7 @@ case OP_AggStep: { assert( pOp[-1].opcode==OP_CollSeq ); u.cb.ctx.pColl = pOp[-1].p4.pColl; } - (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); + (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); /* IMP: R-24505-23230 */ if( u.cb.ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s)); rc = u.cb.ctx.isError; @@ -65064,6 +65655,7 @@ case OP_VFilter: { /* jump */ u.ch.pQuery = &aMem[pOp->p3]; u.ch.pArgc = &u.ch.pQuery[1]; u.ch.pCur = p->apCsr[pOp->p1]; + assert( memIsValid(u.ch.pQuery) ); REGISTER_TRACE(pOp->p3, u.ch.pQuery); assert( u.ch.pCur->pVtabCursor ); u.ch.pVtabCursor = u.ch.pCur->pVtabCursor; @@ -65121,6 +65713,7 @@ case OP_VColumn: { assert( pCur->pVtabCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.ci.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.ci.pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(u.ci.pDest); break; @@ -65223,10 +65816,12 @@ case OP_VRename: { u.ck.pVtab = pOp->p4.pVtab->pVtab; u.ck.pName = &aMem[pOp->p1]; assert( u.ck.pVtab->pModule->xRename ); + assert( memIsValid(u.ck.pName) ); REGISTER_TRACE(pOp->p1, u.ck.pName); assert( u.ck.pName->flags & MEM_Str ); rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z); importVtabErrMsg(p, u.ck.pVtab); + p->expired = 0; break; } @@ -65275,6 +65870,8 @@ case OP_VUpdate: { u.cl.apArg = p->apArg; u.cl.pX = &aMem[pOp->p3]; for(u.cl.i=0; u.cl.iiColumn = (ynVar)i; testcase( i==0 ); testcase( i==1 ); @@ -69214,8 +69810,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ #endif /* -** Generate code for scalar subqueries used as an expression -** and IN operators. Examples: +** Generate code for scalar subqueries used as a subquery expression, EXISTS, +** or IN operators. Examples: ** ** (SELECT a FROM b) -- subquery ** EXISTS (SELECT a FROM b) -- EXISTS subquery @@ -69278,10 +69874,10 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { - char affinity; - KeyInfo keyInfo; - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; + char affinity; /* Affinity of the LHS of the IN */ + KeyInfo keyInfo; /* Keyinfo for the generated table */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ if( rMayHaveNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); @@ -69304,6 +69900,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( */ pExpr->iTable = pParse->nTab++; addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; @@ -69596,7 +70193,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; char *zV; - sqlite3AtoF(z, &value); + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; zV = dup8bytes(v, (char*)&value); @@ -69610,9 +70207,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ ** Generate an instruction that will put the integer describe by ** text z[0..n-1] into register iMem. ** -** The z[] string will probably not be zero-terminated. But the -** z[n] character is guaranteed to be something that does not look -** like the continuation of the number. +** Expr.u.zToken is always UTF8 and zero-terminated. */ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ Vdbe *v = pParse->pVdbe; @@ -69621,13 +70216,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ if( negFlag ) i = -i; sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); }else{ + int c; + i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - if( sqlite3FitsIn64Bits(z, negFlag) ){ - i64 value; + c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + if( c==0 || (c==2 && negFlag) ){ char *zV; - sqlite3Atoi64(z, &value); - if( negFlag ) value = -value; + if( negFlag ){ value = -value; } zV = dup8bytes(v, (char*)&value); sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ @@ -69912,73 +70508,6 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ -/* -** If the last instruction coded is an ephemeral copy of any of -** the registers in the nReg registers beginning with iReg, then -** convert the last instruction from OP_SCopy to OP_Copy. -*/ -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse *pParse, int iReg, int nReg){ - VdbeOp *pOp; - Vdbe *v; - - assert( pParse->db->mallocFailed==0 ); - v = pParse->pVdbe; - assert( v!=0 ); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - if( pOp->opcode==OP_SCopy && pOp->p1>=iReg && pOp->p1opcode = OP_Copy; - } -} - -/* -** Generate code to store the value of the iAlias-th alias in register -** target. The first time this is called, pExpr is evaluated to compute -** the value of the alias. The value is stored in an auxiliary register -** and the number of that register is returned. On subsequent calls, -** the register number is returned without generating any code. -** -** Note that in order for this to work, code must be generated in the -** same order that it is executed. -** -** Aliases are numbered starting with 1. So iAlias is in the range -** of 1 to pParse->nAlias inclusive. -** -** pParse->aAlias[iAlias-1] records the register number where the value -** of the iAlias-th alias is stored. If zero, that means that the -** alias has not yet been computed. -*/ -static int codeAlias(Parse *pParse, int iAlias, Expr *pExpr, int target){ -#if 0 - sqlite3 *db = pParse->db; - int iReg; - if( pParse->nAliasAllocnAlias ){ - pParse->aAlias = sqlite3DbReallocOrFree(db, pParse->aAlias, - sizeof(pParse->aAlias[0])*pParse->nAlias ); - testcase( db->mallocFailed && pParse->nAliasAlloc>0 ); - if( db->mallocFailed ) return 0; - memset(&pParse->aAlias[pParse->nAliasAlloc], 0, - (pParse->nAlias-pParse->nAliasAlloc)*sizeof(pParse->aAlias[0])); - pParse->nAliasAlloc = pParse->nAlias; - } - assert( iAlias>0 && iAlias<=pParse->nAlias ); - iReg = pParse->aAlias[iAlias-1]; - if( iReg==0 ){ - if( pParse->iCacheLevel>0 ){ - iReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - }else{ - iReg = ++pParse->nMem; - sqlite3ExprCode(pParse, pExpr, iReg); - pParse->aAlias[iAlias-1] = iReg; - } - } - return iReg; -#else - UNUSED_PARAMETER(iAlias); - return sqlite3ExprCodeTarget(pParse, pExpr, target); -#endif -} - /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -70087,7 +70616,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } case TK_AS: { - inReg = codeAlias(pParse, pExpr->iTable, pExpr->pLeft, target); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; } #ifndef SQLITE_OMIT_CAST @@ -70519,6 +71048,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) opCompare.op = TK_EQ; opCompare.pLeft = &cacheX; pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; } for(i=0; i0 && target<=pParse->nMem ); - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + if( pExpr && pExpr->op==TK_REGISTER ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); + }else{ + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + } } return target; } @@ -70788,19 +71326,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( int i, n; assert( pList!=0 ); assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ n = pList->nExpr; for(pItem=pList->a, i=0; iiAlias ){ - int iReg = codeAlias(pParse, pItem->iAlias, pItem->pExpr, target+i); - Vdbe *v = sqlite3GetVdbe(pParse); - if( iReg!=target+i ){ - sqlite3VdbeAddOp2(v, OP_SCopy, iReg, target+i); - } - }else{ - sqlite3ExprCode(pParse, pItem->pExpr, target+i); - } - if( doHardCopy && !pParse->db->mallocFailed ){ - sqlite3ExprHardCopy(pParse, target, n); + Expr *pExpr = pItem->pExpr; + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, + inReg, target+i); } } return n; @@ -71782,6 +72315,11 @@ static char *whereTempTriggers(Parse *pParse, Table *pTab){ } } } + if( zWhere ){ + char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); + sqlite3DbFree(pParse->db, zWhere); + zWhere = zNew; + } return zWhere; } @@ -72389,7 +72927,8 @@ static void analyzeOneTable( int i; /* Loop counter */ int topOfLoop; /* The top of the loop */ int endOfLoop; /* The end of the loop */ - int addr; /* The address of an instruction */ + int addr = 0; /* The address of an instruction */ + int jZeroRows = 0; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ @@ -72408,8 +72947,15 @@ static void analyzeOneTable( #endif v = sqlite3GetVdbe(pParse); - if( v==0 || NEVER(pTab==0) || pTab->pIndex==0 ){ - /* Do no analysis for tables that have no indices */ + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + /* Do not gather statistics on system tables */ return; } assert( sqlite3BtreeHoldsAllMutexes(db) ); @@ -72426,6 +72972,7 @@ static void analyzeOneTable( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol = pIdx->nColumn; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); @@ -72440,10 +72987,7 @@ static void analyzeOneTable( (char *)pKey, P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - /* Populate the registers containing the table and index names. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); - } + /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); #ifdef SQLITE_ENABLE_STAT2 @@ -72578,8 +73122,10 @@ static void analyzeOneTable( ** If K>0 then it is always the case the D>0 so division by zero ** is never possible. */ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); + if( jZeroRows==0 ){ + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + } for(i=0; ipIndex==0 ){ + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); + VdbeComment((v, "%s", pTab->zName)); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); + }else{ + assert( jZeroRows>0 ); + addr = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, jZeroRows); + } + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + if( pParse->nMemnMem = regRec; + if( jZeroRows ){ sqlite3VdbeJumpHere(v, addr); } } /* ** Generate code that will cause the most recent index analysis to -** be laoded into internal hash tables where is can be used. +** be loaded into internal hash tables where is can be used. */ static void loadAnalysis(Parse *pParse, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); @@ -72729,33 +73297,46 @@ struct analysisInfo { ** This callback is invoked once for each index when reading the ** sqlite_stat1 table. ** -** argv[0] = name of the index -** argv[1] = results of analysis - on integer for each column +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. */ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; - int i, c; + Table *pTable; + int i, c, n; unsigned int v; const char *z; - assert( argc==2 ); + assert( argc==3 ); UNUSED_PARAMETER2(NotUsed, argc); - if( argv==0 || argv[0]==0 || argv[1]==0 ){ + if( argv==0 || argv[0]==0 || argv[2]==0 ){ return 0; } - pIndex = sqlite3FindIndex(pInfo->db, argv[0], pInfo->zDatabase); - if( pIndex==0 ){ + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ return 0; } - z = argv[1]; - for(i=0; *z && i<=pIndex->nColumn; i++){ + if( argv[1] ){ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + }else{ + pIndex = 0; + } + n = pIndex ? pIndex->nColumn : 0; + z = argv[2]; + for(i=0; *z && i<=n; i++){ v = 0; while( (c=z[0])>='0' && c<='9' ){ v = v*10 + c - '0'; z++; } + if( i==0 ) pTable->nRowEst = v; + if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; } @@ -72831,7 +73412,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, - "SELECT idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ @@ -73048,9 +73629,8 @@ static void attachFunc( ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE, - db->openFlags | SQLITE_OPEN_MAIN_DB, - &aNew->pBt); + rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, + db->openFlags | SQLITE_OPEN_MAIN_DB); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; @@ -73291,7 +73871,8 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ 0, /* xStep */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -73312,7 +73893,8 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p 0, /* xStep */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -74441,8 +75023,9 @@ SQLITE_PRIVATE void sqlite3StartTable( */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && iDb>1 ){ - /* If creating a temp table, the name may not be qualified */ + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); return; } @@ -74490,17 +75073,18 @@ SQLITE_PRIVATE void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ + char *zDb = db->aDb[iDb].zName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } - pTable = sqlite3FindTable(db, zName, db->aDb[iDb].zName); + pTable = sqlite3FindTable(db, zName, zDb); if( pTable ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "table %T already exists", pName); } goto begin_table_error; } - if( sqlite3FindIndex(db, zName, 0)!=0 && (iDb==0 || !db->init.busy) ){ + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); goto begin_table_error; } @@ -74517,6 +75101,7 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; + pTable->nRowEst = 1000000; assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -75363,12 +75948,10 @@ SQLITE_PRIVATE void sqlite3CreateView( } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 ){ + if( p==0 || pParse->nErr ){ sqlite3SelectDelete(db, pSelect); return; } - assert( pParse->nErr==0 ); /* If sqlite3StartTable return non-NULL then - ** there could not have been an error */ sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) @@ -76486,7 +77069,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC); + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), + P4_DYNAMIC); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } @@ -76547,14 +77131,14 @@ exit_create_index: SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ unsigned *a = pIdx->aiRowEst; int i; + unsigned n; assert( a!=0 ); - a[0] = 1000000; - for(i=pIdx->nColumn; i>=5; i--){ - a[i] = 5; - } - while( i>=1 ){ - a[i] = 11 - i; - i--; + a[0] = pIdx->pTable->nRowEst; + if( a[0]<10 ) a[0] = 10; + n = 10; + for(i=1; i<=pIdx->nColumn; i++){ + a[i] = n; + if( n>5 ) n--; } if( pIdx->onError!=OE_None ){ a[pIdx->nColumn] = 1; @@ -76614,7 +77198,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q", + "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); @@ -77106,7 +77690,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, &pBt); + rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); @@ -77763,7 +78347,7 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** priority to built-in functions. ** ** Except, if createFlag is true, that means that we are trying to - ** install a new function. Whatever FuncDef structure is returned will + ** install a new function. Whatever FuncDef structure is returned it will ** have fields overwritten with new information appropriate for the ** new function. But the FuncDefs for built-in functions are read-only. ** So we must not search for built-ins when creating a new function. @@ -78779,7 +79363,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r); + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -79944,10 +80528,10 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) }else{ pInfo = (struct compareInfo*)&likeInfoNorm; } - sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0); + sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY, - (struct compareInfo*)&globInfo, likeFunc, 0,0); + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); setLikeOptFlag(db, "like", caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); @@ -80031,10 +80615,10 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), /* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0}, + {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, FUNCTION(hex, 1, 0, 0, hexFunc ), /* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0}, + {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), @@ -80061,7 +80645,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0}, + {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), @@ -80472,7 +81056,7 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); for(i=0; ipBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit); + sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -87070,7 +87654,6 @@ static void pushOntoSorter( sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); sqlite3VdbeJumpHere(v, addr2); - pSelect->iLimit = 0; } } @@ -87119,11 +87702,13 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate an error message when a SELECT is used within a subexpression ** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error occurs in multiple -** places. +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) */ static int checkForMultiColumnSelectError( Parse *pParse, /* Parse context. */ @@ -87139,6 +87724,7 @@ static int checkForMultiColumnSelectError( return 0; } } +#endif /* ** This routine generates the code for the inside of the inner loop @@ -87218,10 +87804,6 @@ static void selectInnerLoop( } } - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - return; - } - switch( eDest ){ /* In this mode, write each query result to the key of the temporary ** table iParm. @@ -87350,11 +87932,11 @@ static void selectInnerLoop( #endif } - /* Jump to the end of the loop if the LIMIT is reached. + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. */ - if( p->iLimit ){ - assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to - ** pushOntoSorter() would have cleared p->iLimit */ + if( pOrderBy==0 && p->iLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } } @@ -87489,10 +88071,6 @@ static void generateSortTail( sqlite3ReleaseTempReg(pParse, regRow); sqlite3ReleaseTempReg(pParse, regRowid); - /* LIMIT has been implemented by the pushOntoSorter() routine. - */ - assert( p->iLimit==0 ); - /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -88131,6 +88709,7 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -88809,7 +89388,6 @@ static int multiSelectOrderBy( /* Separate the left and the right query from one another */ p->pPrior = 0; - pPrior->pRightmost = 0; sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); if( pPrior->pPrior==0 ){ sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); @@ -90093,7 +90671,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 0); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); }else{ nArg = 0; regAgg = 0; @@ -90253,6 +90831,15 @@ SQLITE_PRIVATE int sqlite3Select( v = sqlite3GetVdbe(pParse); if( v==0 ) goto select_end; + /* If writing to memory or generating a set + ** only a single column may be output. + */ +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; + } +#endif + /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -90326,15 +90913,6 @@ SQLITE_PRIVATE int sqlite3Select( } #endif - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - goto select_end; - } -#endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. ** GROUP BY might use an index, DISTINCT never does. */ @@ -90397,6 +90975,7 @@ SQLITE_PRIVATE int sqlite3Select( pKeyInfo = keyInfoFromExprList(pParse, p->pEList); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ distinct = -1; } @@ -92872,6 +93451,7 @@ static void updateVirtualTable( assert( v ); ephemTab = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); /* fill the ephemeral table */ @@ -93011,6 +93591,10 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); return SQLITE_ERROR; } + if( db->activeVdbeCnt>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; + } /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and @@ -93612,7 +94196,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - zWhere = sqlite3MPrintf(db, "name='%q'", pTab->zName); + zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); @@ -94852,11 +95436,12 @@ static int isLikeOrGlob( } if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; - pVal = sqlite3VdbeGetValue(pReprepare, pRight->iColumn, SQLITE_AFF_NONE); + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, pRight->iColumn); + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; @@ -94874,7 +95459,7 @@ static int isLikeOrGlob( *ppPrefix = pPrefix; if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); + sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE @@ -95738,7 +96323,8 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ ** Required because bestIndex() is called by bestOrClauseIndex() */ static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*); + Parse*, WhereClause*, struct SrcList_item*, + Bitmask, Bitmask, ExprList*, WhereCost*); /* ** This routine attempts to find an scanning strategy that can be used @@ -95751,7 +96337,8 @@ static void bestOrClauseIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ @@ -95787,7 +96374,7 @@ static void bestOrClauseIndex( )); if( pOrTerm->eOperator==WO_AND ){ WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; @@ -95795,7 +96382,7 @@ static void bestOrClauseIndex( tempWC.op = TK_AND; tempWC.a = pOrTerm; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); }else{ continue; } @@ -95888,7 +96475,7 @@ static void bestAutomaticIndex( assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; - nTableRow = pTable->pIndex ? pTable->pIndex->aiRowEst[0] : 1000000; + nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); if( costTempIdx>=pCost->rCost ){ @@ -96242,7 +96829,8 @@ static void bestVirtualIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for index */ + Bitmask notValid, /* Cursors not valid for any purpose */ ExprList *pOrderBy, /* The order by clause */ WhereCost *pCost, /* Lowest cost query plan */ sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ @@ -96372,7 +96960,7 @@ static void bestVirtualIndex( /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -96498,7 +97086,7 @@ static int valueFromExpr( assert( pExpr->op!=TK_VARIABLE ); if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){ int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); return SQLITE_OK; } @@ -96653,7 +97241,8 @@ static void bestBtreeIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ @@ -96695,23 +97284,14 @@ static void bestBtreeIndex( sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowEst = aiRowEstPk; - aiRowEstPk[1] = 1; sPk.onError = OE_Replace; sPk.pTable = pSrc->pTab; + aiRowEstPk[0] = pSrc->pTab->nRowEst; + aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ sPk.pNext = pFirst; } - /* The aiRowEstPk[0] is an estimate of the total number of rows in the - ** table. Get this information from the ANALYZE information if it is - ** available. If not available, assume the table 1 million rows in size. - */ - if( pFirst ){ - assert( pFirst->aiRowEst!=0 ); /* Allocated together with pFirst */ - aiRowEstPk[0] = pFirst->aiRowEst[0]; - }else{ - aiRowEstPk[0] = 1000000; - } pProbe = &sPk; wsFlagMask = ~( WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE @@ -96924,16 +97504,16 @@ static void bestBtreeIndex( ** with this step if we already know this index will not be chosen. ** Also, never reduce the output row count below 2 using this step. ** - ** Do not reduce the output row count if pSrc is the only table that - ** is notReady; if notReady is a power of two. This will be the case - ** when the main sqlite3WhereBegin() loop is scanning for a table with - ** and "optimal" index, and on such a scan the output row count - ** reduction is not valid because it does not update the "pCost->used" - ** bitmap. The notReady bitmap will also be a power of two when we - ** are scanning for the last table in a 64-way join. We are willing - ** to bypass this optimization in that corner case. + ** It is critical that the notValid mask be used here instead of + ** the notReady mask. When computing an "optimal" index, the notReady + ** mask will only have one bit set - the bit for the current table. + ** The notValid mask, on the other hand, always has all bits set for + ** tables that are not in outer loops. If notReady is used here instead + ** of notValid, then a optimal index that depends on inner joins loops + ** might be selected even when there exists an optimal index that has + ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){ + if( nRow>2 && cost<=pCost->rCost ){ int k; /* Loop counter */ int nSkipEq = nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ @@ -96942,7 +97522,7 @@ static void bestBtreeIndex( thisTab = getMask(pWC->pMaskSet, iCur); for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notReady)!=thisTab ) continue; + if( (pTerm->prereqAll & notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ /* Ignore the first nEq equality matches since the index @@ -97024,7 +97604,7 @@ static void bestBtreeIndex( pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") )); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); pCost->plan.wsFlags |= eqTermMask; } @@ -97039,14 +97619,15 @@ static void bestIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pSrc->pTab) ){ sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p); + bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); if( p->needToFreeIdxStr ){ sqlite3_free(p->idxStr); } @@ -97054,7 +97635,7 @@ static void bestIndex( }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } } @@ -98270,9 +98851,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** other FROM clause terms that are notReady. If no notReady terms are ** used then the "optimal" query plan works. ** + ** Note that the WhereCost.nRow parameter for an optimal scan might + ** not be as small as it would be if the table really were the innermost + ** join. The nRow value can be reduced by WHERE clause constraints + ** that do not use indices. But this nRow reduction only happens if the + ** table really is the innermost join. + ** ** The second loop iteration is only performed if no optimal scan - ** strategies were found by the first loop. This 2nd iteration is used to - ** search for the lowest cost scan overall. + ** strategies were found by the first iteration. This second iteration + ** is used to search for the lowest cost scan overall. ** ** Previous versions of SQLite performed only the second iteration - ** the next outermost loop was always that with the lowest overall @@ -98285,14 +98872,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** ** The best strategy is to iterate through table t1 first. However it ** is not possible to determine this with a simple greedy algorithm. - ** However, since the cost of a linear scan through table t2 is the same + ** Since the cost of a linear scan through table t2 is the same ** as the cost of a linear scan through table t1, a simple greedy ** algorithm may choose to use t2 for the outer loop, which is a much ** costlier approach. */ nUnconstrained = 0; notIndexed = 0; - for(isOptimal=(iFrom=0; isOptimal--){ + for(isOptimal=(iFrom=0 && bestJ<0; isOptimal--){ Bitmask mask; /* Mask of tables not yet ready */ for(j=iFrom, pTabItem=&pTabList->a[j]; jpTab) ){ sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp); + bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost, pp); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost); + bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); @@ -103354,15 +103943,33 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); /************** Continuing where we left off in main.c ***********************/ #endif -/* -** The version of the library -*/ #ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; #endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } #if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) @@ -103483,6 +104090,13 @@ SQLITE_API int sqlite3_initialize(void){ ** sqlite3_initialize(). The recursive calls normally come through ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ @@ -103763,12 +104377,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ sz = 0; pStart = 0; }else if( pBuf==0 ){ - sz = ROUND8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); }else{ - sz = ROUNDDOWN8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ pStart = pBuf; } db->lookaside.pStart = pStart; @@ -103811,14 +104425,14 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); - int sz = va_arg(ap, int); - int cnt = va_arg(ap, int); + void *pBuf = va_arg(ap, void*); /* IMP: R-21112-12275 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ rc = setupLookaside(db, pBuf, sz, cnt); break; } default: { - rc = SQLITE_ERROR; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ break; } } @@ -103923,11 +104537,28 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ db->isTransactionSavepoint = 0; } +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + /* ** Close an existing SQLite database */ SQLITE_API int sqlite3_close(sqlite3 *db){ - HashElem *i; + HashElem *i; /* Hash table iterator */ int j; if( !db ){ @@ -103995,6 +104626,7 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ for(p=db->aFunc.a[j]; p; p=pHash){ pHash = p->pHash; while( p ){ + functionDestroy(db, p); pNext = p->pNext; sqlite3DbFree(db, p); p = pNext; @@ -104269,7 +104901,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( void *pUserData, void (*xFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor ){ FuncDef *p; int nName; @@ -104297,10 +104930,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; @@ -104333,6 +104966,15 @@ SQLITE_PRIVATE int sqlite3CreateFunc( if( !p ){ return SQLITE_NOMEM; } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; p->flags = 0; p->xFunc = xFunc; p->xStep = xStep; @@ -104347,7 +104989,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( */ SQLITE_API int sqlite3_create_function( sqlite3 *db, - const char *zFunctionName, + const char *zFunc, int nArg, int enc, void *p, @@ -104355,9 +104997,41 @@ SQLITE_API int sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - int rc; + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; sqlite3_mutex_enter(db->mutex); - rc = sqlite3CreateFunc(db, zFunctionName, nArg, enc, p, xFunc, xStep, xFinal); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -104379,7 +105053,7 @@ SQLITE_API int sqlite3_create_function16( sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -104410,7 +105084,7 @@ SQLITE_API int sqlite3_overload_function( sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0); + 0, sqlite3InvalidFunction, 0, 0, 0); } rc = sqlite3ApiExit(db, SQLITE_OK); sqlite3_mutex_leave(db->mutex); @@ -104548,7 +105222,10 @@ SQLITE_PRIVATE int sqlite3WalDefaultHook( ** configured by this function. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifndef SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -104678,60 +105355,6 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ #endif } -/* -** This routine is called to create a connection to a database BTree -** driver. If zFilename is the name of a file, then that file is -** opened and used. If zFilename is the magic name ":memory:" then -** the database is stored in memory (and is thus forgotten as soon as -** the connection is closed.) If zFilename is NULL then the database -** is a "virtual" database for transient use only and is deleted as -** soon as the connection is closed. -** -** A virtual database can be either a disk file (that is automatically -** deleted when the file is closed) or it an be held entirely in memory. -** The sqlite3TempInMemory() function is used to determine which. -*/ -SQLITE_PRIVATE int sqlite3BtreeFactory( - sqlite3 *db, /* Main database when opening aux otherwise 0 */ - const char *zFilename, /* Name of the file containing the BTree database */ - int omitJournal, /* if TRUE then do not journal this file */ - int nCache, /* How many pages in the page cache */ - int vfsFlags, /* Flags passed through to vfsOpen */ - Btree **ppBtree /* Pointer to new Btree object written here */ -){ - int btFlags = 0; - int rc; - - assert( sqlite3_mutex_held(db->mutex) ); - assert( ppBtree != 0); - if( omitJournal ){ - btFlags |= BTREE_OMIT_JOURNAL; - } - if( db->flags & SQLITE_NoReadlock ){ - btFlags |= BTREE_NO_READLOCK; - } -#ifndef SQLITE_OMIT_MEMORYDB - if( zFilename==0 && sqlite3TempInMemory(db) ){ - zFilename = ":memory:"; - } -#endif - - if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (zFilename==0 || *zFilename==0) ){ - vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; - } - rc = sqlite3BtreeOpen(zFilename, (sqlite3 *)db, ppBtree, btFlags, vfsFlags); - - /* If the B-Tree was successfully opened, set the pager-cache size to the - ** default value. Except, if the call to BtreeOpen() returned a handle - ** open on an existing shared pager-cache, do not change the pager-cache - ** size. - */ - if( rc==SQLITE_OK && 0==sqlite3BtreeSchema(*ppBtree, 0, 0) ){ - sqlite3BtreeSetCacheSize(*ppBtree, nCache); - } - return rc; -} - /* ** Return UTF-8 encoded English language explanation of the most recent ** error. @@ -104974,17 +105597,39 @@ static const int aHardLimit[] = { */ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; + + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ return -1; } oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ } db->aLimit[limitId] = newLimit; } - return oldLimit; + return oldLimit; /* IMP: R-53341-35419 */ } /* @@ -105008,6 +105653,24 @@ static int openDatabase( if( rc ) return rc; #endif + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; }else if( flags & SQLITE_OPEN_NOMUTEX ){ @@ -105041,7 +105704,8 @@ static int openDatabase( SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_MASTER_JOURNAL | SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL ); /* Allocate the sqlite data structure */ @@ -105113,9 +105777,8 @@ static int openDatabase( /* Open the backend database driver */ db->openFlags = flags; - rc = sqlite3BtreeFactory(db, zFilename, 0, SQLITE_DEFAULT_CACHE_SIZE, - flags | SQLITE_OPEN_MAIN_DB, - &db->aDb[0].pBt); + rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; @@ -105822,6 +106485,22 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -107010,6 +107689,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -108959,6 +109639,9 @@ static int fts3FilterMethod( return rc; } + rc = sqlite3Fts3ReadLock(p); + if( rc!=SQLITE_OK ) return rc; + rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; @@ -109337,11 +110020,14 @@ static int fts3RenameMethod( const char *zName /* New name of table */ ){ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db; /* Database connection */ + sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ - - db = p->db; - rc = SQLITE_OK; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ){ + return rc; + } + fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName @@ -112501,6 +113187,36 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( return SQLITE_OK; } +/* +** This function ensures that the caller has obtained a shared-cache +** table-lock on the %_content table. This is required before reading +** data from the fts3 table. If this lock is not acquired first, then +** the caller may end up holding read-locks on the %_segments and %_segdir +** tables, but no read-lock on the %_content table. If this happens +** a second connection will be able to write to the fts3 table, but +** attempting to commit those writes might return SQLITE_LOCKED or +** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain +** write-locks on the %_segments and %_segdir ** tables). +** +** We try to avoid this because if FTS3 returns any error when committing +** a transaction, the whole transaction will be rolled back. And this is +** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can +** still happen if the user reads data directly from the %_segments or +** %_segdir tables instead of going through FTS3 though. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ + int rc; /* Return code */ + sqlite3_stmt *pStmt; /* Statement used to obtain lock */ + + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -115992,6 +116708,45 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * ** algorithms packaged as an SQLite virtual table module. */ +/* +** Database Format of R-Tree Tables +** -------------------------------- +** +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. +** +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. +*/ + #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) /* @@ -116032,6 +116787,9 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * #define AssignCells splitNodeStartree #endif +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 @@ -116040,6 +116798,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * #ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; typedef unsigned int u32; @@ -116050,6 +116809,8 @@ typedef struct RtreeCursor RtreeCursor; typedef struct RtreeNode RtreeNode; typedef struct RtreeCell RtreeCell; typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ @@ -116119,6 +116880,15 @@ struct Rtree { #define RTREE_REINSERT(p) RTREE_MINCELLS(p) #define RTREE_MAXCELLS 51 +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + /* ** An rtree cursor object. */ @@ -116151,35 +116921,23 @@ union RtreeCoord { ** A search constraint. */ struct RtreeConstraint { - int iCoord; /* Index of constrained coordinate */ - int op; /* Constraining operation */ - double rValue; /* Constraint value. */ + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + double rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 +#define RTREE_EQ 0x41 +#define RTREE_LE 0x42 +#define RTREE_LT 0x43 +#define RTREE_GE 0x44 +#define RTREE_GT 0x45 +#define RTREE_MATCH 0x46 /* ** An rtree structure node. -** -** Data format (RtreeNode.zData): -** -** 1. If the node is the root node (node 1), then the first 2 bytes -** of the node contain the tree depth as a big-endian integer. -** For non-root nodes, the first 2 bytes are left unused. -** -** 2. The next 2 bytes contain the number of entries currently -** stored in the node. -** -** 3. The remainder of the node contains the node entries. Each entry -** consists of a single 8-byte integer followed by an even number -** of 4-byte coordinates. For leaf nodes the integer is the rowid -** of a record. For internal nodes it is the node number of a -** child page. */ struct RtreeNode { RtreeNode *pParent; /* Parent node */ @@ -116199,6 +116957,40 @@ struct RtreeCell { RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; }; + +/* +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). +*/ +#define RTREE_GEOMETRY_MAGIC 0x891245AB + +/* +** An instance of this structure must be supplied as a blob argument to +** the right-hand-side of an SQL MATCH operator used to constrain an +** r-tree query. +*/ +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; + int nParam; + double aParam[1]; +}; + +/* +** When a geometry callback is created (see sqlite3_rtree_geometry_callback), +** a single instance of the following structure is allocated. It is used +** as the context for the user-function created by by s_r_g_c(). The object +** is eventually deleted by the destructor mechanism provided by +** sqlite3_create_function_v2() (which is called by s_r_g_c() to create +** the geometry callback function). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; +}; + #ifndef MAX # define MAX(x,y) ((x) < (y) ? (y) : (x)) #endif @@ -116281,10 +117073,8 @@ static void nodeReference(RtreeNode *p){ ** Clear the content of node p (set all bytes to 0x00). */ static void nodeZero(Rtree *pRtree, RtreeNode *p){ - if( p ){ - memset(&p->zData[2], 0, pRtree->iNodeSize-2); - p->isDirty = 1; - } + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; } /* @@ -116304,7 +117094,6 @@ static int nodeHash(i64 iNode){ */ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ RtreeNode *p; - assert( iNode!=0 ); for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); return p; } @@ -116313,13 +117102,11 @@ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ ** Add node pNode to the node hash table. */ static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ - if( pNode ){ - int iHash; - assert( pNode->pNext==0 ); - iHash = nodeHash(pNode->iNode); - pNode->pNext = pRtree->aHash[iHash]; - pRtree->aHash[iHash] = pNode; - } + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; } /* @@ -116341,11 +117128,11 @@ static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ ** assigned a node number when nodeWrite() is called to write the ** node contents out to the database. */ -static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent, int zero){ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ RtreeNode *pNode; pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); if( pNode ){ - memset(pNode, 0, sizeof(RtreeNode) + (zero?pRtree->iNodeSize:0)); + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); pNode->zData = (u8 *)&pNode[1]; pNode->nRef = 1; pNode->pParent = pParent; @@ -116366,6 +117153,7 @@ nodeAcquire( RtreeNode **ppNode /* OUT: Acquired node */ ){ int rc; + int rc2 = SQLITE_OK; RtreeNode *pNode; /* Check if the requested node is already in the hash table. If so, @@ -116382,40 +117170,64 @@ nodeAcquire( return SQLITE_OK; } - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); - if( !pNode ){ - *ppNode = 0; - return SQLITE_NOMEM; - } - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); rc = sqlite3_step(pRtree->pReadNode); if( rc==SQLITE_ROW ){ const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - assert( sqlite3_column_bytes(pRtree->pReadNode, 0)==pRtree->iNodeSize ); - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); + } + } + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT; + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT; + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT; + } + *ppNode = pNode; }else{ sqlite3_free(pNode); - pNode = 0; + *ppNode = 0; } - *ppNode = pNode; - rc = sqlite3_reset(pRtree->pReadNode); - - if( rc==SQLITE_OK && iNode==1 ){ - pRtree->iDepth = readInt16(pNode->zData); - } - - assert( (rc==SQLITE_OK && pNode) || (pNode==0 && rc!=SQLITE_OK) ); - nodeHashInsert(pRtree, pNode); - return rc; } @@ -116467,8 +117279,7 @@ nodeInsertCell( nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; nCell = NCELL(pNode); - assert(nCell<=nMaxCell); - + assert( nCell<=nMaxCell ); if( nCellzData[2], nCell+1); @@ -116688,6 +117499,25 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return rc; } + +/* +** Free the RtreeCursor.aConstraint[] array and its contents. +*/ +static void freeCursorConstraints(RtreeCursor *pCsr){ + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; + if( pGeom ){ + if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); + sqlite3_free(pGeom); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } +} + /* ** Rtree virtual table module xClose method. */ @@ -116695,7 +117525,7 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int rc; RtreeCursor *pCsr = (RtreeCursor *)cur; - sqlite3_free(pCsr->aConstraint); + freeCursorConstraints(pCsr); rc = nodeRelease(pRtree, pCsr->pNode); sqlite3_free(pCsr); return rc; @@ -116712,13 +117542,39 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ return (pCsr->pNode==0); } +/* +** The r-tree constraint passed as the second argument to this function is +** guaranteed to be a MATCH constraint. +*/ +static int testRtreeGeom( + Rtree *pRtree, /* R-Tree object */ + RtreeConstraint *pConstraint, /* MATCH constraint to test */ + RtreeCell *pCell, /* Cell to test */ + int *pbRes /* OUT: Test result */ +){ + int i; + double aCoord[RTREE_MAX_DIMENSIONS*2]; + int nCoord = pRtree->nDim*2; + + assert( pConstraint->op==RTREE_MATCH ); + assert( pConstraint->pGeom ); + + for(i=0; iaCoord[i]); + } + return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); +} + /* ** Cursor pCursor currently points to a cell in a non-leaf page. -** Return true if the sub-tree headed by the cell is filtered +** Set *pbEof to true if the sub-tree headed by the cell is filtered ** (excluded) by the constraints in the pCursor->aConstraint[] ** array, or false otherwise. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; int bRes = 0; @@ -116730,31 +117586,55 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){ double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ - case RTREE_LE: case RTREE_LT: bRes = p->rValuerValue>cell_max; break; - case RTREE_EQ: + case RTREE_LE: case RTREE_LT: + bRes = p->rValuerValue>cell_max; + break; + + case RTREE_EQ: bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &bRes); + if( rc!=SQLITE_OK ){ + return rc; + } + bRes = !bRes; + break; + } } } - return bRes; + *pbEof = bRes; + return SQLITE_OK; } /* -** Return true if the cell that cursor pCursor currently points to +** Test if the cell that cursor pCursor currently points to ** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array, or false otherwise. +** pCursor->aConstraint[] array. If so, set *pbEof to true before +** returning. If the cell is not filtered (excluded) by the constraints, +** set pbEof to zero. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. ** ** This function assumes that the cell is part of a leaf node. */ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; + *pbEof = 0; nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; iinConstraint; ii++){ @@ -116762,7 +117642,7 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ double coord = DCOORD(cell.aCoord[p->iCoord]); int res; assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ case RTREE_LE: res = (coord<=p->rValue); break; @@ -116770,12 +117650,24 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ case RTREE_GE: res = (coord>=p->rValue); break; case RTREE_GT: res = (coord>p->rValue); break; case RTREE_EQ: res = (coord==p->rValue); break; + default: { + int rc; + assert( p->op==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &res); + if( rc!=SQLITE_OK ){ + return rc; + } + break; + } } - if( !res ) return 1; + if( !res ){ + *pbEof = 1; + return SQLITE_OK; + } } - return 0; + return SQLITE_OK; } /* @@ -116802,13 +117694,13 @@ static int descendToCell( assert( iHeight>=0 ); if( iHeight==0 ){ - isEof = testRtreeEntry(pRtree, pCursor); + rc = testRtreeEntry(pRtree, pCursor, &isEof); }else{ - isEof = testRtreeCell(pRtree, pCursor); + rc = testRtreeCell(pRtree, pCursor, &isEof); } - if( isEof || iHeight==0 ){ + if( rc!=SQLITE_OK || isEof || iHeight==0 ){ *pEof = isEof; - return SQLITE_OK; + return rc; } iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); @@ -116844,24 +117736,34 @@ static int descendToCell( ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. */ -static int nodeRowidIndex(Rtree *pRtree, RtreeNode *pNode, i64 iRowid){ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ int ii; - for(ii=0; nodeGetRowid(pRtree, pNode, ii)!=iRowid; ii++){ - assert( ii<(NCELL(pNode)-1) ); + int nCell = NCELL(pNode); + for(ii=0; iipParent; if( pParent ){ - return nodeRowidIndex(pRtree, pParent, pNode->iNode); + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); } - return -1; + *piIndex = -1; + return SQLITE_OK; } /* @@ -116872,13 +117774,17 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; + /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is + ** already at EOF. It is against the rules to call the xNext() method of + ** a cursor that has already reached EOF. + */ + assert( pCsr->pNode ); + if( pCsr->iStrategy==1 ){ /* This "scan" is a direct lookup by rowid. There is no next entry. */ nodeRelease(pRtree, pCsr->pNode); pCsr->pNode = 0; - } - - else if( pCsr->pNode ){ + }else{ /* Move to the next entry that matches the configured constraints. */ int iHeight = 0; while( pCsr->pNode ){ @@ -116892,7 +117798,10 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ } } pCsr->pNode = pNode->pParent; - pCsr->iCell = nodeParentIndex(pRtree, pNode); + rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); + if( rc!=SQLITE_OK ){ + return rc; + } nodeReference(pCsr->pNode); nodeRelease(pRtree, pNode); iHeight++; @@ -116960,6 +117869,51 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ return rc; } +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *p; + sqlite3_rtree_geometry *pGeom; + int nBlob; + + /* Check that value is actually a blob. */ + if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlobmagic!=RTREE_GEOMETRY_MAGIC + || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) + ){ + sqlite3_free(pGeom); + return SQLITE_ERROR; + } + + pGeom->pContext = p->pContext; + pGeom->nParam = p->nParam; + pGeom->aParam = p->aParam; + + pCons->xGeom = p->xGeom; + pCons->pGeom = pGeom; + return SQLITE_OK; +} /* ** Rtree virtual table module xFilter method. @@ -116978,8 +117932,7 @@ static int rtreeFilter( rtreeReference(pRtree); - sqlite3_free(pCsr->aConstraint); - pCsr->aConstraint = 0; + freeCursorConstraints(pCsr); pCsr->iStrategy = idxNum; if( idxNum==1 ){ @@ -116988,8 +117941,9 @@ static int rtreeFilter( i64 iRowid = sqlite3_value_int64(argv[0]); rc = findLeafNode(pRtree, iRowid, &pLeaf); pCsr->pNode = pLeaf; - if( pLeaf && rc==SQLITE_OK ){ - pCsr->iCell = nodeRowidIndex(pRtree, pLeaf, iRowid); + if( pLeaf ){ + assert( rc==SQLITE_OK ); + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array @@ -117001,12 +117955,24 @@ static int rtreeFilter( if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); assert( (idxStr==0 && argc==0) || strlen(idxStr)==argc*2 ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; p->iCoord = idxStr[ii*2+1]-'a'; - p->rValue = sqlite3_value_double(argv[ii]); + if( p->op==RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + }else{ + p->rValue = sqlite3_value_double(argv[ii]); + } } } } @@ -117066,6 +118032,7 @@ static int rtreeFilter( ** < 0x43 ('C') ** >= 0x44 ('D') ** > 0x45 ('E') +** MATCH 0x46 ('F') ** ---------------------- ** ** The second of each pair of bytes identifies the coordinate column @@ -117104,7 +118071,9 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_OK; } - if( p->usable && p->iColumn>0 ){ + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + int j, opmsk; + static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; u8 op = 0; switch( p->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; @@ -117112,31 +118081,33 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + default: + assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; } - if( op ){ - /* Make sure this particular constraint has not been used before. - ** If it has been used before, ignore it. - ** - ** A <= or < can be used if there is a prior >= or >. - ** A >= or > can be used if there is a prior < or <=. - ** A <= or < is disqualified if there is a prior <=, <, or ==. - ** A >= or > is disqualified if there is a prior >=, >, or ==. - ** A == is disqualifed if there is any prior constraint. - */ - int j, opmsk; - static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; - assert( compatible[RTREE_EQ & 7]==0 ); - assert( compatible[RTREE_LT & 7]==1 ); - assert( compatible[RTREE_LE & 7]==1 ); - assert( compatible[RTREE_GT & 7]==2 ); - assert( compatible[RTREE_GE & 7]==2 ); - cCol = p->iColumn - 1 + 'a'; - opmsk = compatible[op & 7]; - for(j=0; j= or >. + ** A >= or > can be used if there is a prior < or <=. + ** A <= or < is disqualified if there is a prior <=, <, or ==. + ** A >= or > is disqualified if there is a prior >=, >, or ==. + ** A == is disqualifed if there is any prior constraint. + */ + assert( compatible[RTREE_EQ & 7]==0 ); + assert( compatible[RTREE_LT & 7]==1 ); + assert( compatible[RTREE_LE & 7]==1 ); + assert( compatible[RTREE_GT & 7]==2 ); + assert( compatible[RTREE_GE & 7]==2 ); + cCol = p->iColumn - 1 + 'a'; + opmsk = compatible[op & 7]; + for(j=0; jnDim*2); jj+=2){ @@ -117337,22 +118313,31 @@ static int ChooseLeaf( ** the smallest area. */ for(iCell=0; iCelliDepth-1) ){ overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); } -#endif if( (iCell==0) || (overlappParent ){ - RtreeCell cell; RtreeNode *pParent = p->pParent; - int iCell = nodeParentIndex(pRtree, p); + RtreeCell cell; + int iCell; + + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT; + } nodeGetCell(pRtree, pParent, iCell, &cell); if( !cellContains(pRtree, &cell, pCell) ){ @@ -117394,6 +118383,7 @@ static void AdjustTree( p = pParent; } + return SQLITE_OK; } /* @@ -117922,14 +118912,14 @@ static int SplitNode( nCell++; if( pNode->iNode==1 ){ - pRight = nodeNew(pRtree, pNode, 1); - pLeft = nodeNew(pRtree, pNode, 1); + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); pRtree->iDepth++; pNode->isDirty = 1; writeInt16(pNode->zData, pRtree->iDepth); }else{ pLeft = pNode; - pRight = nodeNew(pRtree, pLeft->pParent, 1); + pRight = nodeNew(pRtree, pLeft->pParent); nodeReference(pLeft); } @@ -117946,8 +118936,12 @@ static int SplitNode( goto splitnode_out; } - /* Ensure both child nodes have node numbers assigned to them. */ - if( (0==pRight->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pRight))) + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) ){ goto splitnode_out; @@ -117963,9 +118957,15 @@ static int SplitNode( } }else{ RtreeNode *pParent = pLeft->pParent; - int iCell = nodeParentIndex(pRtree, pLeft); - nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); - AdjustTree(pRtree, pParent, &leftbbox); + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } } if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ goto splitnode_out; @@ -118009,20 +119009,43 @@ splitnode_out: return rc; } +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ int rc = SQLITE_OK; - if( pLeaf->iNode!=1 && pLeaf->pParent==0 ){ - sqlite3_bind_int64(pRtree->pReadParent, 1, pLeaf->iNode); - if( sqlite3_step(pRtree->pReadParent)==SQLITE_ROW ){ - i64 iNode = sqlite3_column_int64(pRtree->pReadParent, 0); - rc = nodeAcquire(pRtree, iNode, 0, &pLeaf->pParent); - }else{ - rc = SQLITE_ERROR; - } - sqlite3_reset(pRtree->pReadParent); - if( rc==SQLITE_OK ){ - rc = fixLeafParent(pRtree, pLeaf->pParent); + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + pChild = pChild->pParent; } return rc; } @@ -118031,18 +119054,24 @@ static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; + int rc2; RtreeNode *pParent; int iCell; assert( pNode->nRef==1 ); /* Remove the entry in the parent cell. */ - iCell = nodeParentIndex(pRtree, pNode); - pParent = pNode->pParent; - pNode->pParent = 0; - if( SQLITE_OK!=(rc = deleteCell(pRtree, pParent, iCell, iHeight+1)) - || SQLITE_OK!=(rc = nodeRelease(pRtree, pParent)) - ){ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ return rc; } @@ -118072,8 +119101,9 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ return SQLITE_OK; } -static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; if( pParent ){ int ii; int nCell = NCELL(pNode); @@ -118085,10 +119115,13 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ cellUnion(pRtree, &box, &cell); } box.iRowid = pNode->iNode; - ii = nodeParentIndex(pRtree, pNode); - nodeOverwriteCell(pRtree, pParent, &box, ii); - fixBoundingBox(pRtree, pParent); + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } } + return rc; } /* @@ -118096,6 +119129,7 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ ** cell, adjust the r-tree data structure if required. */ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; int rc; if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ @@ -118112,14 +119146,13 @@ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ ** cell in the parent node so that it tightly contains the updated ** node. */ - if( pNode->iNode!=1 ){ - RtreeNode *pParent = pNode->pParent; - if( (pParent->iNode!=1 || NCELL(pParent)!=1) - && (NCELL(pNode)pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)iNode currently contains @@ -118256,11 +119289,13 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); #endif }else{ - AdjustTree(pRtree, pNode, pCell); - if( iHeight==0 ){ - rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } } } return rc; @@ -118330,7 +119365,6 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - assert(hashIsEmpty(pRtree)); /* If azData[0] is not an SQL NULL value, it is the rowid of a ** record to delete from the r-tree table. The following block does @@ -118356,8 +119390,10 @@ static int rtreeUpdate( /* Delete the cell in question from the leaf node. */ if( rc==SQLITE_OK ){ int rc2; - iCell = nodeRowidIndex(pRtree, pLeaf, iDelete); - rc = deleteCell(pRtree, pLeaf, iCell, 0); + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ rc = rc2; @@ -118379,19 +119415,20 @@ static int rtreeUpdate( ** the root node (the operation that Gutman's paper says to perform ** in this scenario). */ - if( rc==SQLITE_OK && pRtree->iDepth>0 ){ - if( rc==SQLITE_OK && NCELL(pRoot)==1 ){ - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; } } @@ -118681,7 +119718,7 @@ static int rtreeInit( Rtree *pRtree; int nDb; /* Length of string argv[1] */ int nName; /* Length of string argv[2] */ - int eCoordType = (int)pAux; + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); const char *aErrMsg[] = { 0, /* 0 */ @@ -118827,12 +119864,10 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ ** function "rtreenode". */ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ - int rc = SQLITE_OK; + const int utf8 = SQLITE_UTF8; + int rc; - if( rc==SQLITE_OK ){ - int utf8 = SQLITE_UTF8; - rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); - } + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); if( rc==SQLITE_OK ){ int utf8 = SQLITE_UTF8; rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); @@ -118849,6 +119884,70 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ return rc; } +/* +** A version of sqlite3_free() that can be used as a callback. This is used +** in two places - as the destructor for the blob value returned by the +** invocation of a geometry function, and as the destructor for the geometry +** functions themselves. +*/ +static void doSqlite3Free(void *p){ + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite +** scalar user function. This C function is the callback used for all such +** registered SQL functions. +** +** The scalar user functions return a blob that is interpreted by r-tree +** table MATCH operators. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->xGeom = pGeomCtx->xGeom; + pBlob->pContext = pGeomCtx->pContext; + pBlob->nParam = nArg; + for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); + } + sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *), + void *pContext +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->pContext = pContext; + + /* Create the new user-function. Register a destructor function to delete + ** the context object when it is no longer required. */ + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + ); +} + #if !SQLITE_CORE SQLITE_API int sqlite3_extension_init( sqlite3 *db, diff --git a/db/sqlite3/src/sqlite3.h b/db/sqlite3/src/sqlite3.h index b831fe5fc86..d78ae39b880 100644 --- a/db/sqlite3/src/sqlite3.h +++ b/db/sqlite3/src/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.1" -#define SQLITE_VERSION_NUMBER 3007001 -#define SQLITE_SOURCE_ID "2010-08-21 16:01:46 3613b0695a5e990905ab146fadcab34dd04d5874" +#define SQLITE_VERSION "3.7.3" +#define SQLITE_VERSION_NUMBER 3007003 +#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -757,15 +757,19 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** @@ -776,7 +780,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** **
      @@ -787,7 +791,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; **
    • [SQLITE_OPEN_TRANSIENT_DB] **
    • [SQLITE_OPEN_SUBJOURNAL] **
    • [SQLITE_OPEN_MASTER_JOURNAL] -**
    +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application @@ -806,10 +811,11 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the @@ -818,7 +824,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that @@ -828,13 +834,13 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is @@ -848,10 +854,10 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current @@ -1248,7 +1254,7 @@ struct sqlite3_mem_methods { **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] -**
    • [sqlite3_soft_heap_limit()] +**
    • [sqlite3_soft_heap_limit64()] **
    • [sqlite3_status()] **
    )^ ** ^Memory allocation statistics are enabled by default unless SQLite is @@ -1262,15 +1268,14 @@ struct sqlite3_mem_methods { ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
    • ** **
    SQLITE_CONFIG_PAGECACHE
    @@ -1290,8 +1295,7 @@ struct sqlite3_mem_methods { ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined. ** @@ -1420,8 +1424,14 @@ struct sqlite3_mem_methods { ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE] +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^ ** **
    */ @@ -1726,6 +1736,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries ** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. @@ -1746,7 +1759,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** ** 
    @@ -1770,7 +1783,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
 **        azResult[5] = "28";
 **        azResult[6] = "Cindy";
 **        azResult[7] = "21";
-**
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 @@ -1778,19 +1791,19 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ @@ -1942,7 +1955,9 @@ SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in @@ -2200,17 +2215,28 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. @@ -2269,7 +2295,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2394,17 +2420,22 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a @@ -2433,7 +2464,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ @@ -2451,7 +2482,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -2464,8 +2497,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [GLOB] operators.)^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ @@ -2537,12 +2569,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
    • ** **
  • @@ -2555,11 +2582,16 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
    • ** **
  • -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
    • ** */ @@ -2626,7 +2658,7 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the @@ -2821,6 +2853,8 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); @@ -3011,8 +3045,14 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); @@ -3092,18 +3132,26 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object @@ -3148,10 +3196,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** **
      @@ -3164,22 +3212,22 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); **
    • The initial content is UTF-16 text and sqlite3_column_bytes() or ** sqlite3_column_text() is called. The content must be converted ** to UTF-8.
      • -**
    )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** **
      **
    • sqlite3_column_text() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_blob() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_text16() followed by sqlite3_column_bytes16()
      • -**
    )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result @@ -3217,17 +3265,26 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); @@ -3263,23 +3320,25 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or @@ -3289,10 +3348,10 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. @@ -3304,13 +3363,21 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is invoked when the function is deleted, either by being +** overloaded or when the database connection closes. +** ^When the destructure callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the pointer which was +** the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -3326,11 +3393,6 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not @@ -3357,6 +3419,17 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -3703,46 +3776,70 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. ** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. +** ^(The third argument (eTextRep) must be one of the constants: +**
      +**
    • [SQLITE_UTF8], +**
    • [SQLITE_UTF16LE], +**
    • [SQLITE_UTF16BE], +**
    • [SQLITE_UTF16], or +**
    • [SQLITE_UTF16_ALIGNED]. +**
    )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. ** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it any more). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. ** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ @@ -3750,14 +3847,14 @@ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); @@ -3765,7 +3862,7 @@ SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); @@ -3854,16 +3951,19 @@ SQLITE_API void sqlite3_activate_cerod( /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); @@ -4085,40 +4185,73 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. ** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. ** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. +** ^If the argument N is zero then the soft heap limit is disabled. ** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: ** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. +**
      +**
    • The soft heap limit is set to zero. +**
    • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
    • An alternative page cache implementation is specifed using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). +**
    • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
    )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. */ -SQLITE_API void sqlite3_soft_heap_limit(int); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table @@ -4242,34 +4375,47 @@ SQLITE_API int sqlite3_load_extension( SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions +** CAPI3REF: Automatically Load Statically Linked Extensions ** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. ** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: ** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +** 
    +**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); @@ -4908,7 +5054,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as @@ -5105,7 +5251,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status @@ -5124,7 +5271,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be @@ -5174,7 +5321,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to @@ -5197,7 +5344,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the @@ -5246,6 +5393,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. ** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5372,32 +5522,42 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. @@ -5405,47 +5565,50 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache @@ -5454,36 +5617,35 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** @@ -5961,3 +6123,59 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + diff --git a/storage/public/Makefile.in b/storage/public/Makefile.in index 3f0e91a8fe1..928079cfac3 100644 --- a/storage/public/Makefile.in +++ b/storage/public/Makefile.in @@ -48,6 +48,8 @@ MODULE = storage XPIDL_MODULE = storage GRE_MODULE = 1 +# NOTE When adding something to this list, you probably need to add it to the +# storage.h file too. XPIDLSRCS = \ mozIStorageService.idl \ mozIStorageConnection.idl \ @@ -70,6 +72,7 @@ XPIDLSRCS = \ mozIStorageServiceQuotaManagement.idl \ mozIStorageVacuumParticipant.idl \ $(NULL) +# SEE ABOVE NOTE! EXPORTS_NAMESPACES = mozilla diff --git a/storage/public/storage.h b/storage/public/storage.h index e36fcc2c35d..7466c55c1a3 100644 --- a/storage/public/storage.h +++ b/storage/public/storage.h @@ -59,6 +59,8 @@ #include "mozIStorageBindingParams.h" #include "mozIStorageServiceQuotaManagement.h" #include "mozIStorageVacuumParticipant.h" +#include "mozIStorageCompletionCallback.h" +#include "mozIStorageAsyncStatement.h" //////////////////////////////////////////////////////////////////////////////// //// Native Language Helpers diff --git a/storage/src/mozStorageAsyncStatement.cpp b/storage/src/mozStorageAsyncStatement.cpp index efc7196c1a3..06cdbc4eac7 100644 --- a/storage/src/mozStorageAsyncStatement.cpp +++ b/storage/src/mozStorageAsyncStatement.cpp @@ -271,7 +271,8 @@ AsyncStatement::~AsyncStatement() // nsCOMPtr. Which it is not; it's an nsRefPtr. Connection *forgottenConn = nsnull; mDBConnection.swap(forgottenConn); - (void)::NS_ProxyRelease(forgottenConn->threadOpenedOn, forgottenConn); + (void)::NS_ProxyRelease(forgottenConn->threadOpenedOn, + static_cast(forgottenConn)); } } diff --git a/storage/src/mozStorageConnection.cpp b/storage/src/mozStorageConnection.cpp index d9cf2eb7620..6ce87907409 100644 --- a/storage/src/mozStorageConnection.cpp +++ b/storage/src/mozStorageConnection.cpp @@ -322,7 +322,7 @@ public: return NS_OK; } private: - nsCOMPtr mConnection; + nsRefPtr mConnection; nsCOMPtr mCallingThread; nsCOMPtr mCallbackEvent; }; @@ -352,9 +352,10 @@ Connection::~Connection() (void)Close(); } -NS_IMPL_THREADSAFE_ISUPPORTS1( +NS_IMPL_THREADSAFE_ISUPPORTS2( Connection, - mozIStorageConnection + mozIStorageConnection, + nsIInterfaceRequestor ) nsIEventTarget * @@ -642,6 +643,22 @@ Connection::getFilename() return leafname; } +//////////////////////////////////////////////////////////////////////////////// +//// nsIInterfaceRequestor + +NS_IMETHODIMP +Connection::GetInterface(const nsIID &aIID, + void **_result) +{ + if (aIID.Equals(NS_GET_IID(nsIEventTarget))) { + nsIEventTarget *background = getAsyncExecutionTarget(); + NS_IF_ADDREF(background); + *_result = background; + return NS_OK; + } + return NS_ERROR_NO_INTERFACE; +} + //////////////////////////////////////////////////////////////////////////////// //// mozIStorageConnection diff --git a/storage/src/mozStorageConnection.h b/storage/src/mozStorageConnection.h index 9394de97301..8095ebab4fe 100644 --- a/storage/src/mozStorageConnection.h +++ b/storage/src/mozStorageConnection.h @@ -38,12 +38,13 @@ * * ***** END LICENSE BLOCK ***** */ -#ifndef MOZSTORAGECONNECTION_H -#define MOZSTORAGECONNECTION_H +#ifndef mozilla_storage_Connection_h +#define mozilla_storage_Connection_h #include "nsAutoPtr.h" #include "nsCOMPtr.h" #include "mozilla/Mutex.h" +#include "nsIInterfaceRequestor.h" #include "nsString.h" #include "nsDataHashtable.h" @@ -65,10 +66,12 @@ namespace mozilla { namespace storage { class Connection : public mozIStorageConnection + , public nsIInterfaceRequestor { public: NS_DECL_ISUPPORTS NS_DECL_MOZISTORAGECONNECTION + NS_DECL_NSIINTERFACEREQUESTOR /** * Structure used to describe user functions on the database connection. @@ -244,4 +247,4 @@ private: } // namespace storage } // namespace mozilla -#endif /* MOZSTORAGECONNECTION_H */ +#endif // mozilla_storage_Connection_h diff --git a/storage/test/test_true_async.cpp b/storage/test/test_true_async.cpp index f05018a4b17..23bffb0c1c3 100644 --- a/storage/test/test_true_async.cpp +++ b/storage/test/test_true_async.cpp @@ -39,6 +39,7 @@ #include "storage_test_harness.h" #include "prthread.h" #include "nsIEventTarget.h" +#include "nsIInterfaceRequestorUtils.h" #include "sqlite3.h" @@ -221,6 +222,14 @@ get_conn_async_thread(mozIStorageConnection *db) nsCOMPtr asyncThread; threadMan->GetThreadFromPRThread(last_non_watched_thread, getter_AddRefs(asyncThread)); + + // Additionally, check that the thread we get as the background thread is the + // same one as the one we report from getInterface. + nsCOMPtr target = do_GetInterface(db); + nsCOMPtr allegedAsyncThread = do_QueryInterface(target); + PRThread *allegedPRThread; + (void)allegedAsyncThread->GetPRThread(&allegedPRThread); + do_check_eq(allegedPRThread, last_non_watched_thread); return asyncThread.forget(); } diff --git a/storage/test/unit/test_storage_connection.js b/storage/test/unit/test_storage_connection.js index 3c08b8603b8..eebd4706b6e 100644 --- a/storage/test/unit/test_storage_connection.js +++ b/storage/test/unit/test_storage_connection.js @@ -482,6 +482,18 @@ function test_clone_copies_overridden_functions() run_next_test(); } +function test_getInterface() +{ + let db = getOpenedDatabase(); + let target = db.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIEventTarget); + // Just check that target is non-null. Other tests will ensure that it has + // the correct value. + do_check_true(target != null); + + run_next_test(); +} + //////////////////////////////////////////////////////////////////////////////// //// Test Runner @@ -513,6 +525,7 @@ var tests = [ test_close_clone_fails, test_clone_copies_functions, test_clone_copies_overridden_functions, + test_getInterface, ]; let index = 0; diff --git a/toolkit/components/places/src/AsyncFaviconHelpers.cpp b/toolkit/components/places/src/AsyncFaviconHelpers.cpp index 661f2eab01a..5e4c05e8e5e 100644 --- a/toolkit/components/places/src/AsyncFaviconHelpers.cpp +++ b/toolkit/components/places/src/AsyncFaviconHelpers.cpp @@ -74,14 +74,6 @@ _class::HandleError(mozIStorageError *aError) \ FAVICONSTEP_FAIL_IF_FALSE_RV(false, NS_OK); \ } -#define ASYNC_STATEMENT_EMPTY_HANDLERESULT_IMPL(_class) \ -NS_IMETHODIMP \ -_class::HandleResult(mozIStorageResultSet* aResultSet) \ -{ \ - NS_NOTREACHED("Got an unexpected result?");\ - return NS_OK; \ -} - #define CONTENT_SNIFFING_SERVICES "content-sniffing-services" /** @@ -95,14 +87,6 @@ _class::HandleResult(mozIStorageResultSet* aResultSet) \ namespace mozilla { namespace places { -//////////////////////////////////////////////////////////////////////////////// -//// AsyncFaviconStep - -NS_IMPL_ISUPPORTS0( - AsyncFaviconStep -) - - //////////////////////////////////////////////////////////////////////////////// //// AsyncFaviconStepper @@ -263,10 +247,6 @@ AsyncFaviconStepperInternal::Cancel(bool aNotify) //////////////////////////////////////////////////////////////////////////////// //// GetEffectivePageStep -NS_IMPL_ISUPPORTS_INHERITED0( - GetEffectivePageStep -, AsyncFaviconStep -) ASYNC_STATEMENT_HANDLEERROR_IMPL(GetEffectivePageStep) @@ -415,10 +395,6 @@ GetEffectivePageStep::CheckPageAndProceed() //////////////////////////////////////////////////////////////////////////////// //// FetchDatabaseIconStep -NS_IMPL_ISUPPORTS_INHERITED0( - FetchDatabaseIconStep -, AsyncFaviconStep -) ASYNC_STATEMENT_HANDLEERROR_IMPL(FetchDatabaseIconStep) @@ -515,12 +491,7 @@ FetchDatabaseIconStep::HandleCompletion(PRUint16 aReason) //////////////////////////////////////////////////////////////////////////////// //// EnsureDatabaseEntryStep -NS_IMPL_ISUPPORTS_INHERITED0( - EnsureDatabaseEntryStep -, AsyncFaviconStep -) ASYNC_STATEMENT_HANDLEERROR_IMPL(EnsureDatabaseEntryStep) -ASYNC_STATEMENT_EMPTY_HANDLERESULT_IMPL(EnsureDatabaseEntryStep) void @@ -780,12 +751,7 @@ FetchNetworkIconStep::AsyncOnChannelRedirect(nsIChannel* oldChannel, //////////////////////////////////////////////////////////////////////////////// //// SetFaviconDataStep -NS_IMPL_ISUPPORTS_INHERITED0( - SetFaviconDataStep -, AsyncFaviconStep -) ASYNC_STATEMENT_HANDLEERROR_IMPL(SetFaviconDataStep) -ASYNC_STATEMENT_EMPTY_HANDLERESULT_IMPL(SetFaviconDataStep) void @@ -883,12 +849,7 @@ SetFaviconDataStep::HandleCompletion(PRUint16 aReason) //////////////////////////////////////////////////////////////////////////////// //// AssociateIconWithPageStep -NS_IMPL_ISUPPORTS_INHERITED0( - AssociateIconWithPageStep -, AsyncFaviconStep -) ASYNC_STATEMENT_HANDLEERROR_IMPL(AssociateIconWithPageStep) -ASYNC_STATEMENT_EMPTY_HANDLERESULT_IMPL(AssociateIconWithPageStep) void @@ -956,12 +917,6 @@ AssociateIconWithPageStep::HandleCompletion(PRUint16 aReason) //////////////////////////////////////////////////////////////////////////////// //// NotifyStep -NS_IMPL_ISUPPORTS_INHERITED0( - NotifyStep -, AsyncFaviconStep -) - - void NotifyStep::Run() { diff --git a/toolkit/components/places/src/AsyncFaviconHelpers.h b/toolkit/components/places/src/AsyncFaviconHelpers.h index 4dc7e9f80c6..10606693d24 100644 --- a/toolkit/components/places/src/AsyncFaviconHelpers.h +++ b/toolkit/components/places/src/AsyncFaviconHelpers.h @@ -112,11 +112,9 @@ class AsyncFaviconStepper; * Executes a single async step on a favicon resource. * Once done, call backs to the stepper to proceed to the next step. */ -class AsyncFaviconStep : public nsISupports +class AsyncFaviconStep : public AsyncStatementCallback { public: - NS_DECL_ISUPPORTS - AsyncFaviconStep() {} /** @@ -136,7 +134,6 @@ protected: nsCOMPtr mStepper; }; - /** * Status definitions for the stepper. */ @@ -276,10 +273,8 @@ private: * associate it with. */ class GetEffectivePageStep : public AsyncFaviconStep - , public mozilla::places::AsyncStatementCallback { public: - NS_DECL_ISUPPORTS_INHERITED NS_DECL_MOZISTORAGESTATEMENTCALLBACK GetEffectivePageStep(); @@ -297,10 +292,8 @@ private: * Fetch an existing icon and associated information from the database. */ class FetchDatabaseIconStep : public AsyncFaviconStep - , public mozilla::places::AsyncStatementCallback { public: - NS_DECL_ISUPPORTS_INHERITED NS_DECL_MOZISTORAGESTATEMENTCALLBACK FetchDatabaseIconStep() {}; @@ -313,11 +306,10 @@ public: * Requires mDBInsertIcon statement. */ class EnsureDatabaseEntryStep : public AsyncFaviconStep - , public mozilla::places::AsyncStatementCallback { public: - NS_DECL_ISUPPORTS_INHERITED - NS_DECL_MOZISTORAGESTATEMENTCALLBACK + NS_IMETHOD HandleCompletion(PRUint16 aReason); + NS_IMETHOD HandleError(mozIStorageError* aError); EnsureDatabaseEntryStep() {}; void Run(); @@ -361,11 +353,10 @@ private: * Saves icon data in the database if it has changed. */ class SetFaviconDataStep : public AsyncFaviconStep - , public mozilla::places::AsyncStatementCallback { public: - NS_DECL_ISUPPORTS_INHERITED - NS_DECL_MOZISTORAGESTATEMENTCALLBACK + NS_IMETHOD HandleCompletion(PRUint16 aReason); + NS_IMETHOD HandleError(mozIStorageError* aError); SetFaviconDataStep() {}; void Run(); @@ -376,11 +367,10 @@ public: * Associate icon with page. */ class AssociateIconWithPageStep : public AsyncFaviconStep - , public mozilla::places::AsyncStatementCallback { public: - NS_DECL_ISUPPORTS_INHERITED - NS_DECL_MOZISTORAGESTATEMENTCALLBACK + NS_IMETHOD HandleCompletion(PRUint16 aReason); + NS_IMETHOD HandleError(mozIStorageError* aError); AssociateIconWithPageStep() {}; void Run(); @@ -393,8 +383,6 @@ public: class NotifyStep : public AsyncFaviconStep { public: - NS_DECL_ISUPPORTS_INHERITED - NotifyStep() {}; void Run(); }; diff --git a/toolkit/components/places/src/Helpers.cpp b/toolkit/components/places/src/Helpers.cpp index 67b9348113b..d7ac146b1a0 100644 --- a/toolkit/components/places/src/Helpers.cpp +++ b/toolkit/components/places/src/Helpers.cpp @@ -47,6 +47,24 @@ namespace places { //////////////////////////////////////////////////////////////////////////////// //// AsyncStatementCallback +NS_IMPL_ISUPPORTS1( + AsyncStatementCallback +, mozIStorageStatementCallback +) + +NS_IMETHODIMP +AsyncStatementCallback::HandleResult(mozIStorageResultSet *aResultSet) +{ + NS_ABORT_IF_FALSE(false, "Was not expecting a resultset, but got it."); + return NS_OK; +} + +NS_IMETHODIMP +AsyncStatementCallback::HandleCompletion(PRUint16 aReason) +{ + return NS_OK; +} + NS_IMETHODIMP AsyncStatementCallback::HandleError(mozIStorageError *aError) { diff --git a/toolkit/components/places/src/Helpers.h b/toolkit/components/places/src/Helpers.h index cf8b4f34a97..c00f9a44264 100644 --- a/toolkit/components/places/src/Helpers.h +++ b/toolkit/components/places/src/Helpers.h @@ -55,13 +55,17 @@ namespace places { class AsyncStatementCallback : public mozIStorageStatementCallback { public: - // Implement the error handler for asynchronous statements. - NS_IMETHOD HandleError(mozIStorageError *aError); + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGESTATEMENTCALLBACK + AsyncStatementCallback() {} + +protected: + virtual ~AsyncStatementCallback() {} }; /** - * Macro to use in place of NS_DECL_MOZISTORAGESTATEMENTCALLBACK to declare the - * methods this class does not implement. + * Macros to use in place of NS_DECL_MOZISTORAGESTATEMENTCALLBACK to declare the + * methods this class assumes silent or notreached. */ #define NS_DECL_ASYNCSTATEMENTCALLBACK \ NS_IMETHOD HandleResult(mozIStorageResultSet *); \ @@ -71,6 +75,24 @@ public: * Macros to use for lazy statements initialization in Places services that use * GetStatement() method. */ +#ifdef DEBUG +#define RETURN_IF_STMT(_stmt, _sql) \ + PR_BEGIN_MACRO \ + if (address_of(_stmt) == address_of(aStmt)) { \ + if (!_stmt) { \ + nsresult rv = mDBConn->CreateStatement(_sql, getter_AddRefs(_stmt)); \ + if (NS_FAILED(rv)) { \ + nsCAutoString err; \ + (void)mDBConn->GetLastErrorString(err); \ + (void)fprintf(stderr, "$$$ compiling statement failed with '%s'\n", \ + err.get()); \ + } \ + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && _stmt, nsnull); \ + } \ + return _stmt.get(); \ + } \ + PR_END_MACRO +#else #define RETURN_IF_STMT(_stmt, _sql) \ PR_BEGIN_MACRO \ if (address_of(_stmt) == address_of(aStmt)) { \ @@ -81,6 +103,7 @@ public: return _stmt.get(); \ } \ PR_END_MACRO +#endif // Async statements don't need to be scoped, they are reset when done. // So use this version for statements used async, scoped version for statements diff --git a/toolkit/components/places/src/History.cpp b/toolkit/components/places/src/History.cpp index 2979722d898..1f4276198f0 100644 --- a/toolkit/components/places/src/History.cpp +++ b/toolkit/components/places/src/History.cpp @@ -159,11 +159,9 @@ Step::HandleCompletion(PRUint16 aReason) namespace { -class VisitedQuery : public mozIStorageStatementCallback +class VisitedQuery : public AsyncStatementCallback { public: - NS_DECL_ISUPPORTS - static nsresult Start(nsIURI* aURI) { NS_PRECONDITION(aURI, "Null URI"); @@ -172,7 +170,7 @@ public: // If we are a content process, always remote the request to the // parent process. if (XRE_GetProcessType() == GeckoProcessType_Content) { - mozilla::dom::ContentChild * cpc = + mozilla::dom::ContentChild* cpc = mozilla::dom::ContentChild::GetSingleton(); NS_ASSERTION(cpc, "Content Protocol is NULL!"); (void)cpc->SendStartVisitedQuery(aURI); @@ -180,9 +178,8 @@ public: } #endif - nsNavHistory* navHist = nsNavHistory::GetHistoryService(); - NS_ENSURE_TRUE(navHist, NS_ERROR_FAILURE); - mozIStorageStatement* stmt = navHist->GetStatementById(DB_IS_PAGE_VISITED); + mozIStorageAsyncStatement* stmt = + History::GetService()->GetIsVisitedStatement(); NS_ENSURE_STATE(stmt); // Bind by index for performance. @@ -250,10 +247,6 @@ private: nsCOMPtr mURI; bool mIsVisited; }; -NS_IMPL_ISUPPORTS1( - VisitedQuery, - mozIStorageStatementCallback -) /** * Fail-safe mechanism for ensuring that your task completes, no matter what. @@ -266,7 +259,7 @@ class FailSafeFinishTask { public: FailSafeFinishTask() - : mAppended(false) + : mAppended(false) { } @@ -315,8 +308,6 @@ struct VisitURIData : public FailSafeFinishTask class UpdateFrecencyAndNotifyStep : public Step { public: - NS_DECL_ISUPPORTS - UpdateFrecencyAndNotifyStep(nsAutoPtr aData) : mData(aData) { @@ -346,10 +337,7 @@ public: NS_WARN_IF_FALSE(bookmarks, "Could not get bookmarks service"); if (history && bookmarks) { // Update frecency *after* the visit info is in the db - nsresult rv = history->UpdateFrecency( - mData->placeId, - bookmarks->IsRealBookmark(mData->placeId) - ); + nsresult rv = history->UpdateFrecency(mData->placeId); NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "Could not update frecency"); // Notify nsNavHistory observers of visit, but only for certain types of @@ -378,10 +366,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - UpdateFrecencyAndNotifyStep -, mozIStorageStatementCallback -) /** * Step 5: Get newly created visit ID from moz_history_visits table. @@ -389,8 +373,6 @@ NS_IMPL_ISUPPORTS1( class GetVisitIDStep : public Step { public: - NS_DECL_ISUPPORTS - GetVisitIDStep(nsAutoPtr aData) : mData(aData) { @@ -419,10 +401,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - GetVisitIDStep -, mozIStorageStatementCallback -) /** * Step 4: Add visit to moz_history_visits table. @@ -430,8 +408,6 @@ NS_IMPL_ISUPPORTS1( class AddVisitStep : public Step { public: - NS_DECL_ISUPPORTS - AddVisitStep(nsAutoPtr aData) : mData(aData) { @@ -509,10 +485,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - AddVisitStep -, mozIStorageStatementCallback -) /** * Step 3: Callback for inserting or updating a moz_places entry. @@ -521,8 +493,6 @@ NS_IMPL_ISUPPORTS1( class CheckLastVisitStep : public Step { public: - NS_DECL_ISUPPORTS - CheckLastVisitStep(nsAutoPtr aData) : mData(aData) { @@ -574,10 +544,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - CheckLastVisitStep -, mozIStorageStatementCallback -) /** * Step 2a: Called only when a new entry is put into moz_places. @@ -586,8 +552,6 @@ NS_IMPL_ISUPPORTS1( class FindNewIdStep : public Step { public: - NS_DECL_ISUPPORTS - FindNewIdStep(nsAutoPtr aData) : mData(aData) { @@ -615,10 +579,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - FindNewIdStep -, mozIStorageStatementCallback -) /** * Step 2: Callback for checking for an existing URI in moz_places. @@ -627,8 +587,6 @@ NS_IMPL_ISUPPORTS1( class CheckExistingStep : public Step { public: - NS_DECL_ISUPPORTS - CheckExistingStep(nsAutoPtr aData) : mData(aData) { @@ -712,10 +670,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - CheckExistingStep -, mozIStorageStatementCallback -) /** * Step 1: See if there is an existing URI. @@ -723,8 +677,6 @@ NS_IMPL_ISUPPORTS1( class StartVisitURIStep : public Step { public: - NS_DECL_ISUPPORTS - StartVisitURIStep(nsAutoPtr aData) : mData(aData) { @@ -755,10 +707,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - StartVisitURIStep -, Step -) //////////////////////////////////////////////////////////////////////////////// //// Steps for SetURITitle @@ -775,8 +723,6 @@ struct SetTitleData : public FailSafeFinishTask class TitleNotifyStep: public Step { public: - NS_DECL_ISUPPORTS - TitleNotifyStep(nsAutoPtr aData) : mData(aData) { @@ -794,10 +740,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - TitleNotifyStep -, mozIStorageStatementCallback -) /** * Step 2: Set title. @@ -805,8 +747,6 @@ NS_IMPL_ISUPPORTS1( class SetTitleStep : public Step { public: - NS_DECL_ISUPPORTS - SetTitleStep(nsAutoPtr aData) : mData(aData) { @@ -865,10 +805,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - SetTitleStep -, mozIStorageStatementCallback -) /** * Step 1: See if there is an existing URI. @@ -876,8 +812,6 @@ NS_IMPL_ISUPPORTS1( class StartSetURITitleStep : public Step { public: - NS_DECL_ISUPPORTS - StartSetURITitleStep(nsAutoPtr aData) : mData(aData) { @@ -907,10 +841,6 @@ public: protected: nsAutoPtr mData; }; -NS_IMPL_ISUPPORTS1( - StartSetURITitleStep -, Step -) } // anonymous namespace @@ -1021,6 +951,37 @@ History::NotifyVisited(nsIURI* aURI) mObservers.RemoveEntry(aURI); } +mozIStorageAsyncStatement* +History::GetIsVisitedStatement() +{ + if (mIsVisitedStatement) { + return mIsVisitedStatement; + } + + // If we don't yet have a database connection, go ahead and clone it now. + if (!mReadOnlyDBConn) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, nsnull); + + nsCOMPtr dbConn; + (void)history->GetDBConnection(getter_AddRefs(dbConn)); + NS_ENSURE_TRUE(dbConn, nsnull); + + (void)dbConn->Clone(PR_TRUE, getter_AddRefs(mReadOnlyDBConn)); + NS_ENSURE_TRUE(mReadOnlyDBConn, nsnull); + } + + // Now we can create our cached statement. + nsresult rv = mReadOnlyDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING( + "SELECT h.id " + "FROM moz_places h " + "WHERE url = ?1 " + "AND EXISTS(SELECT id FROM moz_historyvisits WHERE place_id = h.id LIMIT 1) " + ), getter_AddRefs(mIsVisitedStatement)); + NS_ENSURE_SUCCESS(rv, nsnull); + return mIsVisitedStatement; +} + /* static */ History* History::GetService() @@ -1075,6 +1036,14 @@ History::Shutdown() nsCOMPtr deadTaskWalking = dont_AddRef(static_cast(mPendingVisits.PopFront())); } + + // Clean up our statements and connection. + if (mReadOnlyDBConn) { + if (mIsVisitedStatement) { + (void)mIsVisitedStatement->Finalize(); + } + (void)mReadOnlyDBConn->AsyncClose(nsnull); + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/toolkit/components/places/src/History.h b/toolkit/components/places/src/History.h index 16b44b4f9d8..1d40e9acf16 100644 --- a/toolkit/components/places/src/History.h +++ b/toolkit/components/places/src/History.h @@ -48,6 +48,7 @@ #include "nsTArray.h" #include "nsDeque.h" #include "nsIObserver.h" +#include "mozIStorageConnection.h" namespace mozilla { namespace places { @@ -71,7 +72,7 @@ public: * @param aURI * The URI to notify about. */ - void NotifyVisited(nsIURI *aURI); + void NotifyVisited(nsIURI* aURI); /** * Append a task to the queue for SQL queries that need to happen @@ -93,19 +94,38 @@ public: */ void CurrentTaskFinished(); + /** + * Obtains the statement to use to check if a URI is visited or not. + */ + mozIStorageAsyncStatement* GetIsVisitedStatement(); + /** * Obtains a pointer to this service. */ - static History *GetService(); + static History* GetService(); /** * Obtains a pointer that has had AddRef called on it. Used by the service * manager only. */ - static History *GetSingleton(); + static History* GetSingleton(); private: - ~History(); + virtual ~History(); + + /** + * A read-only database connection used for checking if a URI is visited. + * + * @note this should only be accessed by GetIsVisistedStatement and Shutdown. + */ + nsCOMPtr mReadOnlyDBConn; + + /** + * An asynchronous statement to query if a URI is visited or not. + * + * @note this should only be accessed by GetIsVisistedStatement and Shutdown. + */ + nsCOMPtr mIsVisitedStatement; /** * Since visits rapidly fire at once, it's very likely to have race @@ -132,21 +152,21 @@ private: */ void Shutdown(); - static History *gService; + static History* gService; // Ensures new tasks aren't started on destruction. bool mShuttingDown; - typedef nsTArray ObserverArray; + typedef nsTArray ObserverArray; class KeyClass : public nsURIHashKey { public: - KeyClass(const nsIURI *aURI) + KeyClass(const nsIURI* aURI) : nsURIHashKey(aURI) { } - KeyClass(const KeyClass &aOther) + KeyClass(const KeyClass& aOther) : nsURIHashKey(aOther) { NS_NOTREACHED("Do not call me!"); diff --git a/toolkit/components/places/src/Makefile.in b/toolkit/components/places/src/Makefile.in index 0fba61349c1..769f3270189 100644 --- a/toolkit/components/places/src/Makefile.in +++ b/toolkit/components/places/src/Makefile.in @@ -82,11 +82,13 @@ EXTRA_DSO_LDOPTS += \ LOCAL_INCLUDES += -I$(srcdir)/../../build +# This is the default value. Must be in sync with the one defined in SQLite. +DEFINES += -DSQLITE_DEFAULT_PAGE_SIZE=32768 + EXTRA_COMPONENTS = \ toolkitplaces.manifest \ nsLivemarkService.js \ nsTaggingService.js \ - nsPlacesDBFlush.js \ nsPlacesExpiration.js \ nsMicrosummaryService.js \ PlacesCategoriesStarter.js \ diff --git a/toolkit/components/places/src/PlacesDBUtils.jsm b/toolkit/components/places/src/PlacesDBUtils.jsm index db0dd1d90b0..705bedb5b7f 100644 --- a/toolkit/components/places/src/PlacesDBUtils.jsm +++ b/toolkit/components/places/src/PlacesDBUtils.jsm @@ -126,8 +126,6 @@ nsPlacesDBUtils.prototype = { "DELETE FROM moz_annos WHERE id IN ( " + "SELECT id FROM moz_annos a " + "WHERE NOT EXISTS " + - "(SELECT id FROM moz_places_temp WHERE id = a.place_id LIMIT 1) " + - "AND NOT EXISTS " + "(SELECT id FROM moz_places WHERE id = a.place_id LIMIT 1) " + ")"); cleanupStatements.push(deleteOrphanAnnos); @@ -202,7 +200,6 @@ nsPlacesDBUtils.prototype = { ") AND id IN (" + "SELECT b.id FROM moz_bookmarks b " + "WHERE fk NOT NULL AND b.type = :bookmark_type " + - "AND NOT EXISTS (SELECT url FROM moz_places_temp WHERE id = b.fk LIMIT 1) " + "AND NOT EXISTS (SELECT url FROM moz_places WHERE id = b.fk LIMIT 1) " + ")"); deleteNoPlaceItems.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK; @@ -361,7 +358,6 @@ nsPlacesDBUtils.prototype = { // D.11 remove old livemarks status items // Livemark status items are now static but some livemark has still old // status items bookmarks inside it. We should remove them. - // Note: This does not need to query the temp table. let removeLivemarkStaticItems = DBConn.createStatement( "DELETE FROM moz_bookmarks WHERE type = :bookmark_type AND fk IN ( " + "SELECT id FROM moz_places WHERE url = :lmloading OR url = :lmfailed " + @@ -390,8 +386,6 @@ nsPlacesDBUtils.prototype = { "DELETE FROM moz_favicons WHERE id IN (" + "SELECT id FROM moz_favicons f " + "WHERE NOT EXISTS " + - "(SELECT id FROM moz_places_temp WHERE favicon_id = f.id LIMIT 1) " + - "AND NOT EXISTS" + "(SELECT id FROM moz_places WHERE favicon_id = f.id LIMIT 1) " + ")"); cleanupStatements.push(deleteOrphanIcons); @@ -402,8 +396,6 @@ nsPlacesDBUtils.prototype = { "DELETE FROM moz_historyvisits WHERE id IN (" + "SELECT id FROM moz_historyvisits v " + "WHERE NOT EXISTS " + - "(SELECT id FROM moz_places_temp WHERE id = v.place_id LIMIT 1) " + - "AND NOT EXISTS " + "(SELECT id FROM moz_places WHERE id = v.place_id LIMIT 1) " + ")"); cleanupStatements.push(deleteOrphanVisits); @@ -414,8 +406,6 @@ nsPlacesDBUtils.prototype = { "DELETE FROM moz_inputhistory WHERE place_id IN (" + "SELECT place_id FROM moz_inputhistory i " + "WHERE NOT EXISTS " + - "(SELECT id FROM moz_places_temp WHERE id = i.place_id LIMIT 1) " + - "AND NOT EXISTS " + "(SELECT id FROM moz_places WHERE id = i.place_id LIMIT 1) " + ")"); cleanupStatements.push(deleteOrphanInputHistory); @@ -463,22 +453,17 @@ nsPlacesDBUtils.prototype = { /* XXX needs test // L.2 recalculate visit_count - // We're detecting errors only in disk table since temp tables could have - // different values based on the number of visits not yet synced to disk. let detectWrongCountPlaces = DBConn.createStatement( "SELECT id FROM moz_places h " + - "WHERE id NOT IN (SELECT id FROM moz_places_temp) " + - "AND h.visit_count <> " + + "WHERE h.visit_count <> " + "(SELECT count(*) FROM moz_historyvisits " + "WHERE place_id = h.id AND visit_type NOT IN (0,4,7,8))"); while (detectWrongCountPlaces.executeStep()) { let placeId = detectWrongCountPlaces.getInt64(0); let fixCountForPlace = DBConn.createStatement( - "UPDATE moz_places_view SET visit_count = ( " + + "UPDATE moz_places SET visit_count = ( " + "(SELECT count(*) FROM moz_historyvisits " + - "WHERE place_id = :place_id AND visit_type NOT IN (0,4,7,8)) + " + - "(SELECT count(*) FROM moz_historyvisits_temp " + - "WHERE place_id = :place_id AND visit_type NOT IN (0,4,7,8)) + " + + "WHERE place_id = :place_id AND visit_type NOT IN (0,4,7,8)) + " ") WHERE id = :place_id"); fixCountForPlace.params["place_id"] = placeId; cleanupStatements.push(fixCountForPlace); diff --git a/toolkit/components/places/src/PlacesUtils.jsm b/toolkit/components/places/src/PlacesUtils.jsm index 9758eb115ea..b0d65ecb9c0 100644 --- a/toolkit/components/places/src/PlacesUtils.jsm +++ b/toolkit/components/places/src/PlacesUtils.jsm @@ -150,7 +150,6 @@ var PlacesUtils = { TOPIC_DATABASE_LOCKED: "places-database-locked", TOPIC_EXPIRATION_FINISHED: "places-expiration-finished", TOPIC_FEEDBACK_UPDATED: "places-autocomplete-feedback-updated", - TOPIC_SYNC_FINISHED: "places-sync-finished", TOPIC_FAVICONS_EXPIRED: "places-favicons-expired", TOPIC_VACUUM_STARTING: "places-vacuum-starting", diff --git a/toolkit/components/places/src/SQLFunctions.cpp b/toolkit/components/places/src/SQLFunctions.cpp index 857d11aef7c..4d376df0f1e 100644 --- a/toolkit/components/places/src/SQLFunctions.cpp +++ b/toolkit/components/places/src/SQLFunctions.cpp @@ -43,6 +43,10 @@ #include "nsEscape.h" #include "mozIPlacesAutoComplete.h" #include "SQLFunctions.h" +#include "nsMathUtils.h" +#include "nsINavHistoryService.h" +#include "nsPrintfCString.h" +#include "nsNavHistory.h" using namespace mozilla::storage; @@ -294,5 +298,178 @@ namespace places { #undef HAS_BEHAVIOR } + +//////////////////////////////////////////////////////////////////////////////// +//// Frecency Calculation Function + + ////////////////////////////////////////////////////////////////////////////// + //// CalculateFrecencyFunction + + /* static */ + nsresult + CalculateFrecencyFunction::create(mozIStorageConnection *aDBConn) + { + nsCOMPtr function = + new CalculateFrecencyFunction(); + NS_ENSURE_TRUE(function, NS_ERROR_OUT_OF_MEMORY); + + nsresult rv = aDBConn->CreateFunction( + NS_LITERAL_CSTRING("calculate_frecency"), 1, function + ); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + NS_IMPL_THREADSAFE_ISUPPORTS1( + CalculateFrecencyFunction, + mozIStorageFunction + ) + + ////////////////////////////////////////////////////////////////////////////// + //// mozIStorageFunction + + NS_IMETHODIMP + CalculateFrecencyFunction::OnFunctionCall(mozIStorageValueArray *aArguments, + nsIVariant **_result) + { + // Fetch arguments. Use default values if they were omitted. + PRUint32 numEntries; + nsresult rv = aArguments->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(numEntries > 0, "unexpected number of arguments"); + + PRInt64 pageId = aArguments->AsInt64(0); + PRInt32 typed = numEntries > 1 ? aArguments->AsInt32(1) : 0; + PRInt32 fullVisitCount = numEntries > 2 ? aArguments->AsInt32(2) : 0; + PRInt64 bookmarkId = numEntries > 3 ? aArguments->AsInt64(3) : 0; + PRInt32 visitCount = 0; + PRInt32 hidden = 0; + PRInt32 isQuery = 0; + float pointsForSampledVisits = 0.0; + + // This is a const version of the history object for thread-safety. + const nsNavHistory* history = nsNavHistory::GetConstHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + + if (pageId > 0) { + // The page is already in the database, and we can fetch current + // params from the database. + nsCOMPtr getPageInfo = + history->GetStatementByStoragePool(DB_PAGE_INFO_FOR_FRECENCY); + NS_ENSURE_STATE(getPageInfo); + mozStorageStatementScoper infoScoper(getPageInfo); + + rv = getPageInfo->BindInt64ByName(NS_LITERAL_CSTRING("page_id"), pageId); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->BindUTF8StringByName(NS_LITERAL_CSTRING("anno_name"), + NS_LITERAL_CSTRING("livemark/feedURI")); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool hasResult; + rv = getPageInfo->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(hasResult, NS_ERROR_UNEXPECTED); + rv = getPageInfo->GetInt32(0, &typed); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(1, &hidden); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(2, &visitCount); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(3, &fullVisitCount); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt64(4, &bookmarkId); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(5, &isQuery); + NS_ENSURE_SUCCESS(rv, rv); + + // Get a sample of the last visits to the page, to calculate its weight. + nsCOMPtr getVisits = + history->GetStatementByStoragePool(DB_VISITS_FOR_FRECENCY); + NS_ENSURE_STATE(getVisits); + mozStorageStatementScoper visitsScoper(getVisits); + + rv = getVisits->BindInt64ByName(NS_LITERAL_CSTRING("page_id"), pageId); + NS_ENSURE_SUCCESS(rv, rv); + + PRInt32 numSampledVisits = 0; + // The visits query is already limited to the last N visits. + while (NS_SUCCEEDED(getVisits->ExecuteStep(&hasResult)) && hasResult) { + numSampledVisits++; + + PRInt32 visitType; + rv = getVisits->GetInt32(1, &visitType); + NS_ENSURE_SUCCESS(rv, rv); + PRInt32 bonus = history->GetFrecencyTransitionBonus(visitType, true); + + // Always add the bookmark visit bonus. + if (bookmarkId) { + bonus += history->GetFrecencyTransitionBonus(nsINavHistoryService::TRANSITION_BOOKMARK, true); + } + + // If bonus was zero, we can skip the work to determine the weight. + if (bonus) { + PRInt32 ageInDays = getVisits->AsInt32(0); + PRInt32 weight = history->GetFrecencyAgedWeight(ageInDays); + pointsForSampledVisits += (float)(weight * (bonus / 100.0)); + } + } + + // If we found some visits for this page, use the calculated weight. + if (numSampledVisits) { + // fix for bug #412219 + if (!pointsForSampledVisits) { + // For URIs with zero points in the sampled recent visits + // but "browsing" type visits outside the sampling range, set + // frecency to -visit_count, so they're still shown in autocomplete. + NS_IF_ADDREF(*_result = new IntegerVariant(-visitCount)); + NS_ENSURE_TRUE(*_result, NS_ERROR_OUT_OF_MEMORY); + } + else { + // Estimate frecency using the last few visits. + // Use NS_ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete. + NS_IF_ADDREF(*_result = new IntegerVariant((PRInt32) NS_ceilf(fullVisitCount * NS_ceilf(pointsForSampledVisits) / numSampledVisits))); + NS_ENSURE_TRUE(*_result, NS_ERROR_OUT_OF_MEMORY); + } + + return NS_OK; + } + } + + // This page is unknown or has no visits. It could have just been added, so + // use passed in or default values. + + // The code below works well for guessing the frecency on import, and we'll + // correct later once we have visits. + // TODO: What if we don't have visits and we never visit? We could end up + // with a really high value that keeps coming up in ac results? Should we + // only do this on import? Have to figure it out. + PRInt32 bonus = 0; + + // Make it so something bookmarked and typed will have a higher frecency + // than something just typed or just bookmarked. + if (bookmarkId && !isQuery) { + bonus += history->GetFrecencyTransitionBonus(nsINavHistoryService::TRANSITION_BOOKMARK, false);; + // For unvisited bookmarks, produce a non-zero frecency, so that they show + // up in URL bar autocomplete. + fullVisitCount = 1; + } + + if (typed) { + bonus += history->GetFrecencyTransitionBonus(nsINavHistoryService::TRANSITION_TYPED, false); + } + + // Assume "now" as our ageInDays, so use the first bucket. + pointsForSampledVisits = history->GetFrecencyBucketWeight(1) * (bonus / (float)100.0); + + // use NS_ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete + NS_IF_ADDREF(*_result = new IntegerVariant((PRInt32) NS_ceilf(fullVisitCount * NS_ceilf(pointsForSampledVisits)))); + NS_ENSURE_TRUE(*_result, NS_ERROR_OUT_OF_MEMORY); + + return NS_OK; + } + } // namespace places } // namespace mozilla diff --git a/toolkit/components/places/src/SQLFunctions.h b/toolkit/components/places/src/SQLFunctions.h index 85699612a74..77e1a886fb0 100644 --- a/toolkit/components/places/src/SQLFunctions.h +++ b/toolkit/components/places/src/SQLFunctions.h @@ -208,6 +208,43 @@ private: static void fixupURISpec(const nsCString &aURISpec, nsString &_fixedSpec); }; + + +//////////////////////////////////////////////////////////////////////////////// +//// Frecency Calculation Function + +/** + * This function is used to calculate frecency for a page. + * + * In SQL, you'd use it in when setting frecency like: + * SET frecency = CALCULATE_FRECENCY(place_id). + * Optional parameters must be passed in if the page is not yet in the database, + * otherwise they will be fetched from it automatically. + * + * @param pageId + * The id of the page. Pass -1 if the page is being added right now. + * @param [optional] typed + * Whether the page has been typed in. Default is false. + * @param [optional] fullVisitCount + * Count of all the visits (All types). Default is 0. + * @param [optional] isBookmarked + * Whether the page is bookmarked. Default is false. + */ +class CalculateFrecencyFunction : public mozIStorageFunction +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection *aDBConn); +}; + } // namespace places } // namespace storage diff --git a/toolkit/components/places/src/nsAnnoProtocolHandler.cpp b/toolkit/components/places/src/nsAnnoProtocolHandler.cpp index 341e206c3bc..1e1cae93896 100644 --- a/toolkit/components/places/src/nsAnnoProtocolHandler.cpp +++ b/toolkit/components/places/src/nsAnnoProtocolHandler.cpp @@ -103,7 +103,7 @@ class faviconAsyncLoader : public AsyncStatementCallback , public nsIRequestObserver { public: - NS_DECL_ISUPPORTS + NS_DECL_ISUPPORTS_INHERITED faviconAsyncLoader(nsIChannel *aChannel, nsIOutputStream *aOutputStream) : mChannel(aChannel) @@ -219,9 +219,9 @@ private: bool mReturnDefaultIcon; }; -NS_IMPL_ISUPPORTS2( +NS_IMPL_ISUPPORTS_INHERITED1( faviconAsyncLoader, - mozIStorageStatementCallback, + AsyncStatementCallback, nsIRequestObserver ) diff --git a/toolkit/components/places/src/nsAnnotationService.cpp b/toolkit/components/places/src/nsAnnotationService.cpp index 5feaf273bf7..994d8b4e325 100644 --- a/toolkit/components/places/src/nsAnnotationService.cpp +++ b/toolkit/components/places/src/nsAnnotationService.cpp @@ -136,23 +136,17 @@ nsAnnotationService::GetStatement(const nsCOMPtr& aStmt) "SELECT n.name " "FROM moz_anno_attributes n " "JOIN moz_annos a ON a.anno_attribute_id = n.id " - "JOIN ( " - "SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url " - ") AS h ON h.id = a.place_id")); + "JOIN moz_places h ON h.id = a.place_id " + "WHERE h.url = :page_url")); RETURN_IF_STMT(mDBGetPageAnnotationValue, NS_LITERAL_CSTRING( "SELECT a.id, a.place_id, :anno_name, a.mime_type, a.content, a.flags, " "a.expiration, a.type " "FROM moz_anno_attributes n " "JOIN moz_annos a ON n.id = a.anno_attribute_id " - "JOIN ( " - "SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url " - ") AS h ON h.id = a.place_id " - "WHERE n.name = :anno_name")); + "JOIN moz_places h ON h.id = a.place_id " + "WHERE h.url = :page_url " + "AND n.name = :anno_name")); RETURN_IF_STMT(mDBGetItemAnnotationValue, NS_LITERAL_CSTRING( "SELECT a.id, a.item_id, :anno_name, a.mime_type, a.content, a.flags, " @@ -181,26 +175,22 @@ nsAnnotationService::GetStatement(const nsCOMPtr& aStmt) RETURN_IF_STMT(mDBRemovePageAnnotation, NS_LITERAL_CSTRING( "DELETE FROM moz_annos " - "WHERE place_id = ( " - "SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url) " - "AND anno_attribute_id = " - "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name)")); + "WHERE place_id = (SELECT id FROM moz_places WHERE url = :page_url) " + "AND anno_attribute_id = " + "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name)")); RETURN_IF_STMT(mDBRemoveItemAnnotation, NS_LITERAL_CSTRING( "DELETE FROM moz_items_annos " "WHERE item_id = :item_id " - "AND anno_attribute_id = " - "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name)")); + "AND anno_attribute_id = " + "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name)")); RETURN_IF_STMT(mDBGetPagesWithAnnotation, NS_LITERAL_CSTRING( - "SELECT IFNULL(h_t.url, h.url) AS coalesced_url " + "SELECT h.url " "FROM moz_anno_attributes n " "JOIN moz_annos a ON n.id = a.anno_attribute_id " - "LEFT JOIN moz_places h ON h.id = a.place_id " - "LEFT JOIN moz_places_temp h_t ON h_t.id = a.place_id " - "WHERE n.name = :anno_name AND coalesced_url NOT NULL")); + "JOIN moz_places h ON h.id = a.place_id " + "WHERE n.name = :anno_name")); RETURN_IF_STMT(mDBGetItemsWithAnnotation, NS_LITERAL_CSTRING( "SELECT a.item_id " @@ -212,12 +202,10 @@ nsAnnotationService::GetStatement(const nsCOMPtr& aStmt) "SELECT h.id, " "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name) AS nameid, " "a.id, a.dateAdded " - "FROM (SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url " - ") AS h " + "FROM moz_places h " "LEFT JOIN moz_annos a ON a.place_id = h.id " - "AND a.anno_attribute_id = nameid")); + "AND a.anno_attribute_id = nameid " + "WHERE h.url = :page_url")); RETURN_IF_STMT(mDBCheckItemAnnotation, NS_LITERAL_CSTRING( "SELECT b.id, " @@ -1582,9 +1570,7 @@ nsAnnotationService::RemovePageAnnotations(nsIURI* aURI) nsCOMPtr statement; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( "DELETE FROM moz_annos WHERE place_id = " - "(SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url)"), + "(SELECT id FROM moz_places WHERE url = :page_url)"), getter_AddRefs(statement)); NS_ENSURE_SUCCESS(rv, rv); rv = URIBinder::Bind(statement, NS_LITERAL_CSTRING("page_url"), aURI); @@ -1644,18 +1630,13 @@ nsAnnotationService::CopyPageAnnotations(nsIURI* aSourceURI, nsCOMPtr sourceStmt; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( "SELECT h.id, n.id, n.name, a2.id " - "FROM ( " - "SELECT id from moz_places_temp WHERE url = :source_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :source_url " - ") AS h " + "FROM moz_places h " "JOIN moz_annos a ON a.place_id = h.id " "JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id " - "LEFT JOIN moz_annos a2 ON a2.place_id = ( " - "SELECT id FROM moz_places_temp WHERE url = :dest_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :dest_url " - ") AND a2.anno_attribute_id = n.id"), + "LEFT JOIN moz_annos a2 ON a2.place_id = " + "(SELECT id FROM moz_places WHERE url = :dest_url) " + "AND a2.anno_attribute_id = n.id " + "WHERE url = :source_url"), getter_AddRefs(sourceStmt)); NS_ENSURE_SUCCESS(rv, rv); @@ -1669,12 +1650,9 @@ nsAnnotationService::CopyPageAnnotations(nsIURI* aSourceURI, "INSERT INTO moz_annos " "(place_id, anno_attribute_id, mime_type, content, flags, expiration, " "type, dateAdded, lastModified) " - "SELECT ( " - "SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places WHERE url = :page_url " - "), anno_attribute_id, mime_type, content, flags, expiration, type, " - ":date, :date " + "SELECT (SELECT id FROM moz_places WHERE url = :page_url), " + "anno_attribute_id, mime_type, content, flags, expiration, type, " + ":date, :date " "FROM moz_annos " "WHERE place_id = :page_id " "AND anno_attribute_id = :name_id"), diff --git a/toolkit/components/places/src/nsFaviconService.cpp b/toolkit/components/places/src/nsFaviconService.cpp index f824802bdf9..e8a58331e5f 100644 --- a/toolkit/components/places/src/nsFaviconService.cpp +++ b/toolkit/components/places/src/nsFaviconService.cpp @@ -97,8 +97,7 @@ class ExpireFaviconsStatementCallbackNotifier : public AsyncStatementCallback { public: ExpireFaviconsStatementCallbackNotifier(bool* aFaviconsExpirationRunning); - NS_DECL_ISUPPORTS - NS_DECL_ASYNCSTATEMENTCALLBACK + NS_IMETHOD HandleCompletion(PRUint16 aReason); private: bool* mFaviconsExpirationRunning; @@ -171,30 +170,20 @@ nsFaviconService::GetStatement(const nsCOMPtr& aStmt) RETURN_IF_STMT(mDBGetIconInfoWithPage, NS_LITERAL_CSTRING( "SELECT id, length(data), expiration, data, mime_type, " "IFNULL(url = (SELECT f.url " - "FROM ( " - "SELECT favicon_id FROM moz_places_temp " - "WHERE url = :page_url " - "UNION ALL " - "SELECT favicon_id FROM moz_places " - "WHERE url = :page_url " - ") AS h " + "FROM moz_places h " "JOIN moz_favicons f ON h.favicon_id = f.id " + "WHERE h.url = :page_url " "LIMIT 1), " - "0)" + "0) " "FROM moz_favicons WHERE url = :icon_url")); RETURN_IF_STMT(mDBGetURL, NS_LITERAL_CSTRING( "SELECT f.id, f.url, length(f.data), f.expiration " - "FROM ( " - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places_temp " - "WHERE url = :page_url " - "UNION ALL " - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places " - "WHERE url = :page_url " - ") AS h JOIN moz_favicons f ON h.favicon_id = f.id " + "FROM moz_places h " + "JOIN moz_favicons f ON h.favicon_id = f.id " + "WHERE h.url = :page_url " "LIMIT 1")); - RETURN_IF_STMT(mDBGetData, NS_LITERAL_CSTRING( "SELECT f.data, f.mime_type FROM moz_favicons f WHERE url = :icon_url")); @@ -208,10 +197,10 @@ nsFaviconService::GetStatement(const nsCOMPtr& aStmt) "WHERE id = :icon_id")); RETURN_IF_STMT(mDBSetPageFavicon, NS_LITERAL_CSTRING( - "UPDATE moz_places_view SET favicon_id = :icon_id WHERE id = :page_id")); + "UPDATE moz_places SET favicon_id = :icon_id WHERE id = :page_id")); RETURN_IF_STMT(mDBAssociateFaviconURIToPageURI, NS_LITERAL_CSTRING( - "UPDATE moz_places_view " + "UPDATE moz_places " "SET favicon_id = (SELECT id FROM moz_favicons WHERE url = :icon_url) " "WHERE url = :page_url")); @@ -220,15 +209,8 @@ nsFaviconService::GetStatement(const nsCOMPtr& aStmt) "SET favicon_id = NULL " "WHERE favicon_id NOT NULL")); - RETURN_IF_STMT(mDBRemoveTempReferences, NS_LITERAL_CSTRING( - "UPDATE moz_places_temp " - "SET favicon_id = NULL " - "WHERE favicon_id NOT NULL")); - RETURN_IF_STMT(mDBRemoveAllFavicons, NS_LITERAL_CSTRING( "DELETE FROM moz_favicons WHERE id NOT IN (" - "SELECT favicon_id FROM moz_places_temp WHERE favicon_id NOT NULL " - "UNION ALL " "SELECT favicon_id FROM moz_places WHERE favicon_id NOT NULL " ")")); @@ -262,16 +244,11 @@ nsFaviconService::ExpireAllFavicons() { mFaviconsExpirationRunning = true; - // We do this in 2 steps, first we null-out all favicons in the disk table, - // then we do the same in the temp table. This is because the view UPDATE - // trigger does not allow setting a NULL value to prevent dataloss. - mozIStorageBaseStatement *stmts[] = { GetStatement(mDBRemoveOnDiskReferences), - GetStatement(mDBRemoveTempReferences), GetStatement(mDBRemoveAllFavicons), }; - NS_ENSURE_STATE(stmts[0] && stmts[1] && stmts[2]); + NS_ENSURE_STATE(stmts[0] && stmts[1]); nsCOMPtr ps; nsCOMPtr callback = new ExpireFaviconsStatementCallbackNotifier(&mFaviconsExpirationRunning); @@ -1051,7 +1028,6 @@ nsFaviconService::FinalizeStatements() { mDBUpdateIcon, mDBSetPageFavicon, mDBRemoveOnDiskReferences, - mDBRemoveTempReferences, mDBRemoveAllFavicons, mDBAssociateFaviconURIToPageURI, }; @@ -1081,9 +1057,6 @@ nsFaviconService::GetFaviconDataAsync(nsIURI* aFaviconURI, //////////////////////////////////////////////////////////////////////////////// //// ExpireFaviconsStatementCallbackNotifier -NS_IMPL_ISUPPORTS1(ExpireFaviconsStatementCallbackNotifier, - mozIStorageStatementCallback) - ExpireFaviconsStatementCallbackNotifier::ExpireFaviconsStatementCallbackNotifier( bool* aFaviconsExpirationRunning) : mFaviconsExpirationRunning(aFaviconsExpirationRunning) @@ -1111,11 +1084,3 @@ ExpireFaviconsStatementCallbackNotifier::HandleCompletion(PRUint16 aReason) return NS_OK; } - - -NS_IMETHODIMP -ExpireFaviconsStatementCallbackNotifier::HandleResult(mozIStorageResultSet* aResultSet) -{ - NS_ASSERTION(PR_FALSE, "You cannot use this statement callback to get async statements resultset"); - return NS_OK; -} diff --git a/toolkit/components/places/src/nsFaviconService.h b/toolkit/components/places/src/nsFaviconService.h index da9a88738b3..f4d53e2c8c3 100644 --- a/toolkit/components/places/src/nsFaviconService.h +++ b/toolkit/components/places/src/nsFaviconService.h @@ -190,7 +190,6 @@ private: nsCOMPtr mDBSetPageFavicon; nsCOMPtr mDBAssociateFaviconURIToPageURI; nsCOMPtr mDBRemoveOnDiskReferences; - nsCOMPtr mDBRemoveTempReferences; nsCOMPtr mDBRemoveAllFavicons; static nsFaviconService* gFaviconService; diff --git a/toolkit/components/places/src/nsNavBookmarks.cpp b/toolkit/components/places/src/nsNavBookmarks.cpp index 81cdbc76ed9..9f3e68d4834 100644 --- a/toolkit/components/places/src/nsNavBookmarks.cpp +++ b/toolkit/components/places/src/nsNavBookmarks.cpp @@ -251,17 +251,12 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) // Double ordering covers possible lastModified ties, that could happen when // importing, syncing or due to extensions. + // Note: not using a JOIN is cheaper in this case. RETURN_IF_STMT(mDBFindURIBookmarks, NS_LITERAL_CSTRING( "SELECT b.id " "FROM moz_bookmarks b " - "WHERE b.type = :item_type AND b.fk = ( " - "SELECT id FROM moz_places_temp " - "WHERE url = :page_url " - "UNION " - "SELECT id FROM moz_places " - "WHERE url = :page_url " - "LIMIT 1 " - ") " + "WHERE b.fk = (SELECT id FROM moz_places WHERE url = :page_url) " + "AND b.fk NOTNULL " "ORDER BY b.lastModified DESC, b.id DESC ")); // Select all children of a given folder, sorted by position. @@ -271,15 +266,10 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) // by mDBGetURLPageInfo, and additionally contains columns for position, // item_child, and folder_child from moz_bookmarks. RETURN_IF_STMT(mDBGetChildren, NS_LITERAL_CSTRING( - "SELECT IFNULL(h_t.id, h.id), IFNULL(h_t.url, h.url), " - "COALESCE(b.title, h_t.title, h.title), " - "IFNULL(h_t.rev_host, h.rev_host), " - "IFNULL(h_t.visit_count, h.visit_count), " - "IFNULL(h_t.last_visit_date, h.last_visit_date), " - "f.url, null, b.id, b.dateAdded, b.lastModified, b.parent, null, " - "b.position, b.type, b.fk, b.folder_type " + "SELECT h.id, h.url, IFNULL(b.title, h.title), h.rev_host, h.visit_count, " + "h.last_visit_date, f.url, null, b.id, b.dateAdded, b.lastModified, " + "b.parent, null, b.position, b.type, b.fk, b.folder_type " "FROM moz_bookmarks b " - "LEFT JOIN moz_places_temp h_t ON b.fk = h_t.id " "LEFT JOIN moz_places h ON b.fk = h.id " "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " "WHERE b.parent = :parent " @@ -298,8 +288,7 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) // Get bookmark/folder/separator properties. RETURN_IF_STMT(mDBGetItemProperties, NS_LITERAL_CSTRING( "SELECT b.id, " - "IFNULL((SELECT url FROM moz_places_temp WHERE id = b.fk), " - "(SELECT url FROM moz_places WHERE id = b.fk)), " + "(SELECT url FROM moz_places WHERE id = b.fk), " "b.title, b.position, b.fk, b.parent, b.type, b.folder_type, " "b.dateAdded, b.lastModified " "FROM moz_bookmarks b " @@ -323,12 +312,9 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) "SELECT 1 FROM moz_bookmarks WHERE fk = :page_id")); RETURN_IF_STMT(mDBIsURIBookmarkedInDatabase, NS_LITERAL_CSTRING( - "SELECT 1 FROM moz_bookmarks WHERE fk = (" - "SELECT id FROM moz_places_temp WHERE url = :page_url " - "UNION ALL " - "SELECT id FROM moz_places WHERE url = :page_url " - "LIMIT 1" - ")")); + "SELECT 1 FROM moz_bookmarks b " + "JOIN moz_places h ON b.fk = h.id " + "WHERE h.url = :page_url")); // Checks to make sure a place id is a bookmark, and isn't a livemark. RETURN_IF_STMT(mDBIsRealBookmark, NS_LITERAL_CSTRING( @@ -363,19 +349,12 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) RETURN_IF_STMT(mDBSetItemIndex, NS_LITERAL_CSTRING( "UPDATE moz_bookmarks SET position = :item_index WHERE id = :item_id")); - // Get keyword text for bookmarked URI. RETURN_IF_STMT(mDBGetKeywordForURI, NS_LITERAL_CSTRING( "SELECT k.keyword " - "FROM ( " - "SELECT id FROM moz_places_temp " - "WHERE url = :page_url " - "UNION ALL " - "SELECT id FROM moz_places " - "WHERE url = :page_url " - "LIMIT 1 " - ") AS h " + "FROM moz_places h " "JOIN moz_bookmarks b ON b.fk = h.id " - "JOIN moz_keywords k ON k.id = b.keyword_id")); + "JOIN moz_keywords k ON k.id = b.keyword_id " + "WHERE h.url = :page_url ")); RETURN_IF_STMT(mDBAdjustPosition, NS_LITERAL_CSTRING( "UPDATE moz_bookmarks SET position = position + :delta " @@ -422,14 +401,6 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) // For most cases these levels of redirects should be fine though, it's hard // to hit a page that is 4 or 5 levels of redirects below a bookmarked page. // - // Moreover this query does not mix-up all possible cases of disk and temp - // tables. This is because we expect a redirects chain to be completely on - // disk or completely in memory. We never bring back visits from disk to - // memory, we sync visits on a timer (the chained visits have narrow times), - // or on bookmarks changes. The likely possiblity that we break a chain in - // the middle is so much smaller than the perf and readability hit we would - // get making complete crossing joins. - // // As a bonus the query also checks first if place_id is already a bookmark, // so you don't have to check that apart. @@ -442,31 +413,13 @@ nsNavBookmarks::GetStatement(const nsCOMPtr& aStmt) nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY); RETURN_IF_STMT(mDBFindRedirectedBookmark, NS_LITERAL_CSTRING( - "SELECT IFNULL( " - "(SELECT url FROM moz_places_temp WHERE id = :page_id), " + "SELECT " "(SELECT url FROM moz_places WHERE id = :page_id) " - ") " "FROM moz_bookmarks b " "WHERE b.fk = :page_id " "UNION ALL " // Not directly bookmarked. - "SELECT IFNULL( " - "(SELECT url FROM moz_places_temp WHERE id = " COALESCE_PLACEID "), " + "SELECT " "(SELECT url FROM moz_places WHERE id = " COALESCE_PLACEID ") " - ") " - "FROM moz_historyvisits_temp self " - "JOIN moz_bookmarks b ON b.fk = " COALESCE_PLACEID - "LEFT JOIN moz_historyvisits_temp parent ON parent.id = self.from_visit " - "LEFT JOIN moz_historyvisits_temp grandparent ON parent.from_visit = grandparent.id " - "AND parent.visit_type IN (") + redirectsFragment + NS_LITERAL_CSTRING(") " - "LEFT JOIN moz_historyvisits_temp greatgrandparent ON grandparent.from_visit = greatgrandparent.id " - "AND grandparent.visit_type IN (") + redirectsFragment + NS_LITERAL_CSTRING(") " - "WHERE self.visit_type IN (") + redirectsFragment + NS_LITERAL_CSTRING(") " - "AND self.place_id = :page_id " - "UNION ALL " // Not in the temp table. - "SELECT IFNULL( " - "(SELECT url FROM moz_places_temp WHERE id = " COALESCE_PLACEID "), " - "(SELECT url FROM moz_places WHERE id = " COALESCE_PLACEID ") " - ") " "FROM moz_historyvisits self " "JOIN moz_bookmarks b ON b.fk = " COALESCE_PLACEID "LEFT JOIN moz_historyvisits parent ON parent.id = self.from_visit " @@ -986,28 +939,8 @@ nsNavBookmarks::InsertBookmark(PRInt64 aFolder, rv = aURI->GetSpec(url); NS_ENSURE_SUCCESS(rv, rv); - // prevent place: queries from showing up in the URL bar autocomplete results - PRBool isBookmark = !IsQueryURI(url); - - if (isBookmark) { - // if it is a livemark item (the parent is a livemark), - // we pass in false for isBookmark. otherwise, unvisited livemark - // items will appear in URL autocomplete before we visit them. - PRBool parentIsLivemark; - nsCOMPtr lms = - do_GetService(NS_LIVEMARKSERVICE_CONTRACTID, &rv); - NS_ENSURE_SUCCESS(rv, rv); - - rv = lms->IsLivemark(aFolder, &parentIsLivemark); - NS_ENSURE_SUCCESS(rv, rv); - - isBookmark = !parentIsLivemark; - } - - // when we created the moz_place entry for the new bookmark - // (a side effect of calling GetUrlIdFor()) frecency -1; - // now we re-calculate the frecency for this moz_place entry. - rv = history->UpdateFrecency(childID, isBookmark); + // Re-calculate the frecency for this moz_place entry since it was set to -1. + rv = history->UpdateFrecency(childID); NS_ENSURE_SUCCESS(rv, rv); rv = transaction.Commit(); @@ -1119,12 +1052,9 @@ nsNavBookmarks::RemoveItem(PRInt64 aItemId) NS_ENSURE_SUCCESS(rv, rv); if (itemType == TYPE_BOOKMARK) { - // UpdateFrecency needs to know whether placeId is still bookmarked. - // Although we removed aItemId, placeId may still be bookmarked elsewhere; - // IsRealBookmark will know. nsNavHistory* history = nsNavHistory::GetHistoryService(); NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); - rv = history->UpdateFrecency(placeId, IsRealBookmark(placeId)); + rv = history->UpdateFrecency(placeId); NS_ENSURE_SUCCESS(rv, rv); rv = UpdateKeywordsHashForRemovedBookmark(aItemId); @@ -1689,13 +1619,9 @@ nsNavBookmarks::RemoveFolderChildren(PRInt64 aFolderId) folderChildrenInfo child = folderChildrenArray[i]; if (child.itemType == TYPE_BOOKMARK) { PRInt64 placeId = child.placeId; - - // UpdateFrecency needs to know whether placeId is still bookmarked. - // Although we removed a child of aFolderId that bookmarked it, it may - // still be bookmarked elsewhere; IsRealBookmark will know. nsNavHistory* history = nsNavHistory::GetHistoryService(); NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); - rv = history->UpdateFrecency(placeId, IsRealBookmark(placeId)); + rv = history->UpdateFrecency(placeId); NS_ENSURE_SUCCESS(rv, rv); rv = UpdateKeywordsHashForRemovedBookmark(child.itemId); @@ -2566,20 +2492,12 @@ nsNavBookmarks::ChangeBookmarkURI(PRInt64 aBookmarkId, nsIURI* aNewURI) rv = transaction.Commit(); NS_ENSURE_SUCCESS(rv, rv); - // Upon changing the URI for a bookmark, update the frecency for the new place. - // UpdateFrecency needs to know whether placeId is bookmarked (as opposed - // to a livemark item). Bookmarking it is exactly what we did above. - rv = history->UpdateFrecency(placeId, PR_TRUE /* isBookmarked */); + rv = history->UpdateFrecency(placeId); NS_ENSURE_SUCCESS(rv, rv); - // Upon changing the URI for a bookmark, update the frecency for the old place. - // UpdateFrecency again needs to know whether oldPlaceId is bookmarked. It may - // no longer be, so we need to figure out whether it still is. Our strategy - // is: find all bookmarks corresponding to oldPlaceId that are not livemark - // items, i.e., whose parents are not livemarks. If any such bookmarks exist, - // oldPlaceId is still bookmarked. - - rv = history->UpdateFrecency(oldPlaceId, IsRealBookmark(oldPlaceId)); + // Upon changing the URI for a bookmark, update the frecency for the old + // place as well. + rv = history->UpdateFrecency(oldPlaceId); NS_ENSURE_SUCCESS(rv, rv); nsCAutoString spec; @@ -2631,8 +2549,6 @@ nsNavBookmarks::GetBookmarkIdsForURITArray(nsIURI* aURI, DECLARE_AND_ASSIGN_SCOPED_LAZY_STMT(stmt, mDBFindURIBookmarks); nsresult rv = URIBinder::Bind(stmt, NS_LITERAL_CSTRING("page_url"), aURI); NS_ENSURE_SUCCESS(rv, rv); - rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("item_type"), TYPE_BOOKMARK); - NS_ENSURE_SUCCESS(rv, rv); PRBool more; while (NS_SUCCEEDED((rv = stmt->ExecuteStep(&more))) && more) { diff --git a/toolkit/components/places/src/nsNavHistory.cpp b/toolkit/components/places/src/nsNavHistory.cpp index a34f367a6e7..dcfc67cf03a 100644 --- a/toolkit/components/places/src/nsNavHistory.cpp +++ b/toolkit/components/places/src/nsNavHistory.cpp @@ -65,7 +65,7 @@ #include "nsThreadUtils.h" #include "nsAppDirectoryServiceDefs.h" #include "nsMathUtils.h" -#include "mozIStorageCompletionCallback.h" +#include "mozIStorageAsyncStatement.h" #include "nsNavBookmarks.h" #include "nsAnnotationService.h" @@ -134,7 +134,7 @@ using namespace mozilla::places; // This is the schema version, update it at any schema change and add a // corresponding migrateVxx method below. -#define DATABASE_SCHEMA_VERSION 10 +#define DATABASE_SCHEMA_VERSION 11 // Filename of the database. #define DATABASE_FILENAME NS_LITERAL_STRING("places.sqlite") @@ -142,10 +142,6 @@ using namespace mozilla::places; // Filename used to backup corrupt databases. #define DATABASE_CORRUPT_FILENAME NS_LITERAL_STRING("places.sqlite.corrupt") -// We use the TRUNCATE journal mode to reduce the number of fsyncs. Without -// this setting we had a Ts hit on Linux. See bug 460315 for details. -#define DATABASE_JOURNAL_MODE "TRUNCATE" - // Fraction of free pages in the database to force a vacuum between // DATABASE_MAX_TIME_BEFORE_VACUUM and DATABASE_MIN_TIME_BEFORE_VACUUM. #define DATABASE_VACUUM_FREEPAGES_THRESHOLD 0.1 @@ -227,8 +223,6 @@ static void ParseSearchTermsFromQueries(const nsCOMArray& aQu class VacuumDBListener : public AsyncStatementCallback { public: - NS_DECL_ISUPPORTS - VacuumDBListener(nsIPrefBranch* aBranch) : mPrefBranch(aBranch) { @@ -253,8 +247,6 @@ private: nsCOMPtr mPrefBranch; }; -NS_IMPL_ISUPPORTS1(VacuumDBListener, mozIStorageStatementCallback) - } // anonymous namespace namespace mozilla { @@ -389,7 +381,7 @@ protected: } } - const char* mTopic; + const char* const mTopic; bool mDoubleEnqueue; }; @@ -399,6 +391,36 @@ NS_IMPL_ISUPPORTS2( , nsIRunnable ) + +// Used to notify a topic to system observers on async execute completion. +class AsyncStatementCallbackNotifier : public AsyncStatementCallback +{ +public: + AsyncStatementCallbackNotifier(const char* aTopic) + : mTopic(aTopic) + { + } + + NS_IMETHOD HandleCompletion(PRUint16 aReason); + +private: + const char* mTopic; +}; + +NS_IMETHODIMP +AsyncStatementCallbackNotifier::HandleCompletion(PRUint16 aReason) +{ + if (aReason != mozIStorageStatementCallback::REASON_FINISHED) + return NS_ERROR_UNEXPECTED; + + nsCOMPtr obs = mozilla::services::GetObserverService(); + if (obs) { + (void)obs->NotifyObservers(nsnull, mTopic, nsnull); + } + + return NS_OK; +} + } // anonymouse namespace @@ -425,6 +447,8 @@ PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavHistory, gHistoryService) nsNavHistory::nsNavHistory() : mBatchLevel(0) , mBatchDBTransaction(nsnull) +, mDBPageSize(0) +, mCurrentJournalMode(JOURNAL_DELETE) , mCachedNow(0) , mExpireNowTimer(nsnull) , mLastSessionID(0) @@ -669,6 +693,57 @@ nsNavHistory::InitDBFile(PRBool aForceInit) } +nsresult +nsNavHistory::SetJournalMode(enum JournalMode aJournalMode) +{ + nsCAutoString journalMode; + switch (aJournalMode) { + default: + NS_NOTREACHED("Trying to set an unknown journal mode."); + // Fall through to the default mode of DELETE. + case JOURNAL_DELETE: + journalMode.AssignLiteral("delete"); + break; + case JOURNAL_TRUNCATE: + journalMode.AssignLiteral("truncate"); + break; + case JOURNAL_MEMORY: + journalMode.AssignLiteral("memory"); + break; + case JOURNAL_WAL: + journalMode.AssignLiteral("wal"); + break; + } + + nsCOMPtr statement; + nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = ") + journalMode, + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + mozStorageStatementScoper scoper(statement); + PRBool hasResult; + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(hasResult, NS_ERROR_FAILURE); + + nsCAutoString currentJournalMode; + rv = statement->GetUTF8String(0, currentJournalMode); + NS_ENSURE_SUCCESS(rv, rv); + bool succeeded = currentJournalMode.Equals(journalMode); + if (succeeded) { + mCurrentJournalMode = aJournalMode; + } + else { + NS_WARNING(nsPrintfCString(128, "Setting journal mode failed: %s", + PromiseFlatCString(journalMode).get()).get()); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + + nsresult nsNavHistory::InitDB() { @@ -676,19 +751,22 @@ nsNavHistory::InitDB() PRInt32 currentSchemaVersion = 0; nsresult rv = mDBConn->GetSchemaVersion(¤tSchemaVersion); NS_ENSURE_SUCCESS(rv, rv); + { + // Get the page size. This may be different than the default if the + // database file already existed with a different page size. + nsCOMPtr statement; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); - // Get the page size. This may be different than the default if the - // database file already existed with a different page size. - nsCOMPtr statement; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), - getter_AddRefs(statement)); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool hasResult; - mozStorageStatementScoper scoper(statement); - rv = statement->ExecuteStep(&hasResult); - NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); - PRInt32 pageSize = statement->AsInt32(0); + PRBool hasResult; + mozStorageStatementScoper scoper(statement); + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); + rv = statement->GetInt32(0, &mDBPageSize); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(mDBPageSize > 0, NS_ERROR_UNEXPECTED); + } // Ensure that temp tables are held in memory, not on disk. We use temp // tables mainly for fsync and I/O reduction. @@ -720,21 +798,20 @@ nsNavHistory::InitDB() PRInt64 cacheSize = physMem * cachePercentage / 100; // Compute number of cached pages, this will be our cache size. - PRInt64 cachePages = cacheSize / pageSize; - nsCAutoString pageSizePragma("PRAGMA cache_size = "); - pageSizePragma.AppendInt(cachePages); - rv = mDBConn->ExecuteSimpleSQL(pageSizePragma); + PRInt64 cachePages = cacheSize / mDBPageSize; + nsCAutoString cacheSizePragma("PRAGMA cache_size = "); + cacheSizePragma.AppendInt(cachePages); + rv = mDBConn->ExecuteSimpleSQL(cacheSizePragma); NS_ENSURE_SUCCESS(rv, rv); - // Lock the database file. This is done partly to avoid third party - // applications to access it while it's in use, partly for performance. - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA locking_mode = EXCLUSIVE")); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); - NS_ENSURE_SUCCESS(rv, rv); + // Be sure to set journal mode after page_size. WAL would prevent the change + // otherwise. + if (NS_FAILED(SetJournalMode(JOURNAL_WAL))) { + // Ignore errors, if we fail here the database could be considered corrupt + // and we won't be able to go on, even if it's just matter of a bogus file + // system. The default mode (DELETE) will be fine in such a case. + (void)SetJournalMode(JOURNAL_TRUNCATE); + } // We are going to initialize tables, so everything from now on should be in // a transaction for performances. @@ -822,6 +899,12 @@ nsNavHistory::InitDB() NS_ENSURE_SUCCESS(rv, rv); } + // Migrate places up to V11 + if (currentSchemaVersion < 11) { + rv = MigrateV11Up(mDBConn); + NS_ENSURE_SUCCESS(rv, rv); + } + // Schema Upgrades must add migration code here. } else { @@ -877,6 +960,12 @@ nsNavHistory::InitDB() rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_PLACEDATE); NS_ENSURE_SUCCESS(rv, rv); + // Visits triggers. + rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + // This makes a big difference in startup time for large profiles because of // finding bookmark redirects using the referring page. rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_FROMVISIT); @@ -907,8 +996,6 @@ nsNavHistory::InitAdditionalDBItems() { nsresult rv = InitTempTables(); NS_ENSURE_SUCCESS(rv, rv); - rv = InitViews(); - NS_ENSURE_SUCCESS(rv, rv); rv = InitFunctions(); NS_ENSURE_SUCCESS(rv, rv); rv = InitStatements(); @@ -1001,87 +1088,14 @@ mozStorageFunctionGetUnreversedHost::OnFunctionCall( nsresult nsNavHistory::InitTempTables() { - nsresult rv; - - // moz_places_temp - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_TEMP); + nsresult rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_OPENPAGES_TEMP); NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_TEMP_URL); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_TEMP_FAVICON); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_TEMP_REVHOST); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_TEMP_VISITCOUNT); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_TEMP_FRECENCY); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_SYNC_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - - - // moz_historyvisits_temp - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_HISTORYVISITS_TEMP); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_TEMP_PLACEDATE); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_TEMP_FROMVISIT); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_TEMP_VISITDATE); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_HISTORYVISITS_SYNC_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - - // moz_openpages_temp - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_OPENPAGES_TEMP); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(CREATE_REMOVEOPENPAGE_CLEANUP_TRIGGER); NS_ENSURE_SUCCESS(rv, rv); return NS_OK; } -nsresult -nsNavHistory::InitViews() -{ - nsresult rv; - - // moz_places_view - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_VIEW); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_PLACES_VIEW_INSERT_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(CREATE_PLACES_VIEW_DELETE_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(CREATE_PLACES_VIEW_UPDATE_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - - // moz_historyvisits_view - rv = mDBConn->ExecuteSimpleSQL(CREATE_MOZ_HISTORYVISITS_VIEW); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_VIEW_INSERT_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_VIEW_DELETE_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_VIEW_UPDATE_TRIGGER); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; -} - nsresult nsNavHistory::InitFunctions() { @@ -1096,6 +1110,9 @@ nsNavHistory::InitFunctions() rv = MatchAutoCompleteFunction::create(mDBConn); NS_ENSURE_SUCCESS(rv, rv); + rv = CalculateFrecencyFunction::create(mDBConn); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; } @@ -1106,182 +1123,107 @@ nsNavHistory::InitFunctions() nsresult nsNavHistory::InitStatements() { - // mDBGetURLPageInfo - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique urls. nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, url, title, rev_host, visit_count " - "FROM moz_places_temp " - "WHERE url = :page_url " - "UNION ALL " "SELECT id, url, title, rev_host, visit_count " "FROM moz_places " "WHERE url = :page_url " - "LIMIT 1"), - getter_AddRefs(mDBGetURLPageInfo)); + ), getter_AddRefs(mDBGetURLPageInfo)); NS_ENSURE_SUCCESS(rv, rv); - // mDBGetIdPageInfo - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique place ids. - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, url, title, rev_host, visit_count " - "FROM moz_places_temp " - "WHERE id = :page_id " - "UNION ALL " - "SELECT id, url, title, rev_host, visit_count " - "FROM moz_places " - "WHERE id = :page_id " - "LIMIT 1"), - getter_AddRefs(mDBGetIdPageInfo)); + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT id, url, title, rev_host, visit_count " + "FROM moz_places " + "WHERE id = :page_id " + ), getter_AddRefs(mDBGetIdPageInfo)); NS_ENSURE_SUCCESS(rv, rv); - // mDBRecentVisitOfURL - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // expect visits in temp table being the most recent. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, session, visit_date " - "FROM moz_historyvisits_temp " - "WHERE place_id = IFNULL( " - "(SELECT id FROM moz_places_temp WHERE url = :page_url), " - "(SELECT id FROM moz_places WHERE url = :page_url) " - ") " - "UNION ALL " - "SELECT id, session, visit_date " - "FROM moz_historyvisits " - "WHERE place_id = IFNULL( " - "(SELECT id FROM moz_places_temp WHERE url = :page_url), " - "(SELECT id FROM moz_places WHERE url = :page_url) " - ") " - "ORDER BY visit_date DESC " - "LIMIT 1 "), - getter_AddRefs(mDBRecentVisitOfURL)); + "SELECT id, session, visit_date " + "FROM moz_historyvisits " + "WHERE place_id = (SELECT id FROM moz_places WHERE url = :page_url) " + "ORDER BY visit_date DESC " + ), getter_AddRefs(mDBRecentVisitOfURL)); NS_ENSURE_SUCCESS(rv, rv); - // mDBRecentVisitOfPlace - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // expect visits in temp table being the most recent. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id FROM moz_historyvisits_temp " - "WHERE place_id = :page_id " - "AND visit_date = :visit_date " - "AND session = :session " - "UNION ALL " - "SELECT id FROM moz_historyvisits " - "WHERE place_id = :page_id " - "AND visit_date = :visit_date " - "AND session = :session " - "LIMIT 1"), - getter_AddRefs(mDBRecentVisitOfPlace)); + "SELECT id FROM moz_historyvisits " + "WHERE place_id = :page_id " + "AND visit_date = :visit_date " + "AND session = :session " + ), getter_AddRefs(mDBRecentVisitOfPlace)); NS_ENSURE_SUCCESS(rv, rv); - // mDBInsertVisit rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "INSERT INTO moz_historyvisits_view " - "(from_visit, place_id, visit_date, visit_type, session) " - "VALUES (:from_visit, :page_id, :visit_date, :visit_type, :session)"), - getter_AddRefs(mDBInsertVisit)); + "INSERT INTO moz_historyvisits " + "(from_visit, place_id, visit_date, visit_type, session) " + "VALUES (:from_visit, :page_id, :visit_date, :visit_type, :session) " + ), getter_AddRefs(mDBInsertVisit)); NS_ENSURE_SUCCESS(rv, rv); - // mDBGetPageVisitStats (see InternalAdd) - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique place ids. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, visit_count, typed, hidden " - "FROM moz_places_temp " - "WHERE url = :page_url " - "UNION ALL " - "SELECT id, visit_count, typed, hidden " - "FROM moz_places " - "WHERE url = :page_url " - "LIMIT 1"), - getter_AddRefs(mDBGetPageVisitStats)); + "SELECT id, visit_count, typed, hidden " + "FROM moz_places " + "WHERE url = :page_url " + ), getter_AddRefs(mDBGetPageVisitStats)); NS_ENSURE_SUCCESS(rv, rv); - // mDBIsPageVisited - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // only need to know if a visit exists. - // Use indexed params here for performance. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT h.id " - "FROM moz_places_temp h " - "WHERE url = ?1 " - "AND ( " - "EXISTS(SELECT id FROM moz_historyvisits_temp WHERE place_id = h.id LIMIT 1) " - "OR EXISTS(SELECT id FROM moz_historyvisits WHERE place_id = h.id LIMIT 1) " - ") " - "UNION ALL " - "SELECT h.id " - "FROM moz_places h " - "WHERE url = ?1 " - "AND ( " - "EXISTS(SELECT id FROM moz_historyvisits_temp WHERE place_id = h.id LIMIT 1) " - "OR EXISTS(SELECT id FROM moz_historyvisits WHERE place_id = h.id LIMIT 1) " - ") " - "LIMIT 1"), - getter_AddRefs(mDBIsPageVisited)); + "SELECT h.id " + "FROM moz_places h " + "WHERE url = ?1 " + "AND EXISTS(SELECT id FROM moz_historyvisits WHERE place_id = h.id LIMIT 1) " + ), getter_AddRefs(mDBIsPageVisited)); NS_ENSURE_SUCCESS(rv, rv); - // mDBUpdatePageVisitStats (see InternalAdd) - // we don't need to update visit_count since it's maintained - // in sync by triggers, and we must NEVER touch it rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "UPDATE moz_places_view " - "SET hidden = :hidden, typed = :typed " - "WHERE id = :page_id"), - getter_AddRefs(mDBUpdatePageVisitStats)); + "UPDATE moz_places " + "SET hidden = :hidden, typed = :typed " + "WHERE id = :page_id " + ), getter_AddRefs(mDBUpdatePageVisitStats)); NS_ENSURE_SUCCESS(rv, rv); - // mDBAddNewPage (see InternalAddNewPage) + // We have both sync and async users, so it could happen that an async + // statement tries to insert a page when a sync statement just added it. + // We should ignore the insertion in such a case, the async implementer + // will fetch the id of the existing entry. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "INSERT INTO moz_places_view " - "(url, title, rev_host, hidden, typed, frecency) " - "VALUES (:page_url, :page_title, :rev_host, :hidden, :typed, :frecency)"), - getter_AddRefs(mDBAddNewPage)); + "INSERT OR IGNORE INTO moz_places " + "(url, title, rev_host, hidden, typed, frecency) " + "VALUES (:page_url, :page_title, :rev_host, :hidden, :typed, :frecency) " + ), getter_AddRefs(mDBAddNewPage)); NS_ENSURE_SUCCESS(rv, rv); - // mDBGetTags rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "/* do not warn (bug 487594) */ " - "SELECT GROUP_CONCAT(tag_title, ', ') " - "FROM ( " - "SELECT t.title AS tag_title " - "FROM moz_bookmarks b " - "JOIN moz_bookmarks t ON t.id = b.parent " - "WHERE b.fk = IFNULL((SELECT id FROM moz_places_temp WHERE url = :page_url), " - "(SELECT id FROM moz_places WHERE url = :page_url)) " - "AND LENGTH(t.title) > 0 " - "AND b.type = ") + - nsPrintfCString("%d", nsINavBookmarksService::TYPE_BOOKMARK) + - NS_LITERAL_CSTRING(" AND t.parent = :tags_folder " - "ORDER BY t.title COLLATE NOCASE ASC)"), - getter_AddRefs(mDBGetTags)); + "/* do not warn (bug 487594) */ " + "SELECT GROUP_CONCAT(tag_title, ', ') " + "FROM ( " + "SELECT t.title AS tag_title " + "FROM moz_bookmarks b " + "JOIN moz_bookmarks t ON t.id = b.parent " + "WHERE b.fk = (SELECT id FROM moz_places WHERE url = :page_url) " + "AND LENGTH(t.title) > 0 " + "AND b.type = ") + + nsPrintfCString("%d", nsINavBookmarksService::TYPE_BOOKMARK) + + NS_LITERAL_CSTRING(" AND t.parent = :tags_folder " + "ORDER BY t.title COLLATE NOCASE ASC) " + ), getter_AddRefs(mDBGetTags)); NS_ENSURE_SUCCESS(rv, rv); - // mDBGetItemsWithAnno rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT a.item_id, a.content " - "FROM moz_anno_attributes n " - "JOIN moz_items_annos a ON n.id = a.anno_attribute_id " - "WHERE n.name = :anno_name"), - getter_AddRefs(mDBGetItemsWithAnno)); - NS_ENSURE_SUCCESS(rv, rv); - - // mDBSetPlaceTitle - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "UPDATE moz_places_view " - "SET title = :page_title " - "WHERE url = :page_url"), - getter_AddRefs(mDBSetPlaceTitle)); + "SELECT a.item_id, a.content " + "FROM moz_anno_attributes n " + "JOIN moz_items_annos a ON n.id = a.anno_attribute_id " + "WHERE n.name = :anno_name " + ), getter_AddRefs(mDBGetItemsWithAnno)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE moz_places " + "SET title = :page_title " + "WHERE url = :page_url " + ), getter_AddRefs(mDBSetPlaceTitle)); NS_ENSURE_SUCCESS(rv, rv); - // mDBRegisterOpenPage rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( "INSERT OR REPLACE INTO moz_openpages_temp (url, open_count) " "VALUES (:page_url, " @@ -1289,19 +1231,17 @@ nsNavHistory::InitStatements() "(SELECT open_count + 1 FROM moz_openpages_temp WHERE url = :page_url), " "1" ")" - ")"), - getter_AddRefs(mDBRegisterOpenPage)); + ")" + ), getter_AddRefs(mDBRegisterOpenPage)); NS_ENSURE_SUCCESS(rv, rv); - // mDBUnregisterOpenPage rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( "UPDATE moz_openpages_temp " "SET open_count = open_count - 1 " - "WHERE url = :page_url"), - getter_AddRefs(mDBUnregisterOpenPage)); + "WHERE url = :page_url" + ), getter_AddRefs(mDBUnregisterOpenPage)); NS_ENSURE_SUCCESS(rv, rv); - // mDBVisitsForFrecency // NOTE: This is not limited to visits with "visit_type NOT IN (0,4,7,8)" // because otherwise mDBVisitsForFrecency would return no visits // for places with only embed (or undefined) visits. That would @@ -1310,71 +1250,72 @@ nsNavHistory::InitStatements() // frecency. // In case of a temporary or permanent redirect, calculate the frecency as if // the original page was visited. - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT v.visit_date, COALESCE( " - "(SELECT r.visit_type FROM moz_historyvisits_temp r " - "WHERE v.visit_type IN ") + - nsPrintfCString("(%d,%d) ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY) + - NS_LITERAL_CSTRING(" AND r.id = v.from_visit), " - "(SELECT r.visit_type FROM moz_historyvisits r " - "WHERE v.visit_type IN ") + - nsPrintfCString("(%d,%d) ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY) + - NS_LITERAL_CSTRING(" AND r.id = v.from_visit), " - "visit_type) " - "FROM moz_historyvisits_temp v " - "WHERE v.place_id = :page_id " - "UNION ALL " - "SELECT v.visit_date, COALESCE( " - "(SELECT r.visit_type FROM moz_historyvisits_temp r " - "WHERE v.visit_type IN ") + - nsPrintfCString("(%d,%d) ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY) + - NS_LITERAL_CSTRING(" AND r.id = v.from_visit), " - "(SELECT r.visit_type FROM moz_historyvisits r " - "WHERE v.visit_type IN ") + - nsPrintfCString("(%d,%d) ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY) + - NS_LITERAL_CSTRING(" AND r.id = v.from_visit), " - "visit_type) " - "FROM moz_historyvisits v " - "WHERE v.place_id = :page_id " - "AND v.id NOT IN (SELECT id FROM moz_historyvisits_temp) " - "ORDER BY visit_date DESC LIMIT ") + - nsPrintfCString("%d", mNumVisitsForFrecency), - getter_AddRefs(mDBVisitsForFrecency)); + nsCAutoString visitsForFrecencySQL(NS_LITERAL_CSTRING( + "SELECT " + "ROUND((strftime('%s','now','localtime','utc') - v.visit_date/1000000)/86400), " + "COALESCE( " + "(SELECT r.visit_type FROM moz_historyvisits r WHERE v.visit_type IN ") + + nsPrintfCString("(%d,%d) ", TRANSITION_REDIRECT_PERMANENT, + TRANSITION_REDIRECT_TEMPORARY) + + NS_LITERAL_CSTRING(" AND r.id = v.from_visit), " + "visit_type), " + "visit_date " + "FROM moz_historyvisits v " + "WHERE v.place_id = :page_id " + "ORDER BY visit_date DESC LIMIT ") + + nsPrintfCString("%d", mNumVisitsForFrecency) + ); + + rv = mDBConn->CreateStatement(visitsForFrecencySQL, + getter_AddRefs(mDBVisitsForFrecency)); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->CreateStatement(visitsForFrecencySQL, + getter_AddRefs(mDBAsyncThreadVisitsForFrecency)); NS_ENSURE_SUCCESS(rv, rv); - // mDBUpdateFrecencyAndHidden rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "UPDATE moz_places_view SET frecency = :frecency, hidden = :hidden " - "WHERE id = :page_id"), - getter_AddRefs(mDBUpdateFrecencyAndHidden)); + "UPDATE moz_places " + "SET frecency = CALCULATE_FRECENCY(:page_id) " + "WHERE id = :page_id" + ), getter_AddRefs(mDBUpdateFrecency)); NS_ENSURE_SUCCESS(rv, rv); - // mDBGetPlaceVisitStats - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique place ids. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT typed, hidden, frecency " - "FROM moz_places_temp WHERE id = :page_id " - "UNION ALL " - "SELECT typed, hidden, frecency " - "FROM moz_places WHERE id = :page_id " - "LIMIT 1"), - getter_AddRefs(mDBGetPlaceVisitStats)); + "UPDATE moz_places " + "SET hidden = 0 " + "WHERE id = :page_id AND frecency <> 0" + ), getter_AddRefs(mDBUpdateHiddenOnFrecency)); NS_ENSURE_SUCCESS(rv, rv); - // when calculating frecency, we want the visit count to be - // all the visits. rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT " - "(SELECT COUNT(*) FROM moz_historyvisits WHERE place_id = :page_id) + " - "(SELECT COUNT(*) FROM moz_historyvisits_temp WHERE place_id = :page_id " - "AND id NOT IN (SELECT id FROM moz_historyvisits))"), - getter_AddRefs(mDBFullVisitCount)); + "SELECT typed, hidden, frecency " + "FROM moz_places " + "WHERE id = :page_id " + ), getter_AddRefs(mDBGetPlaceVisitStats)); + NS_ENSURE_SUCCESS(rv, rv); + + // When calculating frecency, we need special information for the page. + nsCAutoString pageInfoForFrecencySQL(NS_LITERAL_CSTRING( + "SELECT typed, hidden, visit_count, " + "(SELECT count(*) FROM moz_historyvisits WHERE place_id = :page_id), " + "(SELECT id FROM moz_bookmarks " + "WHERE fk = :page_id " + "AND parent NOT IN (" + "SELECT a.item_id " + "FROM moz_items_annos a " + "JOIN moz_anno_attributes n ON a.anno_attribute_id = n.id " + "WHERE n.name = :anno_name" + ") " + "LIMIT 1), " + "(url > 'place:' AND url < 'place;') " + "FROM moz_places " + "WHERE id = :page_id " + )); + rv = mDBConn->CreateStatement(pageInfoForFrecencySQL, + getter_AddRefs(mDBPageInfoForFrecency)); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->CreateStatement(pageInfoForFrecencySQL, + getter_AddRefs(mDBAsyncThreadPageInfoForFrecency)); NS_ENSURE_SUCCESS(rv, rv); return NS_OK; @@ -1807,8 +1748,7 @@ nsNavHistory::MigrateV9Up(mozIStorageConnection *aDBConn) // Added in Bug 488966. The last_visit_date column caches the last // visit date, this enhances SELECT performances when we // need to sort visits by visit date. - // The cached value is synced by INSERT and DELETE triggers on - // moz_historyvisits_view, on every added or removed visit. + // The cached value is synced by triggers on every added or removed visit. // See nsPlacesTriggers.h for details on the triggers. PRBool oldIndexExists = PR_FALSE; nsresult rv = mDBConn->IndexExists( @@ -1828,25 +1768,12 @@ nsNavHistory::MigrateV9Up(mozIStorageConnection *aDBConn) // This query can be really slow due to disk access, since it will basically // dupe the table contents in the journal file, and then write them down // in the database. - // We will temporary use a memory journal file, this has the advantage of - // reducing write times by a half, but will temporary consume more memory - // and increase risks of corruption if we should crash in the middle of this - // update. - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = MEMORY")); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "UPDATE moz_places SET last_visit_date = " "(SELECT MAX(visit_date) " "FROM moz_historyvisits " "WHERE place_id = moz_places.id)")); NS_ENSURE_SUCCESS(rv, rv); - - // Restore the default journal mode. - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); - NS_ENSURE_SUCCESS(rv, rv); } return transaction.Commit(); @@ -1865,7 +1792,60 @@ nsNavHistory::MigrateV10Up(mozIStorageConnection *aDBConn) return NS_OK; } - + + +nsresult +nsNavHistory::MigrateV11Up(mozIStorageConnection *aDBConn) +{ + // Temp tables are going away, so we need the visit_count triggers to return. + + // Regardless triggers existance, every time we pass through this migration + // step, we must ensure correctness of visit_count values. + nsresult rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "UPDATE moz_places SET visit_count = " + "(SELECT count(*) FROM moz_historyvisits " + "WHERE place_id = moz_places.id " + "AND visit_type NOT IN ") + + nsPrintfCString("(0,%d,%d,%d) ", + nsINavHistoryService::TRANSITION_EMBED, + nsINavHistoryService::TRANSITION_FRAMED_LINK, + nsINavHistoryService::TRANSITION_DOWNLOAD) + + NS_LITERAL_CSTRING(")") + ); + NS_ENSURE_SUCCESS(rv, rv); + + // Now create triggers if needed. + nsCOMPtr triggerDetection; + rv = aDBConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT name " + "FROM sqlite_master " + "WHERE type = 'trigger' " + "AND name = :trigger_name"), + getter_AddRefs(triggerDetection)); + NS_ENSURE_SUCCESS(rv, rv); + + // Both visits triggers were removed in the past at the same time, so we can + // check just one of them. + PRBool triggerExists; + rv = triggerDetection->BindUTF8StringByName( + NS_LITERAL_CSTRING("trigger_name"), + NS_LITERAL_CSTRING("moz_historyvisits_afterinsert_v2_trigger") + ); + NS_ENSURE_SUCCESS(rv, rv); + rv = triggerDetection->ExecuteStep(&triggerExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!triggerExists) { + rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mDBConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + + // nsNavHistory::GetUrlIdFor // // Called by the bookmarks and annotation services, this function returns the @@ -1932,7 +1912,6 @@ nsNavHistory::InternalAddNewPage(nsIURI* aURI, nsresult rv = URIBinder::Bind(mDBAddNewPage, NS_LITERAL_CSTRING("page_url"), aURI); NS_ENSURE_SUCCESS(rv, rv); - // title if (aTitle.IsVoid()) { rv = mDBAddNewPage->BindNullByName(NS_LITERAL_CSTRING("page_title")); } @@ -1954,34 +1933,22 @@ nsNavHistory::InternalAddNewPage(nsIURI* aURI, } NS_ENSURE_SUCCESS(rv, rv); - // hidden rv = mDBAddNewPage->BindInt32ByName(NS_LITERAL_CSTRING("hidden"), aHidden); NS_ENSURE_SUCCESS(rv, rv); - - // typed rv = mDBAddNewPage->BindInt32ByName(NS_LITERAL_CSTRING("typed"), aTyped); NS_ENSURE_SUCCESS(rv, rv); - - nsCAutoString url; - rv = aURI->GetSpec(url); + nsCAutoString spec; + rv = aURI->GetSpec(spec); NS_ENSURE_SUCCESS(rv, rv); - - // frecency - PRInt32 frecency = -1; - if (aCalculateFrecency) { - rv = CalculateFrecency(-1 /* no page id, since this page doesn't exist */, - aTyped, aVisitCount, url, &frecency); - NS_ENSURE_SUCCESS(rv, rv); - } - - rv = mDBAddNewPage->BindInt32ByName(NS_LITERAL_CSTRING("frecency"), frecency); + rv = mDBAddNewPage->BindInt32ByName(NS_LITERAL_CSTRING("frecency"), + IsQueryURI(spec) ? 0 : -1); NS_ENSURE_SUCCESS(rv, rv); rv = mDBAddNewPage->Execute(); NS_ENSURE_SUCCESS(rv, rv); - // If the caller wants the page ID, go get it - if (aPageID) { + PRInt64 pageId; + { mozStorageStatementScoper scoper(mDBGetURLPageInfo); rv = URIBinder::Bind(mDBGetURLPageInfo, NS_LITERAL_CSTRING("page_url"), aURI); @@ -1991,8 +1958,17 @@ nsNavHistory::InternalAddNewPage(nsIURI* aURI, rv = mDBGetURLPageInfo->ExecuteStep(&hasResult); NS_ENSURE_SUCCESS(rv, rv); NS_ASSERTION(hasResult, "hasResult is false but the call succeeded?"); + pageId = mDBGetURLPageInfo->AsInt64(0); + } - *aPageID = mDBGetURLPageInfo->AsInt64(0); + if (aCalculateFrecency) { + rv = UpdateFrecency(pageId); + NS_ENSURE_SUCCESS(rv, rv); + } + + // If the caller wants the page ID, return it. + if (aPageID) { + *aPageID = pageId; } return NS_OK; @@ -2186,12 +2162,11 @@ nsNavHistory::GetNewSessionID() // Extract the last session ID, so we know where to pick up. There is no // index over sessions so we use the visit_date index. - // This happens on the first visit, so we don't care about temp tables. nsCOMPtr selectSession; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT session FROM moz_historyvisits " - "ORDER BY visit_date DESC LIMIT 1"), - getter_AddRefs(selectSession)); + "SELECT session FROM moz_historyvisits " + "ORDER BY visit_date DESC " + ), getter_AddRefs(selectSession)); NS_ENSURE_SUCCESS(rv, rv); PRBool hasSession; if (NS_SUCCEEDED(selectSession->ExecuteStep(&hasSession)) && hasSession) @@ -2234,8 +2209,6 @@ nsNavHistory::GetDaysOfHistory() { "strftime('%s','now','localtime','utc') - " "( " "SELECT visit_date FROM moz_historyvisits " - "UNION ALL " - "SELECT visit_date FROM moz_historyvisits_temp " "ORDER BY visit_date ASC LIMIT 1 " ")/1000000 " ")/86400) AS daysOfHistory "), @@ -2606,9 +2579,7 @@ nsNavHistory::GetHasHistoryEntries(PRBool* aHasEntries) nsCOMPtr dbSelectStatement; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT 1 " - "WHERE EXISTS (SELECT id FROM moz_historyvisits_temp LIMIT 1) " - "OR EXISTS (SELECT id FROM moz_historyvisits LIMIT 1)"), + "SELECT 1 FROM moz_historyvisits "), getter_AddRefs(dbSelectStatement)); NS_ENSURE_SUCCESS(rv, rv); rv = dbSelectStatement->ExecuteStep(aHasEntries); @@ -2618,6 +2589,7 @@ nsNavHistory::GetHasHistoryEntries(PRBool* aHasEntries) return NS_OK; } + nsresult nsNavHistory::FixInvalidFrecenciesForExcludedPlaces() { @@ -2626,14 +2598,11 @@ nsNavHistory::FixInvalidFrecenciesForExcludedPlaces() // set frecency to 0 so that it is excluded from url bar autocomplete. nsCOMPtr dbUpdateStatement; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "UPDATE moz_places_view " + "UPDATE moz_places " "SET frecency = 0 WHERE id IN (" "SELECT h.id FROM moz_places h " "WHERE h.url >= 'place:' AND h.url < 'place;' " "UNION " - "SELECT h.id FROM moz_places_temp h " - "WHERE h.url >= 'place:' AND h.url < 'place;' " - "UNION " // Unvisited child of a livemark "SELECT b.fk FROM moz_bookmarks b " "JOIN moz_bookmarks bp ON bp.id = b.parent " @@ -2642,8 +2611,6 @@ nsNavHistory::FixInvalidFrecenciesForExcludedPlaces() "WHERE n.name = :anno_name " "AND b.fk IN( " "SELECT id FROM moz_places WHERE visit_count = 0 AND frecency < 0 " - "UNION ALL " - "SELECT id FROM moz_places_temp WHERE visit_count = 0 AND frecency < 0 " ") " ")"), getter_AddRefs(dbUpdateStatement)); @@ -2660,30 +2627,6 @@ nsNavHistory::FixInvalidFrecenciesForExcludedPlaces() return NS_OK; } -nsresult -nsNavHistory::CalculateFullVisitCount(PRInt64 aPlaceId, PRInt32 *aVisitCount) -{ - mozStorageStatementScoper scope(mDBFullVisitCount); - - nsresult rv = mDBFullVisitCount->BindInt64ByName( - NS_LITERAL_CSTRING("page_id"), aPlaceId - ); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool hasVisits = PR_TRUE; - rv = mDBFullVisitCount->ExecuteStep(&hasVisits); - NS_ENSURE_SUCCESS(rv, rv); - - if (hasVisits) { - rv = mDBFullVisitCount->GetInt32(0, aVisitCount); - NS_ENSURE_SUCCESS(rv, rv); - } - else - *aVisitCount = 0; - - return NS_OK; -} - // Call this method before visiting a URL in order to help determine the // transition type of the visit. @@ -2921,9 +2864,7 @@ nsNavHistory::AddVisit(nsIURI* aURI, PRTime aTime, nsIURI* aReferringURI, // Update frecency (*after* the visit info is in the db) // Swallow errors here, since if we've gotten this far, it's more // important to notify the observers below. - nsNavBookmarks *bs = nsNavBookmarks::GetBookmarksService(); - NS_ENSURE_TRUE(bs, NS_ERROR_OUT_OF_MEMORY); - (void)UpdateFrecency(pageID, bs->IsRealBookmark(pageID)); + (void)UpdateFrecency(pageID); // Notify observers: The hidden detection code must match that in // GetQueryResults to maintain consistency. @@ -3321,47 +3262,13 @@ PlacesSQLQueryBuilder::SelectAsURI() "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " "h.last_visit_date, f.url, v.session, null, null, null, null, ") + tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places_temp h " - "JOIN moz_historyvisits_temp v ON h.id = v.place_id " + "FROM moz_places h " + "JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " // WHERE 1 is a no-op since additonal conditions will start with AND. "WHERE 1 " "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " "{ADDITIONAL_CONDITIONS} " - "GROUP BY h.id " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "h.last_visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places_temp h " - "JOIN moz_historyvisits v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT place_id FROM moz_historyvisits_temp) " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "GROUP BY h.id " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "h.last_visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places h " - "JOIN moz_historyvisits_temp v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "GROUP BY h.id " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "h.last_visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places h " - "JOIN moz_historyvisits v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " - "AND h.id NOT IN (SELECT place_id FROM moz_historyvisits_temp) " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " "GROUP BY h.id "); break; @@ -3379,23 +3286,6 @@ PlacesSQLQueryBuilder::SelectAsURI() tagsSqlFragment); mQueryString = NS_LITERAL_CSTRING( - "SELECT b2.fk, h.url, COALESCE(b2.title, h.title), h.rev_host, " - "h.visit_count, h.last_visit_date, f.url, null, b2.id, " - "b2.dateAdded, b2.lastModified, b2.parent, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_bookmarks b2 " - "JOIN (SELECT b.fk " - "FROM moz_bookmarks b " - // ADDITIONAL_CONDITIONS will filter on parent. - "WHERE b.type = 1 {ADDITIONAL_CONDITIONS} " - ") AS seed ON b2.fk = seed.fk " - "JOIN moz_places_temp h ON h.id = b2.fk " - "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE NOT EXISTS ( " - "SELECT id FROM moz_bookmarks WHERE id = b2.parent AND parent = ") + - nsPrintfCString("%lld", history->GetTagsFolder()) + - NS_LITERAL_CSTRING(") " - "UNION ALL " "SELECT b2.fk, h.url, COALESCE(b2.title, h.title), h.rev_host, " "h.visit_count, h.last_visit_date, f.url, null, b2.id, " "b2.dateAdded, b2.lastModified, b2.parent, ") + @@ -3412,7 +3302,6 @@ PlacesSQLQueryBuilder::SelectAsURI() "SELECT id FROM moz_bookmarks WHERE id = b2.parent AND parent = ") + nsPrintfCString("%lld", history->GetTagsFolder()) + NS_LITERAL_CSTRING(") " - "AND h.id NOT IN (SELECT id FROM moz_places_temp) " "ORDER BY b2.fk DESC, b2.lastModified DESC"); } else { @@ -3421,20 +3310,6 @@ PlacesSQLQueryBuilder::SelectAsURI() mHasSearchTerms, tagsSqlFragment); mQueryString = NS_LITERAL_CSTRING( - "SELECT b.fk, h.url, COALESCE(b.title, h.title), h.rev_host, " - "h.visit_count, h.last_visit_date, f.url, null, b.id, " - "b.dateAdded, b.lastModified, b.parent, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_bookmarks b " - "JOIN moz_places_temp h ON b.fk = h.id AND b.type = 1 " - "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE NOT EXISTS " - "(SELECT id FROM moz_bookmarks " - "WHERE id = b.parent AND parent = ") + - nsPrintfCString("%lld", history->GetTagsFolder()) + - NS_LITERAL_CSTRING(") " - "{ADDITIONAL_CONDITIONS}" - "UNION ALL " "SELECT b.fk, h.url, COALESCE(b.title, h.title), h.rev_host, " "h.visit_count, h.last_visit_date, f.url, null, b.id, " "b.dateAdded, b.lastModified, b.parent, ") + @@ -3442,8 +3317,7 @@ PlacesSQLQueryBuilder::SelectAsURI() "FROM moz_bookmarks b " "JOIN moz_places h ON b.fk = h.id AND b.type = 1 " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " - "AND NOT EXISTS " + "WHERE NOT EXISTS " "(SELECT id FROM moz_bookmarks " "WHERE id = b.parent AND parent = ") + nsPrintfCString("%lld", history->GetTagsFolder()) + @@ -3472,42 +3346,11 @@ PlacesSQLQueryBuilder::SelectAsVisit() "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " "v.visit_date, f.url, v.session, null, null, null, null, ") + tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places_temp h " - "JOIN moz_historyvisits_temp v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - // WHERE 1 is a no-op since additonal conditions will start with AND. - "WHERE 1 " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "v.visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places_temp h " + "FROM moz_places h " "JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " // WHERE 1 is a no-op since additonal conditions will start with AND. "WHERE 1 " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "v.visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places h " - "JOIN moz_historyvisits_temp v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION ALL " - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "v.visit_date, f.url, v.session, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places h " - "JOIN moz_historyvisits v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " "{ADDITIONAL_CONDITIONS} "); @@ -3682,12 +3525,6 @@ PlacesSQLQueryBuilder::SelectAsDay() "%s AS beginTime, " "%s AS endTime " "WHERE EXISTS ( " - "SELECT id FROM moz_historyvisits_temp " - "WHERE visit_date >= %s " - "AND visit_date < %s " - "AND visit_type NOT IN (0,%d,%d) " - "{QUERY_OPTIONS_VISITS} " - "UNION ALL " "SELECT id FROM moz_historyvisits " "WHERE visit_date >= %s " "AND visit_date < %s " @@ -3699,10 +3536,6 @@ PlacesSQLQueryBuilder::SelectAsDay() sqlFragmentContainerBeginTime.get(), sqlFragmentContainerEndTime.get(), sqlFragmentSearchBeginTime.get(), - sqlFragmentSearchEndTime.get(), - nsINavHistoryService::TRANSITION_EMBED, - nsINavHistoryService::TRANSITION_FRAMED_LINK, - sqlFragmentSearchBeginTime.get(), sqlFragmentSearchEndTime.get(), nsINavHistoryService::TRANSITION_EMBED, nsINavHistoryService::TRANSITION_FRAMED_LINK); @@ -3736,15 +3569,8 @@ PlacesSQLQueryBuilder::SelectAsSite() "'place:type=%ld&sort=%ld&domain=&domainIsHost=true', " ":localhost, :localhost, null, null, null, null, null, null, null " "WHERE EXISTS ( " - "SELECT id FROM moz_places_temp " - "WHERE hidden <> 1 " - "AND rev_host = '.' " - "AND visit_count > 0 " - "AND url BETWEEN 'file://' AND 'file:/~' " - "UNION ALL " "SELECT id FROM moz_places " - "WHERE id NOT IN (SELECT id FROM moz_places_temp) " - "AND hidden <> 1 " + "WHERE hidden <> 1 " "AND rev_host = '.' " "AND visit_count > 0 " "AND url BETWEEN 'file://' AND 'file:/~' " @@ -3756,14 +3582,8 @@ PlacesSQLQueryBuilder::SelectAsSite() "FROM ( " "SELECT get_unreversed_host(rev_host) host " "FROM ( " - "SELECT DISTINCT rev_host FROM moz_places_temp " - "WHERE hidden <> 1 " - "AND rev_host <> '.' " - "AND visit_count > 0 " - "UNION ALL " "SELECT DISTINCT rev_host FROM moz_places " - "WHERE id NOT IN (SELECT id FROM moz_places_temp) " - "AND hidden <> 1 " + "WHERE hidden <> 1 " "AND rev_host <> '.' " "AND visit_count > 0 " ") " @@ -3789,33 +3609,6 @@ PlacesSQLQueryBuilder::SelectAsSite() "AND h.url BETWEEN 'file://' AND 'file:/~' " "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT h.id " - "FROM moz_places_temp h " - "JOIN moz_historyvisits v ON v.place_id = h.id " - "WHERE h.hidden <> 1 AND h.rev_host = '.' " - "AND h.visit_count > 0 " - "AND h.url BETWEEN 'file://' AND 'file:/~' " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT h.id " - "FROM moz_places h " - "JOIN moz_historyvisits_temp v ON v.place_id = h.id " - "WHERE h.hidden <> 1 AND h.rev_host = '.' " - "AND h.visit_count > 0 " - "AND h.url BETWEEN 'file://' AND 'file:/~' " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT h.id " - "FROM moz_places_temp h " - "JOIN moz_historyvisits_temp v ON v.place_id = h.id " - "WHERE h.hidden <> 1 AND h.rev_host = '.' " - "AND h.visit_count > 0 " - "AND h.url BETWEEN 'file://' AND 'file:/~' " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " ") " "UNION ALL " "SELECT DISTINCT null, " @@ -3830,30 +3623,6 @@ PlacesSQLQueryBuilder::SelectAsSite() "AND h.visit_count > 0 " "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT DISTINCT get_unreversed_host(rev_host) AS host " - "FROM moz_places_temp h " - "JOIN moz_historyvisits v ON v.place_id = h.id " - "WHERE h.rev_host <> '.' " - "AND h.visit_count > 0 " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT DISTINCT get_unreversed_host(rev_host) AS host " - "FROM moz_places h " - "JOIN moz_historyvisits_temp v ON v.place_id = h.id " - "WHERE h.rev_host <> '.' " - "AND h.visit_count > 0 " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " - "UNION " - "SELECT DISTINCT get_unreversed_host(rev_host) AS host " - "FROM moz_places_temp h " - "JOIN moz_historyvisits_temp v ON v.place_id = h.id " - "WHERE h.rev_host <> '.' " - "AND h.visit_count > 0 " - "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " - "{ADDITIONAL_CONDITIONS} " "ORDER BY 1 ASC " ") ", nsINavHistoryQueryOptions::RESULTS_AS_URI, @@ -3905,11 +3674,6 @@ PlacesSQLQueryBuilder::Where() else if (mRedirectsMode == nsINavHistoryQueryOptions::REDIRECTS_MODE_TARGET) { additionalVisitsConditions += NS_LITERAL_CSTRING( "AND NOT EXISTS ( " - "SELECT id FROM moz_historyvisits_temp WHERE from_visit = v.id " - "AND visit_type IN ") + - nsPrintfCString("(%d,%d) ", nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT, - nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY) + - NS_LITERAL_CSTRING(") AND NOT EXISTS ( " "SELECT id FROM moz_historyvisits WHERE from_visit = v.id " "AND visit_type IN ") + nsPrintfCString("(%d,%d) ", nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT, @@ -3976,9 +3740,7 @@ PlacesSQLQueryBuilder::OrderBy() switch(mSortingMode) { case nsINavHistoryQueryOptions::SORT_BY_NONE: - // If this is a URI query the sorting could change based on the - // sync status of disk and temp tables, we must ensure sorting does not - // change between queries. + // Ensure sorting does not change based on tables status. if (mResultType == nsINavHistoryQueryOptions::RESULTS_AS_URI) { if (mQueryType == nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS) mQueryString += NS_LITERAL_CSTRING(" ORDER BY b.id ASC "); @@ -4118,40 +3880,13 @@ nsNavHistory::ConstructQueryString( // Generate an optimized query for the history menu and most visited // smart bookmark. queryString = NS_LITERAL_CSTRING( - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, h.last_visit_date, " - "f.url, null, null, null, null, null, ") + - tagsSqlFragment + NS_LITERAL_CSTRING( - "FROM moz_places_temp h " - "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE h.hidden <> 1 " - "AND EXISTS (SELECT id FROM moz_historyvisits_temp WHERE place_id = h.id " - "AND visit_type NOT IN ") + - nsPrintfCString("(0,%d,%d) ", - nsINavHistoryService::TRANSITION_EMBED, - nsINavHistoryService::TRANSITION_FRAMED_LINK) + - NS_LITERAL_CSTRING("UNION ALL " - "SELECT id FROM moz_historyvisits WHERE place_id = h.id " - "AND visit_type NOT IN ") + - nsPrintfCString("(0,%d,%d) ", - nsINavHistoryService::TRANSITION_EMBED, - nsINavHistoryService::TRANSITION_FRAMED_LINK) + - NS_LITERAL_CSTRING("LIMIT 1) " - "{QUERY_OPTIONS} " - "UNION ALL " "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, h.last_visit_date, " "f.url, null, null, null, null, null, ") + tagsSqlFragment + NS_LITERAL_CSTRING( "FROM moz_places h " "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " "WHERE h.hidden <> 1 " - "AND h.id NOT IN (SELECT id FROM moz_places_temp) " - "AND EXISTS (SELECT id FROM moz_historyvisits_temp WHERE place_id = h.id " - "AND visit_type NOT IN ") + - nsPrintfCString("(0,%d,%d) ", - nsINavHistoryService::TRANSITION_EMBED, - nsINavHistoryService::TRANSITION_FRAMED_LINK) + - NS_LITERAL_CSTRING("UNION ALL " - "SELECT id FROM moz_historyvisits WHERE place_id = h.id " + "AND EXISTS (SELECT id FROM moz_historyvisits WHERE place_id = h.id " "AND visit_type NOT IN ") + nsPrintfCString("(0,%d,%d) ", nsINavHistoryService::TRANSITION_EMBED, @@ -4173,48 +3908,24 @@ nsNavHistory::ConstructQueryString( if (aOptions->RedirectsMode() == nsINavHistoryQueryOptions::REDIRECTS_MODE_SOURCE) { additionalQueryOptions += nsPrintfCString(256, - "AND NOT EXISTS ( " - "SELECT id FROM moz_historyvisits_temp WHERE place_id = h.id " - "AND visit_type IN (%d,%d)" - ") " "AND NOT EXISTS ( " "SELECT id FROM moz_historyvisits WHERE place_id = h.id " "AND visit_type IN (%d,%d)" ") ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY, - TRANSITION_REDIRECT_PERMANENT, TRANSITION_REDIRECT_TEMPORARY); } else if (aOptions->RedirectsMode() == nsINavHistoryQueryOptions::REDIRECTS_MODE_TARGET) { additionalQueryOptions += nsPrintfCString(1024, "AND NOT EXISTS ( " - "SELECT id " - "FROM moz_historyvisits_temp v " - "WHERE place_id = h.id " - "AND EXISTS(SELECT id FROM moz_historyvisits_temp " - "WHERE from_visit = v.id AND visit_type IN (%d,%d) " - "UNION ALL " - "SELECT id FROM moz_historyvisits " - "WHERE from_visit = v.id AND visit_type IN (%d,%d)) " - "UNION ALL " "SELECT id " "FROM moz_historyvisits v " "WHERE place_id = h.id " - "AND EXISTS(SELECT id FROM moz_historyvisits_temp " - "WHERE from_visit = v.id AND visit_type IN (%d,%d) " - "UNION ALL " - "SELECT id FROM moz_historyvisits " + "AND EXISTS(SELECT id FROM moz_historyvisits " "WHERE from_visit = v.id AND visit_type IN (%d,%d)) " ") ", TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY, - TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY, - TRANSITION_REDIRECT_PERMANENT, - TRANSITION_REDIRECT_TEMPORARY, - TRANSITION_REDIRECT_PERMANENT, TRANSITION_REDIRECT_TEMPORARY); } queryString.ReplaceSubstring("{QUERY_OPTIONS}", @@ -4459,27 +4170,11 @@ nsNavHistory::GetLastPageVisited(nsACString & aLastPageVisited) NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); nsCOMPtr statement; - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // expect newest visits being in temp table. nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT url, visit_date FROM moz_historyvisits_temp v " - "JOIN moz_places_temp h ON v.place_id = h.id " - "WHERE h.hidden <> 1 " - "UNION ALL " - "SELECT url, visit_date FROM moz_historyvisits_temp v " - "JOIN moz_places h ON v.place_id = h.id " - "WHERE h.hidden <> 1 " - "UNION ALL " - "SELECT url, visit_date FROM moz_historyvisits v " - "JOIN moz_places_temp h ON v.place_id = h.id " - "WHERE h.hidden <> 1 " - "UNION ALL " "SELECT url, visit_date FROM moz_historyvisits v " "JOIN moz_places h ON v.place_id = h.id " "WHERE h.hidden <> 1 " - "ORDER BY visit_date DESC " - "LIMIT 1 "), + "ORDER BY visit_date DESC "), getter_AddRefs(statement)); NS_ENSURE_SUCCESS(rv, rv); @@ -4537,7 +4232,7 @@ nsNavHistory::RemovePagesInternal(const nsCString& aPlaceIdsQueryString) // Delete all visits for the specified place ids. rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "DELETE FROM moz_historyvisits_view WHERE place_id IN (") + + "DELETE FROM moz_historyvisits WHERE place_id IN (") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(")")); NS_ENSURE_SUCCESS(rv, rv); @@ -4576,21 +4271,12 @@ nsNavHistory::PreparePlacesForVisitsDelete(const nsCString& aPlaceIdsQueryString // to figure out which places to recalculate frecency first. // Pay attention to not set frecency = 0 if visit_count = 0 nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "UPDATE moz_places_view " + "UPDATE moz_places " "SET frecency = -MAX(visit_count, 1) " "WHERE id IN ( " - "SELECT h.id " - "FROM moz_places_temp h " - "WHERE h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") " - "AND ( " - "EXISTS (SELECT b.id FROM moz_bookmarks b WHERE b.fk =h.id) " - "OR EXISTS (SELECT a.id FROM moz_annos a WHERE a.place_id = h.id) " - ") " - "UNION ALL " "SELECT h.id " "FROM moz_places h " "WHERE h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") " - "AND h.id NOT IN (SELECT id FROM moz_places_temp) " "AND ( " "EXISTS (SELECT b.id FROM moz_bookmarks b WHERE b.fk =h.id) " "OR EXISTS (SELECT a.id FROM moz_annos a WHERE a.place_id = h.id) " @@ -4625,16 +4311,9 @@ nsNavHistory::CleanupPlacesOnVisitsDelete(const nsCString& aPlaceIdsQueryString) // Note that we do NOT delete favicons. Any unreferenced favicons will be // deleted next time the browser is shut down. nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "DELETE FROM moz_places_view WHERE id IN (" - "SELECT h.id FROM moz_places_temp h " - "WHERE h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") " - "AND SUBSTR(h.url, 1, 6) <> 'place:' " - "AND NOT EXISTS " - "(SELECT b.id FROM moz_bookmarks b WHERE b.fk = h.id LIMIT 1) " - "UNION ALL " + "DELETE FROM moz_places WHERE id IN ( " "SELECT h.id FROM moz_places h " - "WHERE h.id NOT IN (SELECT id FROM moz_places_temp) " - "AND h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") " + "WHERE h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") " "AND SUBSTR(h.url, 1, 6) <> 'place:' " "AND NOT EXISTS " "(SELECT b.id FROM moz_bookmarks b WHERE b.fk = h.id LIMIT 1) " @@ -4773,12 +4452,8 @@ nsNavHistory::RemovePagesFromHost(const nsACString& aHost, PRBool aEntireDomain) // create statement depending on delete type rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id FROM moz_places_temp " - "WHERE ") + conditionString + NS_LITERAL_CSTRING( - "UNION ALL " "SELECT id FROM moz_places " - "WHERE id NOT IN (SELECT id FROM moz_places_temp) " - "AND ") + conditionString, + "WHERE ") + conditionString, getter_AddRefs(statement)); NS_ENSURE_SUCCESS(rv, rv); rv = statement->BindStringByIndex(0, revHostDot); @@ -4830,20 +4505,9 @@ nsNavHistory::RemovePagesByTimeframe(PRTime aBeginTime, PRTime aEndTime) // this query is faster than actually selecting in moz_historyvisits nsCOMPtr selectByTime; rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT h.id FROM moz_places_temp h WHERE " - "EXISTS " - "(SELECT id FROM moz_historyvisits v WHERE v.place_id = h.id " - "AND v.visit_date >= :from_date AND v.visit_date <= :to_date LIMIT 1)" - "OR EXISTS " - "(SELECT id FROM moz_historyvisits_temp v WHERE v.place_id = h.id " - "AND v.visit_date >= :from_date AND v.visit_date <= :to_date LIMIT 1) " - "UNION " "SELECT h.id FROM moz_places h WHERE " "EXISTS " "(SELECT id FROM moz_historyvisits v WHERE v.place_id = h.id " - "AND v.visit_date >= :from_date AND v.visit_date <= :to_date LIMIT 1)" - "OR EXISTS " - "(SELECT id FROM moz_historyvisits_temp v WHERE v.place_id = h.id " "AND v.visit_date >= :from_date AND v.visit_date <= :to_date LIMIT 1)"), getter_AddRefs(selectByTime)); NS_ENSURE_SUCCESS(rv, rv); @@ -4903,19 +4567,11 @@ nsNavHistory::RemoveVisitsByTimeframe(PRTime aBeginTime, PRTime aEndTime) nsCOMPtr selectByTime; mozStorageStatementScoper scope(selectByTime); rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT place_id " - "FROM moz_historyvisits_temp " - "WHERE :from_date <= visit_date AND visit_date <= :to_date " - "UNION " "SELECT place_id " "FROM moz_historyvisits " "WHERE :from_date <= visit_date AND visit_date <= :to_date " "EXCEPT " "SELECT place_id " - "FROM moz_historyvisits_temp " - "WHERE visit_date < :from_date OR :to_date < visit_date " - "EXCEPT " - "SELECT place_id " "FROM moz_historyvisits " "WHERE visit_date < :from_date OR :to_date < visit_date"), getter_AddRefs(selectByTime)); @@ -4950,7 +4606,7 @@ nsNavHistory::RemoveVisitsByTimeframe(PRTime aBeginTime, PRTime aEndTime) // Delete all visits within the timeframe. nsCOMPtr deleteVisitsStmt; rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "DELETE FROM moz_historyvisits_view " + "DELETE FROM moz_historyvisits " "WHERE :from_date <= visit_date AND visit_date <= :to_date"), getter_AddRefs(deleteVisitsStmt)); NS_ENSURE_SUCCESS(rv, rv); @@ -4990,19 +4646,13 @@ nsNavHistory::RemoveAllPages() // idle query to figure out which places to recalcuate frecency first. // We must do this before deleting visits. nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "UPDATE moz_places_view SET frecency = -MAX(visit_count, 1) " - "WHERE id IN(" - "SELECT h.id FROM moz_places_temp h " - "WHERE EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) " - "UNION ALL " - "SELECT h.id FROM moz_places h " - "WHERE EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) " - ")")); + "UPDATE moz_places SET frecency = -MAX(visit_count, 1) " + "WHERE id IN(SELECT b.fk FROM moz_bookmarks b WHERE b.fk NOTNULL)")); NS_ENSURE_SUCCESS(rv, rv); // Expire visits, then let the paranoid functions do the cleanup for us. rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "DELETE FROM moz_historyvisits_view")); + "DELETE FROM moz_historyvisits")); NS_ENSURE_SUCCESS(rv, rv); // Some of the remaining places could be place: urls or @@ -5995,39 +5645,25 @@ nsNavHistory::VacuumDatabase() nsnull); } - // Actually vacuuming a database is a slow operation, since it could take - // seconds. Part of the time is spent in updating the journal file on disk - // and this is particularly bad on devices with slow I/O. Temporary - // moving the journal to memory could increase a bit the possibility of - // corruption if we crash during this time, but makes the process really - // faster. - nsCOMPtr journalToMemory; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = MEMORY"), - getter_AddRefs(journalToMemory)); - NS_ENSURE_SUCCESS(rv, rv); + // If journal mode is WAL, a VACUUM cannot upgrade page_size value. + // If current page_size is not the expected one, journal mode must be + // changed to a rollback one. Once done we won't be able to go back to WAL + // mode though, since non-reset statements exist. Just keep using + // compatible mode till next restart. + // See http://www.sqlite.org/wal.html + if (mCurrentJournalMode == JOURNAL_WAL && + mDBPageSize != SQLITE_DEFAULT_PAGE_SIZE) { + (void)SetJournalMode(JOURNAL_TRUNCATE); + } - nsCOMPtr vacuum; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("VACUUM"), - getter_AddRefs(vacuum)); + nsCOMPtr vacuum; + rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING("VACUUM"), + getter_AddRefs(vacuum)); NS_ENSURE_SUCCESS(rv, rv); - - nsCOMPtr journalToDefault; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE), - getter_AddRefs(journalToDefault)); - NS_ENSURE_SUCCESS(rv, rv); - - mozIStorageBaseStatement *stmts[] = { - journalToMemory, - vacuum, - journalToDefault - }; nsCOMPtr ps; nsRefPtr vacuumDBListener = new VacuumDBListener(mPrefBranch); - rv = mDBConn->ExecuteAsync(stmts, NS_ARRAY_LENGTH(stmts), - vacuumDBListener, getter_AddRefs(ps)); + rv = vacuum->ExecuteAsync(vacuumDBListener, getter_AddRefs(ps)); NS_ENSURE_SUCCESS(rv, rv); } @@ -6038,17 +5674,15 @@ nsNavHistory::VacuumDatabase() nsresult nsNavHistory::DecayFrecency() { - // Update frecency values. nsresult rv = FixInvalidFrecencies(); NS_ENSURE_SUCCESS(rv, rv); // Globally decay places frecency rankings to estimate reduced frecency // values of pages that haven't been visited for a while, i.e., they do - // not get an updated frecency. We directly modify moz_places to avoid - // bringing the whole database into places_temp through places_view. A - // scaling factor of .975 results in .5 the original value after 28 days. - nsCOMPtr decayFrecency; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + // not get an updated frecency. A scaling factor of .975 results in .5 the + // original value after 28 days. + nsCOMPtr decayFrecency; + rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING( "UPDATE moz_places SET frecency = ROUND(frecency * .975) " "WHERE frecency > 0"), getter_AddRefs(decayFrecency)); @@ -6056,15 +5690,15 @@ nsNavHistory::DecayFrecency() // Decay potentially unused adaptive entries (e.g. those that are at 1) // to allow better chances for new entries that will start at 1. - nsCOMPtr decayAdaptive; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + nsCOMPtr decayAdaptive; + rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING( "UPDATE moz_inputhistory SET use_count = use_count * .975"), getter_AddRefs(decayAdaptive)); NS_ENSURE_SUCCESS(rv, rv); // Delete any adaptive entries that won't help in ordering anymore. - nsCOMPtr deleteAdaptive; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + nsCOMPtr deleteAdaptive; + rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING( "DELETE FROM moz_inputhistory WHERE use_count < .01"), getter_AddRefs(deleteAdaptive)); NS_ENSURE_SUCCESS(rv, rv); @@ -6257,9 +5891,6 @@ nsNavHistory::QueryToSelectClause(nsNavHistoryQuery* aQuery, // const nsPrintfCString param(":transition%d_", i); clause.Str("EXISTS (SELECT 1 FROM moz_historyvisits " "WHERE place_id = h.id AND visit_type = " - ).Param(param.get()).Str(" UNION ALL " - "SELECT 1 FROM moz_historyvisits_temp " - "WHERE place_id = h.id AND visit_type = " ).Param(param.get()).Str(" LIMIT 1)"); if (i < transitions.Length() - 1) clause.Str("AND"); @@ -7550,286 +7181,54 @@ void ParseSearchTermsFromQueries(const nsCOMArray& aQueries, } // anonymous namespace +// if we calculated a non-zero frecency we should unhide this place +// so that previously hidden (non-livebookmark item) bookmarks +// will now appear in autocomplete +// if we calculated a zero frecency, we re-use the old hidden value. nsresult -nsNavHistory::UpdateFrecency(PRInt64 aPlaceId, PRBool aIsBookmarked) +nsNavHistory::UpdateFrecency(PRInt64 aPlaceId) { - mozStorageStatementScoper statsScoper(mDBGetPlaceVisitStats); - nsresult rv = mDBGetPlaceVisitStats->BindInt64ByName( - NS_LITERAL_CSTRING("page_id"), aPlaceId - ); - NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_STATE(mDBUpdateFrecency && mDBUpdateHiddenOnFrecency); - PRBool hasResults = PR_FALSE; - rv = mDBGetPlaceVisitStats->ExecuteStep(&hasResults); - NS_ENSURE_SUCCESS(rv, rv); +#define ASYNC_BIND(_stmt) \ + PR_BEGIN_MACRO \ + nsCOMPtr paramsArray; \ + nsresult rv = _stmt->NewBindingParamsArray(getter_AddRefs(paramsArray)); \ + NS_ENSURE_SUCCESS(rv, rv); \ + nsCOMPtr params; \ + rv = paramsArray->NewBindingParams(getter_AddRefs(params)); \ + NS_ENSURE_SUCCESS(rv, rv); \ + rv = params->BindInt64ByName(NS_LITERAL_CSTRING("page_id"), aPlaceId); \ + NS_ENSURE_SUCCESS(rv, rv); \ + rv = paramsArray->AddParams(params); \ + NS_ENSURE_SUCCESS(rv, rv); \ + rv = _stmt->BindParameters(paramsArray); \ + NS_ENSURE_SUCCESS(rv, rv); \ + PR_END_MACRO - if (!hasResults) { - NS_WARNING("attempting to update frecency for a bogus place"); - // before I added the check for itemType == TYPE_BOOKMARK - // I hit this with aPlaceId of 0 (on import) - return NS_OK; - } + ASYNC_BIND(mDBUpdateFrecency); + ASYNC_BIND(mDBUpdateHiddenOnFrecency); - PRInt32 typed = 0; - rv = mDBGetPlaceVisitStats->GetInt32(0, &typed); - NS_ENSURE_SUCCESS(rv, rv); + mozIStorageBaseStatement *stmts[] = { + mDBUpdateFrecency + , mDBUpdateHiddenOnFrecency + }; - PRInt32 hidden = 0; - rv = mDBGetPlaceVisitStats->GetInt32(1, &hidden); - NS_ENSURE_SUCCESS(rv, rv); - - PRInt32 oldFrecency = 0; - rv = mDBGetPlaceVisitStats->GetInt32(2, &oldFrecency); - NS_ENSURE_SUCCESS(rv, rv); - - rv = UpdateFrecencyInternal(aPlaceId, typed, hidden, oldFrecency, - aIsBookmarked); + nsCOMPtr callback = + new AsyncStatementCallbackNotifier(TOPIC_FRECENCY_UPDATED); + nsCOMPtr ps; + nsresult rv = mDBConn->ExecuteAsync(stmts, NS_ARRAY_LENGTH(stmts), callback, + getter_AddRefs(ps)); NS_ENSURE_SUCCESS(rv, rv); return NS_OK; +#undef ASYNC_BIND } -nsresult -nsNavHistory::UpdateFrecencyInternal(PRInt64 aPlaceId, PRInt32 aTyped, - PRInt32 aHidden, PRInt32 aOldFrecency, PRBool aIsBookmarked) -{ - PRInt32 visitCountForFrecency = 0; - // Since visit_count excludes visit with visit_type NOT IN(0,4,7,8), it can't - // be uses for calculating frecency. Instead it must must be recalculated. - nsresult rv = CalculateFullVisitCount(aPlaceId, &visitCountForFrecency); - NS_ENSURE_SUCCESS(rv, rv); - - PRInt32 newFrecency = 0; - rv = CalculateFrecencyInternal(aPlaceId, aTyped, visitCountForFrecency, - aIsBookmarked, &newFrecency); - NS_ENSURE_SUCCESS(rv, rv); - - // save ourselves the UPDATE if the frecency hasn't changed - // One way this can happen is with livemarks. - // when we added the livemark, the frecency was 0. - // On refresh, when we remove and then add the livemark items, - // the frecency (for a given moz_places) will not have changed - // (if we've never visited that place). - // Additionally, don't bother overwriting a valid frecency with an invalid one - if ((newFrecency == aOldFrecency) || (aOldFrecency && newFrecency < 0)) - return NS_OK; - - mozStorageStatementScoper updateScoper(mDBUpdateFrecencyAndHidden); - rv = mDBUpdateFrecencyAndHidden->BindInt64ByName( - NS_LITERAL_CSTRING("page_id"), aPlaceId - ); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBUpdateFrecencyAndHidden->BindInt32ByName( - NS_LITERAL_CSTRING("frecency"), newFrecency - ); - NS_ENSURE_SUCCESS(rv, rv); - - // if we calculated a non-zero frecency we should unhide this place - // so that previously hidden (non-livebookmark item) bookmarks - // will now appear in autocomplete - // if we calculated a zero frecency, we re-use the old hidden value. - rv = mDBUpdateFrecencyAndHidden->BindInt32ByName( - NS_LITERAL_CSTRING("hidden"), newFrecency ? 0 /* not hidden */ : aHidden - ); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mDBUpdateFrecencyAndHidden->Execute(); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; -} - -nsresult -nsNavHistory::CalculateFrecencyInternal(PRInt64 aPlaceId, - PRInt32 aTyped, - PRInt32 aVisitCount, - PRBool aIsBookmarked, - PRInt32 *aFrecency) -{ - PRTime normalizedNow = NormalizeTimeRelativeToday(GetNow()); - - float pointsForSampledVisits = 0.0; - - if (aPlaceId != -1) { - PRInt32 numSampledVisits = 0; - - mozStorageStatementScoper scoper(mDBVisitsForFrecency); - nsresult rv = mDBVisitsForFrecency->BindInt64ByName( - NS_LITERAL_CSTRING("page_id"), aPlaceId - ); - NS_ENSURE_SUCCESS(rv, rv); - - // mDBVisitsForFrecency is limited by the browser.frecency.numVisits pref - PRBool hasMore = PR_FALSE; - while (NS_SUCCEEDED(mDBVisitsForFrecency->ExecuteStep(&hasMore)) - && hasMore) { - numSampledVisits++; - - PRInt32 visitType = mDBVisitsForFrecency->AsInt32(1); - - PRInt32 bonus = 0; - - switch (visitType) { - case nsINavHistoryService::TRANSITION_EMBED: - bonus = mEmbedVisitBonus; - break; - case nsINavHistoryService::TRANSITION_FRAMED_LINK: - bonus = mFramedLinkVisitBonus; - break; - case nsINavHistoryService::TRANSITION_LINK: - bonus = mLinkVisitBonus; - break; - case nsINavHistoryService::TRANSITION_TYPED: - bonus = mTypedVisitBonus; - break; - case nsINavHistoryService::TRANSITION_BOOKMARK: - bonus = mBookmarkVisitBonus; - break; - case nsINavHistoryService::TRANSITION_DOWNLOAD: - bonus = mDownloadVisitBonus; - break; - case nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT: - bonus = mPermRedirectVisitBonus; - break; - case nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY: - bonus = mTempRedirectVisitBonus; - break; - default: - // 0 == undefined (see bug #375777 for details) - NS_WARN_IF_FALSE(!visitType, "new transition but no weight for frecency"); - bonus = mDefaultVisitBonus; - break; - } - - // Always add the bookmark visit bonus. - if (aIsBookmarked) - bonus += mBookmarkVisitBonus; - -#ifdef DEBUG_FRECENCY - printf("CalculateFrecency() for place %lld has a bonus of %d\n", aPlaceId, bonus); -#endif - - // if bonus was zero, we can skip the work to determine the weight - if (bonus) { - PRTime visitDate = mDBVisitsForFrecency->AsInt64(0); - PRInt64 ageInDays = GetAgeInDays(normalizedNow, visitDate); - - PRInt32 weight = 0; - - if (ageInDays <= mFirstBucketCutoffInDays) - weight = mFirstBucketWeight; - else if (ageInDays <= mSecondBucketCutoffInDays) - weight = mSecondBucketWeight; - else if (ageInDays <= mThirdBucketCutoffInDays) - weight = mThirdBucketWeight; - else if (ageInDays <= mFourthBucketCutoffInDays) - weight = mFourthBucketWeight; - else - weight = mDefaultWeight; - - pointsForSampledVisits += (float)(weight * (bonus / 100.0)); - } - } - - if (numSampledVisits) { - // fix for bug #412219 - if (!pointsForSampledVisits) { - // For URIs with zero points in the sampled recent visits - // but "browsing" type visits outside the sampling range, set - // frecency to -visit_count, so they're still shown in autocomplete. - PRInt32 visitCount = 0; - mozStorageStatementScoper scoper(mDBGetIdPageInfo); - rv = mDBGetIdPageInfo->BindInt64ByName(NS_LITERAL_CSTRING("page_id"), aPlaceId); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool hasVisits = PR_TRUE; - if (NS_SUCCEEDED(mDBGetIdPageInfo->ExecuteStep(&hasVisits)) && hasVisits) { - rv = mDBGetIdPageInfo->GetInt32(nsNavHistory::kGetInfoIndex_VisitCount, - &visitCount); - NS_ENSURE_SUCCESS(rv, rv); - } - // If we don't have visits set to 0 - *aFrecency = -visitCount; - } - else { - // Estimate frecency using the last few visits. - // Use NS_ceilf() so that we don't round down to 0, which - // would cause us to completely ignore the place during autocomplete. - *aFrecency = (PRInt32) NS_ceilf(aVisitCount * NS_ceilf(pointsForSampledVisits) / numSampledVisits); - } - -#ifdef DEBUG_FRECENCY - printf("CalculateFrecency() for place %lld: %d = %d * %f / %d\n", aPlaceId, *aFrecency, aVisitCount, pointsForSampledVisits, numSampledVisits); -#endif - - return NS_OK; - } - } - - // XXX the code below works well for guessing the frecency on import, and we'll correct later once we have - // visits. - // what if we don't have visits and we never visit? we could end up with a really high value - // that keeps coming up in ac results? only do this on import? something to figure out. - PRInt32 bonus = 0; - - // not the same logic above, as a single visit could not be both - // a bookmark visit and a typed visit. but when estimating a frecency - // for a place that doesn't have any visits, this will make it so - // something bookmarked and typed will have a higher frecency than - // something just typed or just bookmarked. - if (aIsBookmarked) - bonus += mUnvisitedBookmarkBonus; - if (aTyped) - bonus += mUnvisitedTypedBonus; - - // assume "now" as our ageInDays, so use the first bucket. - pointsForSampledVisits = mFirstBucketWeight * (bonus / (float)100.0); - - // for a unvisited bookmark, produce a non-zero frecency - // so that unvisited bookmarks show up in URL bar autocomplete - if (!aVisitCount && aIsBookmarked) - aVisitCount = 1; - - // use NS_ceilf() so that we don't round down to 0, which - // would cause us to completely ignore the place during autocomplete - *aFrecency = (PRInt32) NS_ceilf(aVisitCount * NS_ceilf(pointsForSampledVisits)); -#ifdef DEBUG_FRECENCY - printf("CalculateFrecency() for unvisited: frecency %d = %f points (b: %d, t: %d) * visit count %d\n", *aFrecency, pointsForSampledVisits, aIsBookmarked, aTyped, aVisitCount); -#endif - return NS_OK; -} - -nsresult -nsNavHistory::CalculateFrecency(PRInt64 aPlaceId, - PRInt32 aTyped, - PRInt32 aVisitCount, - nsCAutoString &aURL, - PRInt32 *aFrecency) -{ - *aFrecency = 0; - - PRBool isBookmark = PR_FALSE; - - // determine if the place is a (non-livemark item) bookmark and prevent - // place: queries from showing up in the URL bar autocomplete results - if (!IsQueryURI(aURL) && aPlaceId != -1) { - nsNavBookmarks *bs = nsNavBookmarks::GetBookmarksService(); - NS_ENSURE_TRUE(bs, NS_ERROR_OUT_OF_MEMORY); - isBookmark = bs->IsRealBookmark(aPlaceId); - } - - nsresult rv = CalculateFrecencyInternal(aPlaceId, aTyped, aVisitCount, - isBookmark, aFrecency); - NS_ENSURE_SUCCESS(rv, rv); - return NS_OK; -} nsresult nsNavHistory::FixInvalidFrecencies() { - mozStorageTransaction transaction(mDBConn, PR_TRUE); - // Find all places with invalid frecencies (frecency < 0) that occur when: // 1) we've done "clear private data" // 2) we've expired or deleted visits @@ -7843,33 +7242,19 @@ nsNavHistory::FixInvalidFrecencies() // Note, we are not limiting ourselves to places with visits because we may // not have any if the place is a bookmark and we expired or deleted all the // visits. - nsCOMPtr invalidFrecencies; + nsCOMPtr fixInvalidFrecenciesStmt; nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, typed, hidden, frecency, url " - "FROM moz_places_view " + "UPDATE moz_places " + "SET frecency = CALCULATE_FRECENCY(id) " "WHERE frecency < 0"), - getter_AddRefs(invalidFrecencies)); + getter_AddRefs(fixInvalidFrecenciesStmt)); NS_ENSURE_SUCCESS(rv, rv); - PRBool hasMore = PR_FALSE; - while (NS_SUCCEEDED(invalidFrecencies->ExecuteStep(&hasMore)) && hasMore) { - PRInt64 placeId = invalidFrecencies->AsInt64(0); - PRInt32 typed = invalidFrecencies->AsInt32(1); - PRInt32 hidden = invalidFrecencies->AsInt32(2); - PRInt32 oldFrecency = invalidFrecencies->AsInt32(3); - nsCAutoString url; - invalidFrecencies->GetUTF8String(4, url); - - PRBool isBook = PR_FALSE; - if (!IsQueryURI(url)) { - nsNavBookmarks *bookmarks = nsNavBookmarks::GetBookmarksService(); - NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); - isBook = bookmarks->IsRealBookmark(placeId); - } - - rv = UpdateFrecencyInternal(placeId, typed, hidden, oldFrecency, isBook); - NS_ENSURE_SUCCESS(rv, rv); - } + nsCOMPtr callback = + new AsyncStatementCallbackNotifier(TOPIC_FRECENCY_UPDATED); + nsCOMPtr ps; + rv = fixInvalidFrecenciesStmt->ExecuteAsync(callback, getter_AddRefs(ps)); + NS_ENSURE_SUCCESS(rv, rv); return NS_OK; } @@ -7877,45 +7262,6 @@ nsNavHistory::FixInvalidFrecencies() #ifdef MOZ_XUL -namespace { - -// Used to notify a topic to system observers on async execute completion. -class AutoCompleteStatementCallbackNotifier : public AsyncStatementCallback -{ -public: - NS_DECL_ISUPPORTS - NS_DECL_ASYNCSTATEMENTCALLBACK -}; - -NS_IMPL_ISUPPORTS1(AutoCompleteStatementCallbackNotifier, - mozIStorageStatementCallback) - -NS_IMETHODIMP -AutoCompleteStatementCallbackNotifier::HandleCompletion(PRUint16 aReason) -{ - if (aReason != mozIStorageStatementCallback::REASON_FINISHED) - return NS_ERROR_UNEXPECTED; - - nsCOMPtr observerService = - do_GetService(NS_OBSERVERSERVICE_CONTRACTID); - if (observerService) { - (void)observerService->NotifyObservers(nsnull, - TOPIC_AUTOCOMPLETE_FEEDBACK_UPDATED, - nsnull); - } - - return NS_OK; -} - -NS_IMETHODIMP -AutoCompleteStatementCallbackNotifier::HandleResult(mozIStorageResultSet *aResultSet) -{ - NS_ASSERTION(PR_FALSE, "You cannot use AutoCompleteStatementCallbackNotifier to get async statements resultset"); - return NS_OK; -} - -} // anonymous namespace - nsresult nsNavHistory::AutoCompleteFeedback(PRInt32 aIndex, nsIAutoCompleteController *aController) @@ -7942,8 +7288,8 @@ nsNavHistory::AutoCompleteFeedback(PRInt32 aIndex, NS_ENSURE_SUCCESS(rv, rv); // We do the update asynchronously and we do not care about failures. - nsCOMPtr callback = - new AutoCompleteStatementCallbackNotifier(); + nsCOMPtr callback = + new AsyncStatementCallbackNotifier(TOPIC_AUTOCOMPLETE_FEEDBACK_UPDATED); nsCOMPtr canceler; rv = stmt->ExecuteAsync(callback, getter_AddRefs(canceler)); NS_ENSURE_SUCCESS(rv, rv); @@ -7962,15 +7308,9 @@ nsNavHistory::GetDBFeedbackIncrease() "INSERT OR REPLACE INTO moz_inputhistory " // use_count will asymptotically approach the max of 10. "SELECT h.id, IFNULL(i.input, :input_text), IFNULL(i.use_count, 0) * .9 + 1 " - "FROM moz_places_temp h " - "LEFT JOIN moz_inputhistory i ON i.place_id = h.id AND i.input = :input_text " - "WHERE url = :page_url " - "UNION ALL " - "SELECT h.id, IFNULL(i.input, :input_text), IFNULL(i.use_count, 0) * .9 + 1 " "FROM moz_places h " "LEFT JOIN moz_inputhistory i ON i.place_id = h.id AND i.input = :input_text " - "WHERE url = :page_url " - "AND h.id NOT IN (SELECT id FROM moz_places_temp)"), + "WHERE url = :page_url "), getter_AddRefs(mDBFeedbackIncrease)); NS_ENSURE_SUCCESS(rv, nsnull); @@ -8038,27 +7378,14 @@ nsNavHistory::GetDBVisitToVisitResult() if (mDBVisitToVisitResult) return mDBVisitToVisitResult; - // mDBVisitToVisitResult, should match kGetInfoIndex_* (see GetQueryResults) - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique visit ids. + // Should match kGetInfoIndex_* (see GetQueryResults) nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "v.visit_date, f.url, v.session, null, null, null, null " - "FROM moz_places_temp h " - "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE v.id = :visit_id OR v_t.id = :visit_id " - "UNION ALL " "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " "v.visit_date, f.url, v.session, null, null, null, null " "FROM moz_places h " - "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " + "JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE v.id = :visit_id OR v_t.id = :visit_id " - "LIMIT 1"), + "WHERE v.id = :visit_id "), getter_AddRefs(mDBVisitToVisitResult)); NS_ENSURE_SUCCESS(rv, nsnull); @@ -8071,27 +7398,14 @@ nsNavHistory::GetDBVisitToURLResult() if (mDBVisitToURLResult) return mDBVisitToURLResult; - // mDBVisitToURLResult, should match kGetInfoIndex_* (see GetQueryResults) - // We are not checking for duplicated ids into the unified table - // for perf reasons, LIMIT 1 will discard duplicates faster since we - // have unique visit ids. + // Should match kGetInfoIndex_* (see GetQueryResults) nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " - "h.last_visit_date, f.url, null, null, null, null, null, null " - "FROM moz_places_temp h " - "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE v.id = :visit_id OR v_t.id = :visit_id " - "UNION ALL " "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " "h.last_visit_date, f.url, null, null, null, null, null, null " "FROM moz_places h " - "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " + "JOIN moz_historyvisits v ON h.id = v.place_id " "LEFT JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE v.id = :visit_id OR v_t.id = :visit_id " - "LIMIT 1"), + "WHERE v.id = :visit_id "), getter_AddRefs(mDBVisitToURLResult)); NS_ENSURE_SUCCESS(rv, nsnull); @@ -8109,14 +7423,6 @@ nsNavHistory::GetDBBookmarkToUrlResult() // for perf reasons, LIMIT 1 will discard duplicates faster since we // have unique place ids. nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT b.fk, h.url, COALESCE(b.title, h.title), " - "h.rev_host, h.visit_count, h.last_visit_date, f.url, null, b.id, " - "b.dateAdded, b.lastModified, b.parent, null " - "FROM moz_bookmarks b " - "JOIN moz_places_temp h ON b.fk = h.id " - "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id " - "WHERE b.id = :item_id " - "UNION ALL " "SELECT b.fk, h.url, COALESCE(b.title, h.title), " "h.rev_host, h.visit_count, h.last_visit_date, f.url, null, b.id, " "b.dateAdded, b.lastModified, b.parent, null " @@ -8153,11 +7459,14 @@ nsNavHistory::FinalizeStatements() { mDBVisitToVisitResult, mDBBookmarkToUrlResult, mDBVisitsForFrecency, - mDBUpdateFrecencyAndHidden, + mDBUpdateFrecency, + mDBUpdateHiddenOnFrecency, mDBGetPlaceVisitStats, - mDBFullVisitCount, + mDBPageInfoForFrecency, mDBRegisterOpenPage, mDBUnregisterOpenPage, + mDBAsyncThreadPageInfoForFrecency, + mDBAsyncThreadVisitsForFrecency, }; for (PRUint32 i = 0; i < NS_ARRAY_LENGTH(stmts); i++) { diff --git a/toolkit/components/places/src/nsNavHistory.h b/toolkit/components/places/src/nsNavHistory.h index 95f12759384..99a1ccfcade 100644 --- a/toolkit/components/places/src/nsNavHistory.h +++ b/toolkit/components/places/src/nsNavHistory.h @@ -61,6 +61,7 @@ #include "nsICharsetResolver.h" #include "nsNetCID.h" #include "nsToolkitCompsCID.h" +#include "nsThreadUtils.h" #include "nsINavBookmarksService.h" #include "nsIPrivateBrowsingService.h" @@ -94,6 +95,12 @@ #define TOPIC_AUTOCOMPLETE_FEEDBACK_UPDATED "places-autocomplete-feedback-updated" #endif +// Fired after frecency has been updated. +#define TOPIC_FRECENCY_UPDATED "places-frecency-updated" + +// Fired after frecency has been updated. +#define TOPIC_FRECENCY_UPDATED "places-frecency-updated" + // Fired when Places is shutting down. Any code should stop accessing Places // APIs after this notification. If you need to listen for Places shutdown // you should only use this notification, next ones are intended only for @@ -121,15 +128,29 @@ namespace places { enum HistoryStatementId { DB_GET_PAGE_INFO_BY_URL = 0 - , DB_GET_TAGS = 1 - , DB_IS_PAGE_VISITED = 2 - , DB_INSERT_VISIT = 3 - , DB_RECENT_VISIT_OF_URL = 4 - , DB_GET_PAGE_VISIT_STATS = 5 - , DB_UPDATE_PAGE_VISIT_STATS = 6 - , DB_ADD_NEW_PAGE = 7 - , DB_GET_URL_PAGE_INFO = 8 - , DB_SET_PLACE_TITLE = 9 + , DB_GET_TAGS + , DB_IS_PAGE_VISITED + , DB_INSERT_VISIT + , DB_RECENT_VISIT_OF_URL + , DB_GET_PAGE_VISIT_STATS + , DB_UPDATE_PAGE_VISIT_STATS + , DB_ADD_NEW_PAGE + , DB_GET_URL_PAGE_INFO + , DB_SET_PLACE_TITLE + , DB_PAGE_INFO_FOR_FRECENCY + , DB_VISITS_FOR_FRECENCY + }; + + enum JournalMode { + // Default SQLite journal mode. + JOURNAL_DELETE = 0 + // Can reduce fsyncs on Linux when journal is deleted (See bug 460315). + // We fallback to this mode when WAL is unavailable. + , JOURNAL_TRUNCATE + // Unsafe in case of crashes on database swap or low memory. + , JOURNAL_MEMORY + // Can reduce number of fsyncs. We try to use this mode by default. + , JOURNAL_WAL }; } // namespace places @@ -200,6 +221,19 @@ public: return gHistoryService; } + /** + * Used by other components in the places directory to get a reference to a + * const version of this history object. + * + * @return A pointer to a const version of the service if it exists, + * NULL otherwise. + */ + static const nsNavHistory* GetConstHistoryService() + { + const nsNavHistory* const history = gHistoryService; + return history; + } + /** * Returns the database ID for the given URI, or 0 if not found and autoCreate * is false. @@ -207,12 +241,7 @@ public: nsresult GetUrlIdFor(nsIURI* aURI, PRInt64* aEntryID, PRBool aAutoCreate); - nsresult CalculateFullVisitCount(PRInt64 aPlaceId, PRInt32 *aVisitCount); - - nsresult UpdateFrecency(PRInt64 aPlaceId, PRBool aIsBookmark); - nsresult UpdateFrecencyInternal(PRInt64 aPlaceId, PRInt32 aTyped, - PRInt32 aHidden, PRInt32 aOldFrecency, - PRBool aIsBookmark); + nsresult UpdateFrecency(PRInt64 aPlaceId); /** * Calculate frecencies for places that don't have a valid value yet @@ -416,9 +445,12 @@ public: mozIStorageStatement* GetStatementById( enum mozilla::places::HistoryStatementId aStatementId - ) + ) const { using namespace mozilla::places; + + NS_ASSERTION(NS_IsMainThread(), "Can only get statement on main thread"); + switch(aStatementId) { case DB_GET_PAGE_INFO_BY_URL: return mDBGetURLPageInfo; @@ -440,10 +472,91 @@ public: return mDBGetURLPageInfo; case DB_SET_PLACE_TITLE: return mDBSetPlaceTitle; + case DB_PAGE_INFO_FOR_FRECENCY: + return mDBPageInfoForFrecency; + case DB_VISITS_FOR_FRECENCY: + return mDBVisitsForFrecency; } return nsnull; } + mozIStorageStatement* GetStatementByStoragePool( + enum mozilla::places::HistoryStatementId aStatementId + ) const + { + using namespace mozilla::places; + + switch(aStatementId) { + case DB_PAGE_INFO_FOR_FRECENCY: + return NS_IsMainThread() ? mDBPageInfoForFrecency + : mDBAsyncThreadPageInfoForFrecency; + case DB_VISITS_FOR_FRECENCY: + return NS_IsMainThread() ? mDBVisitsForFrecency + : mDBAsyncThreadVisitsForFrecency; + } + return nsnull; + } + + PRInt32 GetFrecencyAgedWeight(PRInt32 aAgeInDays) const + { + if (aAgeInDays <= mFirstBucketCutoffInDays) { + return mFirstBucketWeight; + } + if (aAgeInDays <= mSecondBucketCutoffInDays) { + return mSecondBucketWeight; + } + if (aAgeInDays <= mThirdBucketCutoffInDays) { + return mThirdBucketWeight; + } + if (aAgeInDays <= mFourthBucketCutoffInDays) { + return mFourthBucketWeight; + } + return mDefaultWeight; + } + + PRInt32 GetFrecencyBucketWeight(PRInt32 aBucketIndex) const + { + switch(aBucketIndex) { + case 1: + return mFirstBucketWeight; + case 2: + return mSecondBucketWeight; + case 3: + return mThirdBucketWeight; + case 4: + return mFourthBucketWeight; + default: + return mDefaultWeight; + } + } + + PRInt32 GetFrecencyTransitionBonus(PRInt32 aTransitionType, + bool aVisited) const + { + switch (aTransitionType) { + case nsINavHistoryService::TRANSITION_EMBED: + return mEmbedVisitBonus; + case nsINavHistoryService::TRANSITION_FRAMED_LINK: + return mFramedLinkVisitBonus; + case nsINavHistoryService::TRANSITION_LINK: + return mLinkVisitBonus; + case nsINavHistoryService::TRANSITION_TYPED: + return aVisited ? mTypedVisitBonus : mUnvisitedTypedBonus; + case nsINavHistoryService::TRANSITION_BOOKMARK: + return aVisited ? mBookmarkVisitBonus : mUnvisitedBookmarkBonus; + case nsINavHistoryService::TRANSITION_DOWNLOAD: + return mDownloadVisitBonus; + case nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT: + return mPermRedirectVisitBonus; + case nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY: + return mTempRedirectVisitBonus; + default: + // 0 == undefined (see bug #375777 for details) + NS_WARN_IF_FALSE(!aTransitionType, "new transition but no bonus for frecency"); + return mDefaultVisitBonus; + } + } + PRInt64 GetNewSessionID(); /** @@ -479,6 +592,7 @@ protected: nsCOMPtr mDBService; nsCOMPtr mDBConn; nsCOMPtr mDBFile; + PRInt32 mDBPageSize; nsCOMPtr mDBGetURLPageInfo; // kGetInfoIndex_* results nsCOMPtr mDBGetIdPageInfo; // kGetInfoIndex_* results @@ -531,9 +645,16 @@ protected: nsresult CalculateFrecency(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, nsCAutoString &aURL, PRInt32 *aFrecency); nsresult CalculateFrecencyInternal(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, PRBool aIsBookmarked, PRInt32 *aFrecency); nsCOMPtr mDBVisitsForFrecency; - nsCOMPtr mDBUpdateFrecencyAndHidden; + nsCOMPtr mDBUpdateFrecency; + nsCOMPtr mDBUpdateHiddenOnFrecency; nsCOMPtr mDBGetPlaceVisitStats; - nsCOMPtr mDBFullVisitCount; + nsCOMPtr mDBPageInfoForFrecency; + + // Cached statements used in frecency calculation. Since it could happen on + // both main thread or storage async thread, we keep two versions of them + // for thread-safety. + nsCOMPtr mDBAsyncThreadVisitsForFrecency; + nsCOMPtr mDBAsyncThreadPageInfoForFrecency; /** * Initializes the database file. If the database does not exist, was @@ -546,6 +667,12 @@ protected: */ nsresult InitDBFile(PRBool aForceInit); + /** + * Set journal mode on the database. + */ + nsresult SetJournalMode(enum mozilla::places::JournalMode aJournalMode); + enum mozilla::places::JournalMode mCurrentJournalMode; + /** * Initializes the database. This performs any necessary migrations for the * database. All migration is done inside a transaction that is rolled back @@ -560,7 +687,6 @@ protected: */ nsresult InitAdditionalDBItems(); nsresult InitTempTables(); - nsresult InitViews(); nsresult InitFunctions(); nsresult InitStatements(); nsresult ForceMigrateBookmarksDB(mozIStorageConnection *aDBConn); @@ -570,6 +696,7 @@ protected: nsresult MigrateV8Up(mozIStorageConnection *aDBConn); nsresult MigrateV9Up(mozIStorageConnection *aDBConn); nsresult MigrateV10Up(mozIStorageConnection *aDBConn); + nsresult MigrateV11Up(mozIStorageConnection *aDBConn); nsresult RemovePagesInternal(const nsCString& aPlaceIdsQueryString); nsresult PreparePlacesForVisitsDelete(const nsCString& aPlaceIdsQueryString); diff --git a/toolkit/components/places/src/nsPlacesAutoComplete.js b/toolkit/components/places/src/nsPlacesAutoComplete.js index 93e9a1eedb3..3940c5bf802 100644 --- a/toolkit/components/places/src/nsPlacesAutoComplete.js +++ b/toolkit/components/places/src/nsPlacesAutoComplete.js @@ -108,33 +108,6 @@ const kTitleTagsSeparator = " \u2013 "; const kBrowserUrlbarBranch = "browser.urlbar."; -//////////////////////////////////////////////////////////////////////////////// -//// Global Functions - -/** - * Generates the SQL subquery to get the best favicon for a given revhost. This - * is the favicon for the most recent visit. - * - * @param aTableName - * The table to join to the moz_favicons table with. This must have a - * column called favicon_id. - * @return the SQL subquery (in string form) to get the best favicon. - */ -function best_favicon_for_revhost(aTableName) -{ - return "(" + - "SELECT f.url " + - "FROM " + aTableName + " " + - "JOIN moz_favicons f ON f.id = favicon_id " + - "WHERE rev_host = IFNULL( " + - "(SELECT rev_host FROM moz_places_temp WHERE id = b.fk), " + - "(SELECT rev_host FROM moz_places WHERE id = b.fk) " + - ") " + - "ORDER BY frecency DESC " + - "LIMIT 1 " + - ")"; -} - //////////////////////////////////////////////////////////////////////////////// //// AutoCompleteStatementCallbackWrapper class @@ -211,32 +184,24 @@ function nsPlacesAutoComplete() ////////////////////////////////////////////////////////////////////////////// //// Shared Constants for Smart Getters - // Define common pieces of various queries. // TODO bug 412736 in case of a frecency tie, break it with h.typed and // h.visit_count which is better than nothing. This is slow, so not doing it // yet... - // Note: h.frecency is only selected because we need it for ordering. - function sql_base_fragment(aTableName) { - return "SELECT h.url, h.title, f.url, " + kBookTagSQLFragment + ", " + - "h.visit_count, h.typed, h.id, :query_type, t.open_count, " + - "h.frecency " + - "FROM " + aTableName + " h " + - "LEFT OUTER JOIN moz_favicons f ON f.id = h.favicon_id " + - "LEFT OUTER JOIN moz_openpages_temp t ON t.url = h.url " + - "WHERE h.frecency <> 0 " + - "AND AUTOCOMPLETE_MATCH(:searchString, h.url, " + - "IFNULL(bookmark, h.title), tags, " + - "h.visit_count, h.typed, parent, " + - "t.open_count, " + - ":matchBehavior, :searchBehavior) " + - "{ADDITIONAL_CONDITIONS} "; - } - const SQL_BASE = sql_base_fragment("moz_places_temp") + - "UNION ALL " + - sql_base_fragment("moz_places") + - "AND +h.id NOT IN (SELECT id FROM moz_places_temp) " + - "ORDER BY h.frecency DESC, h.id DESC " + - "LIMIT :maxResults"; + const SQL_BASE = "SELECT h.url, h.title, f.url, " + kBookTagSQLFragment + ", " + + "h.visit_count, h.typed, h.id, :query_type, " + + "t.open_count " + + "FROM moz_places h " + + "LEFT JOIN moz_favicons f ON f.id = h.favicon_id " + + "LEFT JOIN moz_openpages_temp t ON t.url = h.url " + + "WHERE h.frecency <> 0 " + + "AND AUTOCOMPLETE_MATCH(:searchString, h.url, " + + "IFNULL(bookmark, h.title), tags, " + + "h.visit_count, h.typed, parent, " + + "t.open_count, " + + ":matchBehavior, :searchBehavior) " + + "{ADDITIONAL_CONDITIONS} " + + "ORDER BY h.frecency DESC, h.id DESC " + + "LIMIT :maxResults"; ////////////////////////////////////////////////////////////////////////////// //// Smart Getters @@ -269,60 +234,56 @@ function nsPlacesAutoComplete() XPCOMUtils.defineLazyGetter(this, "_defaultQuery", function() { let replacementText = ""; - return this._db.createStatement( + return this._db.createAsyncStatement( SQL_BASE.replace("{ADDITIONAL_CONDITIONS}", replacementText, "g") ); }); XPCOMUtils.defineLazyGetter(this, "_historyQuery", function() { let replacementText = "AND h.visit_count > 0"; - return this._db.createStatement( + return this._db.createAsyncStatement( SQL_BASE.replace("{ADDITIONAL_CONDITIONS}", replacementText, "g") ); }); XPCOMUtils.defineLazyGetter(this, "_bookmarkQuery", function() { let replacementText = "AND bookmark IS NOT NULL"; - return this._db.createStatement( + return this._db.createAsyncStatement( SQL_BASE.replace("{ADDITIONAL_CONDITIONS}", replacementText, "g") ); }); XPCOMUtils.defineLazyGetter(this, "_tagsQuery", function() { let replacementText = "AND tags IS NOT NULL"; - return this._db.createStatement( + return this._db.createAsyncStatement( SQL_BASE.replace("{ADDITIONAL_CONDITIONS}", replacementText, "g") ); }); XPCOMUtils.defineLazyGetter(this, "_openPagesQuery", function() { - return this._db.createStatement( - "/* do not warn (bug 487789) */ " + - "SELECT t.url, " + - "COALESCE(h_t.title, h.title, t.url) AS c_title, f.url, " + - kBookTagSQLFragment + ", " + - "IFNULL(h_t.visit_count, h.visit_count) AS c_visit_count, " + - "IFNULL(h_t.typed, h.typed) AS c_typed, " + - "IFNULL(h_t.id, h.id), :query_type, t.open_count, " + - "IFNULL(h_t.frecency, h.frecency) AS c_frecency " + - "FROM moz_openpages_temp t " + - "LEFT JOIN moz_places_temp h_t ON h_t.url = t.url " + - "LEFT JOIN moz_places h ON h.url = t.url " + - "LEFT JOIN moz_favicons f ON f.id = IFNULL(h_t.favicon_id, h.favicon_id) " + - "WHERE t.open_count > 0 " + - "AND AUTOCOMPLETE_MATCH(:searchString, t.url, " + - "COALESCE(bookmark, c_title, t.url), tags, " + - "c_visit_count, c_typed, parent, " + - "t.open_count, " + - ":matchBehavior, :searchBehavior) " + - "ORDER BY c_frecency DESC, t.ROWID DESC " + - "LIMIT :maxResults" + return this._db.createAsyncStatement( + "SELECT t.url, " + + "IFNULL(h.title, t.url) AS c_title, f.url, " + + kBookTagSQLFragment + ", " + + "h.visit_count, h.typed, " + + "h.id, :query_type, t.open_count, h.frecency " + + "FROM moz_openpages_temp t " + + "LEFT JOIN moz_places h ON h.url = t.url " + + "LEFT JOIN moz_favicons f ON f.id = h.favicon_id " + + "WHERE t.open_count > 0 " + + "AND AUTOCOMPLETE_MATCH(:searchString, t.url, " + + "COALESCE(bookmark, c_title, t.url), tags, " + + "h.visit_count, h.typed, parent, " + + "t.open_count, " + + ":matchBehavior, :searchBehavior) " + + "ORDER BY h.frecency DESC, t.ROWID DESC " + + "LIMIT :maxResults" ); }); XPCOMUtils.defineLazyGetter(this, "_typedQuery", function() { let replacementText = "AND h.typed = 1"; - return this._db.createStatement( + return this._db.createAsyncStatement( SQL_BASE.replace("{ADDITIONAL_CONDITIONS}", replacementText, "g") ); }); @@ -331,58 +292,55 @@ function nsPlacesAutoComplete() // In this query, we are taking kBookTagSQLFragment only for h.id because it // uses data from the moz_bookmarks table and we sync tables on bookmark // insert. So, most likely, h.id will always be populated when we have any - // bookmark. We still need to join on moz_places_temp for other data (eg. - // title). - return this._db.createStatement( - "/* do not warn (bug 487789) */ " + - "SELECT IFNULL(h_t.url, h.url) AS c_url, " + - "IFNULL(h_t.title, h.title) AS c_title, f.url, " + - kBookTagSQLFragment + ", " + - "IFNULL(h_t.visit_count, h.visit_count) AS c_visit_count, " + - "IFNULL(h_t.typed, h.typed) AS c_typed, " + - "IFNULL(h_t.id, h.id), :query_type, t.open_count, rank " + - "FROM ( " + - "SELECT ROUND(MAX(((i.input = :search_string) + " + - "(SUBSTR(i.input, 1, LENGTH(:search_string)) = :search_string)) * " + - "i.use_count), 1) AS rank, place_id " + - "FROM moz_inputhistory i " + - "GROUP BY i.place_id " + - "HAVING rank > 0 " + - ") AS i " + - "LEFT JOIN moz_places h ON h.id = i.place_id " + - "LEFT JOIN moz_places_temp h_t ON h_t.id = i.place_id " + - "LEFT JOIN moz_favicons f ON f.id = IFNULL(h_t.favicon_id, h.favicon_id) " + - "LEFT JOIN moz_openpages_temp t ON t.url = c_url " + - "WHERE c_url NOTNULL " + - "AND AUTOCOMPLETE_MATCH(:searchString, c_url, " + - "IFNULL(bookmark, c_title), tags, " + - "c_visit_count, c_typed, parent, " + - "t.open_count, " + - ":matchBehavior, :searchBehavior) " + - "ORDER BY rank DESC, IFNULL(h_t.frecency, h.frecency) DESC" + // bookmark. + return this._db.createAsyncStatement( + "/* do not warn (bug 487789) */ " + + "SELECT h.url, h.title, f.url, " + kBookTagSQLFragment + ", " + + "h.visit_count, h.typed, h.id, :query_type, t.open_count, rank " + + "FROM ( " + + "SELECT ROUND( " + + "MAX(((i.input = :search_string) + " + + "(SUBSTR(i.input, 1, LENGTH(:search_string)) = :search_string) " + + ") * i.use_count " + + ") , 1 " // Round at first decimal. + + ") AS rank, place_id " + + "FROM moz_inputhistory i " + + "GROUP BY i.place_id " + + "HAVING rank > 0 " + + ") AS i " + + "JOIN moz_places h ON h.id = i.place_id " + + "LEFT JOIN moz_favicons f ON f.id = h.favicon_id " + + "LEFT JOIN moz_openpages_temp t ON t.url = h.url " + + "WHERE AUTOCOMPLETE_MATCH(:searchString, h.url, " + + "IFNULL(bookmark, h.title), tags, " + + "h.visit_count, h.typed, parent, " + + "t.open_count, " + + ":matchBehavior, :searchBehavior) " + + "ORDER BY rank DESC, h.frecency DESC " ); }); XPCOMUtils.defineLazyGetter(this, "_keywordQuery", function() { - return this._db.createStatement( - "/* do not warn (bug 487787) */ " + - "SELECT IFNULL( " + - "(SELECT REPLACE(url, '%s', :query_string) FROM moz_places_temp WHERE id = b.fk), " + - "(SELECT REPLACE(url, '%s', :query_string) FROM moz_places WHERE id = b.fk) " + - ") AS search_url, IFNULL(h_t.title, h.title), " + - "COALESCE(f.url, " + best_favicon_for_revhost("moz_places_temp") + "," + - best_favicon_for_revhost("moz_places") + "), b.parent, " + - "b.title, NULL, IFNULL(h_t.visit_count, h.visit_count), " + - "IFNULL(h_t.typed, h.typed), COALESCE(h_t.id, h.id, b.fk), " + - ":query_type, t.open_count " + - "FROM moz_keywords k " + - "JOIN moz_bookmarks b ON b.keyword_id = k.id " + - "LEFT JOIN moz_places AS h ON h.url = search_url " + - "LEFT JOIN moz_places_temp AS h_t ON h_t.url = search_url " + - "LEFT JOIN moz_favicons f ON f.id = IFNULL(h_t.favicon_id, h.favicon_id) " + - "LEFT JOIN moz_openpages_temp t ON t.url = search_url " + - "WHERE LOWER(k.keyword) = LOWER(:keyword) " + - "ORDER BY IFNULL(h_t.frecency, h.frecency) DESC" + return this._db.createAsyncStatement( + "/* do not warn (bug 487787) */ " + + "SELECT " + + "(SELECT REPLACE(url, '%s', :query_string) FROM moz_places WHERE id = b.fk) " + + "AS search_url, h.title, " + + "IFNULL(f.url, (SELECT f.url " + + "FROM moz_places " + + "JOIN moz_favicons f ON f.id = favicon_id " + + "WHERE rev_host = (SELECT rev_host FROM moz_places WHERE id = b.fk) " + + "ORDER BY frecency DESC " + + "LIMIT 1) " + + "), b.parent, b.title, NULL, h.visit_count, h.typed, IFNULL(h.id, b.fk), " + + ":query_type, t.open_count " + + "FROM moz_keywords k " + + "JOIN moz_bookmarks b ON b.keyword_id = k.id " + + "LEFT JOIN moz_places h ON h.url = search_url " + + "LEFT JOIN moz_favicons f ON f.id = h.favicon_id " + + "LEFT JOIN moz_openpages_temp t ON t.url = search_url " + + "WHERE LOWER(k.keyword) = LOWER(:keyword) " + + "ORDER BY h.frecency DESC " ); }); diff --git a/toolkit/components/places/src/nsPlacesDBFlush.js b/toolkit/components/places/src/nsPlacesDBFlush.js deleted file mode 100644 index 7ec878a8ebb..00000000000 --- a/toolkit/components/places/src/nsPlacesDBFlush.js +++ /dev/null @@ -1,384 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -//////////////////////////////////////////////////////////////////////////////// -//// Constants - -const Cc = Components.classes; -const Ci = Components.interfaces; -const Cr = Components.results; -const Cu = Components.utils; - -// A database flush should be the last operation during the shutdown process. -const kTopicShutdown = "places-connection-closing"; -const kSyncFinished = "places-sync-finished"; -const kDebugStopSync = "places-debug-stop-sync"; -const kDebugStartSync = "places-debug-start-sync"; - -const kSyncPrefName = "places.syncDBTableIntervalInSecs"; -const kDefaultSyncInterval = 120; - -// Query Constants. These describe the queries we use. -const kQuerySyncPlacesId = 0; -const kQuerySyncHistoryVisitsId = 1; - -//////////////////////////////////////////////////////////////////////////////// -//// Modules - -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/PlacesUtils.jsm"); - -//////////////////////////////////////////////////////////////////////////////// -//// nsPlacesDBFlush class - -function nsPlacesDBFlush() -{ - // Get our sync interval - try { - // We want to silently fail since getIntPref throws if it does not exist, - // and use a default to fallback to. - this._syncInterval = Services.prefs.getIntPref(kSyncPrefName); - if (this._syncInterval <= 0) - this._syncInterval = kDefaultSyncInterval; - } - catch (e) { - // The preference did not exist, so use the default. - this._syncInterval = kDefaultSyncInterval; - } - - // Register observers - Services.obs.addObserver(this, kTopicShutdown, false); - Services.obs.addObserver(this, kDebugStopSync, false); - Services.obs.addObserver(this, kDebugStartSync, false); - - let (pb2 = Services.prefs.QueryInterface(Ci.nsIPrefBranch2)) - pb2.addObserver(kSyncPrefName, this, false); - - // Create our timer to update everything - this._timer = this._newTimer(); - - ////////////////////////////////////////////////////////////////////////////// - //// Smart Getters - - XPCOMUtils.defineLazyGetter(this, "_db", function() { - return PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase) - .DBConnection; - }); -} - -nsPlacesDBFlush.prototype = { - ////////////////////////////////////////////////////////////////////////////// - //// nsIObserver - - observe: function DBFlush_observe(aSubject, aTopic, aData) - { - if (aTopic == kTopicShutdown) { - Services.obs.removeObserver(this, kTopicShutdown); - Services.obs.removeObserver(this, kDebugStopSync); - Services.obs.removeObserver(this, kDebugStartSync); - - let (pb2 = Services.prefs.QueryInterface(Ci.nsIPrefBranch2)) - pb2.removeObserver(kSyncPrefName, this); - - if (this._timer) { - this._timer.cancel(); - this._timer = null; - } - - // Flush any remaining change to disk tables. - this._flushWithQueries([kQuerySyncPlacesId, kQuerySyncHistoryVisitsId]); - this._finalizeInternalStatements(); - } - else if (aTopic == "nsPref:changed" && aData == kSyncPrefName) { - // Get the new pref value, and then update our timer - this._syncInterval = Services.prefs.getIntPref(kSyncPrefName); - if (this._syncInterval <= 0) - this._syncInterval = kDefaultSyncInterval; - - // We may have canceled the timer already for batch updates, so we want to - // exit early. - if (!this._timer) - return; - - this._timer.cancel(); - this._timer = this._newTimer(); - } - else if (aTopic == kDebugStopSync) { - this._syncStopped = true; - } - else if (aTopic == kDebugStartSync) { - if (_syncStopped in this) - delete this._syncStopped; - } - }, - - ////////////////////////////////////////////////////////////////////////////// - //// nsINavBookmarkObserver - - onBeginUpdateBatch: function DBFlush_onBeginUpdateBatch() - { - this._inBatchMode = true; - - // We do not want to sync while we are doing batch work. - this._timer.cancel(); - this._timer = null; - }, - - onEndUpdateBatch: function DBFlush_onEndUpdateBatch() - { - this._inBatchMode = false; - - // Restore our timer - this._timer = this._newTimer(); - - // We need to sync now - this._flushWithQueries([kQuerySyncPlacesId, kQuerySyncHistoryVisitsId]); - }, - - onItemAdded: function(aItemId, aParentId, aIndex, aItemType) - { - // Sync only if we added a TYPE_BOOKMARK item. Note, we want to run the - // least amount of queries as possible here for performance reasons. - if (!this._inBatchMode && aItemType == PlacesUtils.bookmarks.TYPE_BOOKMARK) - this._flushWithQueries([kQuerySyncPlacesId]); - }, - - onItemChanged: function DBFlush_onItemChanged(aItemId, aProperty, - aIsAnnotationProperty, - aNewValue, aLastModified, - aItemType) - { - if (!this._inBatchMode && aProperty == "uri") - this._flushWithQueries([kQuerySyncPlacesId]); - }, - - onBeforeItemRemoved: function() { }, - onItemRemoved: function() { }, - onItemVisited: function() { }, - onItemMoved: function() { }, - - ////////////////////////////////////////////////////////////////////////////// - //// nsINavHistoryObserver - - // We currently only use the history observer to know when the history service - // is activated. At that point, we actually get initialized, and our timer - // to sync history is added. - - // These methods share the name of the ones on nsINavBookmarkObserver, so - // the implementations can be found above. - //onBeginUpdateBatch: function() { }, - //onEndUpdateBatch: function() { }, - onVisit: function() { }, - onTitleChanged: function() { }, - onBeforeDeleteURI: function() { }, - onDeleteURI: function() { }, - onClearHistory: function() { }, - onPageChanged: function() { }, - onDeleteVisits: function() { }, - - ////////////////////////////////////////////////////////////////////////////// - //// nsITimerCallback - - notify: function DBFlush_timerCallback() - { - let queries = [ - kQuerySyncPlacesId, - kQuerySyncHistoryVisitsId, - ]; - this._flushWithQueries(queries); - }, - - ////////////////////////////////////////////////////////////////////////////// - //// mozIStorageStatementCallback - - handleResult: function DBFlush_handleResult(aResultSet) - { - }, - - handleError: function DBFlush_handleError(aError) - { - Cu.reportError("Async statement execution returned with '" + - aError.result + "', '" + aError.message + "'"); - }, - - handleCompletion: function DBFlush_handleCompletion(aReason) - { - if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) { - // Dispatch a notification that sync has finished. - Services.obs.notifyObservers(null, kSyncFinished, null); - } - }, - - ////////////////////////////////////////////////////////////////////////////// - //// nsPlacesDBFlush - _syncInterval: kDefaultSyncInterval, - - /** - * Execute async statements to flush tables with the specified queries. - * - * @param aQueryNames - * The names of the queries to use with this flush. - */ - _flushWithQueries: function DBFlush_flushWithQueries(aQueryNames) - { - // No need to do extra work if we are in batch mode - if (this._inBatchMode || this._syncStopped) - return; - - let statements = []; - for (let i = 0; i < aQueryNames.length; i++) - statements.push(this._getQuery(aQueryNames[i])); - - // Execute sync statements async in a transaction - this._db.executeAsync(statements, statements.length, this); - }, - - /** - * Finalizes all of our mozIStorageStatements so we can properly close the - * database. - */ - _finalizeInternalStatements: function DBFlush_finalizeInternalStatements() - { - this._cachedStatements.forEach(function(stmt) { - if (stmt instanceof Ci.mozIStorageAsyncStatement) - stmt.finalize(); - }); - }, - - /** - * Generate the statement to synchronizes the moz_{aTableName} and - * moz_{aTableName}_temp by copying all the data from the temporary table - * into the permanent one. - * Most of the work is done through triggers defined in nsPlacesTriggers.h, - * they sync back to disk, then delete the data in the temporary table. - * - * @param aQueryType - * Type of the query to build statement for. - */ - _cachedStatements: [], - _getQuery: function DBFlush_getQuery(aQueryType) - { - // Statement creating can be expensive, so always cache if we can. - if (aQueryType in this._cachedStatements) { - let stmt = this._cachedStatements[aQueryType]; - - // Bind the appropriate parameters. - let params = stmt.params; - switch (aQueryType) { - case kQuerySyncHistoryVisitsId: - case kQuerySyncPlacesId: - params.transition_type = Ci.nsINavHistoryService.TRANSITION_EMBED; - break; - } - - return stmt; - } - - switch(aQueryType) { - case kQuerySyncHistoryVisitsId: - // For history table we want to leave embed visits in memory, since - // those are expired with current session, so we are filtering them out. - // Notice that instead we _want_ to sync framed_link visits, since those - // come from user's actions. - this._cachedStatements[aQueryType] = this._db.createAsyncStatement( - "DELETE FROM moz_historyvisits_temp " + - "WHERE visit_type <> :transition_type" - ); - break; - - case kQuerySyncPlacesId: - // For places table we want to leave places associated with embed visits - // in memory, they usually have hidden = 1 and at least an embed visit - // in historyvisits_temp table. - this._cachedStatements[aQueryType] = this._db.createAsyncStatement( - "DELETE FROM moz_places_temp " + - "WHERE id IN ( " + - "SELECT id FROM moz_places_temp h " + - "WHERE h.hidden <> 1 OR NOT EXISTS ( " + - "SELECT id FROM moz_historyvisits_temp " + - "WHERE place_id = h.id AND visit_type = :transition_type " + - "LIMIT 1 " + - ") " + - ")" - ); - break; - - default: - throw "Unexpected statement!"; - } - - // We only bind our own parameters when we have a cached statement, so we - // call ourself since we now have a cached statement. - return this._getQuery(aQueryType); - }, - - /** - * Creates a new timer based on this._syncInterval. - * - * @returns a REPEATING_SLACK nsITimer that runs every this._syncInterval. - */ - _newTimer: function DBFlush_newTimer() - { - let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); - timer.initWithCallback(this, this._syncInterval * 1000, - Ci.nsITimer.TYPE_REPEATING_SLACK); - return timer; - }, - - ////////////////////////////////////////////////////////////////////////////// - //// nsISupports - - classID: Components.ID("c1751cfc-e8f1-4ade-b0bb-f74edfb8ef6a"), - - QueryInterface: XPCOMUtils.generateQI([ - Ci.nsIObserver, - Ci.nsINavBookmarkObserver, - Ci.nsINavHistoryObserver, - Ci.nsITimerCallback, - Ci.mozIStorageStatementCallback, - ]) -}; - -//////////////////////////////////////////////////////////////////////////////// -//// Module Registration - -let components = [nsPlacesDBFlush]; -var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/toolkit/components/places/src/nsPlacesExpiration.js b/toolkit/components/places/src/nsPlacesExpiration.js index 501a4a61aa7..f35bbaa80e6 100644 --- a/toolkit/components/places/src/nsPlacesExpiration.js +++ b/toolkit/components/places/src/nsPlacesExpiration.js @@ -211,23 +211,10 @@ const EXPIRATION_QUERIES = { QUERY_FIND_VISITS_TO_EXPIRE: { sql: "INSERT INTO expiration_notify " + "(v_id, url, visit_date, expected_results) " - + "SELECT v.id, IFNULL(h_t.url, h.url) AS url, " - + "v.visit_date AS visit_date, " - + ":limit_visits AS expected_results " - + "FROM moz_historyvisits_temp v " - + "LEFT JOIN moz_places_temp AS h_t ON h_t.id = v.place_id " - + "LEFT JOIN moz_places AS h ON h.id = v.place_id " - + "WHERE ((SELECT COUNT(*) FROM moz_places_temp) + " - + "(SELECT COUNT(*) FROM moz_places)) > :max_uris " - + "UNION ALL " - + "SELECT v.id, IFNULL(h_t.url, h.url) AS url, " - + "v.visit_date AS visit_date, " - + ":limit_visits AS expected_results " + + "SELECT v.id, h.url, v.visit_date, :limit_visits " + "FROM moz_historyvisits v " - + "LEFT JOIN moz_places_temp AS h_t ON h_t.id = v.place_id " - + "LEFT JOIN moz_places AS h ON h.id = v.place_id " - + "WHERE ((SELECT COUNT(*) FROM moz_places_temp) + " - + "(SELECT COUNT(*) FROM moz_places)) > :max_uris " + + "JOIN moz_places h ON h.id = v.place_id " + + "WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris " + "ORDER BY v.visit_date ASC " + "LIMIT :limit_visits", actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE | @@ -236,7 +223,7 @@ const EXPIRATION_QUERIES = { // Removes the previously found visits. QUERY_EXPIRE_VISITS: { - sql: "DELETE FROM moz_historyvisits_view WHERE id IN ( " + sql: "DELETE FROM moz_historyvisits WHERE id IN ( " + "SELECT v_id FROM expiration_notify WHERE v_id NOTNULL " + ")", actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE | @@ -249,26 +236,13 @@ const EXPIRATION_QUERIES = { QUERY_FIND_URIS_TO_EXPIRE: { sql: "INSERT INTO expiration_notify " + "(p_id, url, visit_date, expected_results) " - + "SELECT h.id, h.url, h.last_visit_date AS visit_date, " - + ":limit_uris AS expected_results " - + "FROM moz_places_temp h " - + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - + "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " - + "WHERE v.id IS NULL " - + "AND v_t.id IS NULL " - + "AND b.id IS NULL " - + "AND h.ROWID <> IFNULL(:null_skips_last, (SELECT MAX(ROWID) FROM moz_places_temp)) " - + "UNION ALL " - + "SELECT h.id, h.url, h.last_visit_date AS visit_date, " - + ":limit_uris AS expected_results " + + "SELECT h.id, h.url, h.last_visit_date, :limit_uris " + "FROM moz_places h " + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - + "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " + "WHERE v.id IS NULL " - + "AND v_t.id IS NULL " + "AND b.id IS NULL " + + "AND h.ROWID <> IFNULL(:null_skips_last, (SELECT MAX(ROWID) FROM moz_places)) " + "LIMIT :limit_uris", actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG @@ -276,7 +250,7 @@ const EXPIRATION_QUERIES = { // Expire found URIs from the database. QUERY_EXPIRE_URIS: { - sql: "DELETE FROM moz_places_view WHERE id IN ( " + sql: "DELETE FROM moz_places WHERE id IN ( " + "SELECT p_id FROM expiration_notify WHERE p_id NOTNULL " + ")", actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | @@ -285,26 +259,13 @@ const EXPIRATION_QUERIES = { // Expire orphan URIs from the database. QUERY_SILENT_EXPIRE_ORPHAN_URIS: { - sql: "DELETE FROM moz_places_view WHERE id IN ( " - + "SELECT h.id " - + "FROM moz_places_temp h " - + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - + "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " - + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " - + "WHERE v.id IS NULL " - + "AND v_t.id IS NULL " - + "AND b.id IS NULL " - + "AND SUBSTR(h.url, 1, 6) <> 'place:' " - + "UNION ALL " + sql: "DELETE FROM moz_places WHERE id IN ( " + "SELECT h.id " + "FROM moz_places h " + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " - + "LEFT JOIN moz_historyvisits_temp v_t ON h.id = v_t.place_id " + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " + "WHERE v.id IS NULL " - + "AND v_t.id IS NULL " + "AND b.id IS NULL " - + "AND SUBSTR(h.url, 1, 6) <> 'place:' " + "LIMIT :limit_uris " + ")", actions: ACTION.CLEAR_HISTORY @@ -315,9 +276,7 @@ const EXPIRATION_QUERIES = { sql: "DELETE FROM moz_favicons WHERE id IN ( " + "SELECT f.id FROM moz_favicons f " + "LEFT JOIN moz_places h ON f.id = h.favicon_id " - + "LEFT JOIN moz_places_temp h_t ON f.id = h_t.favicon_id " + "WHERE h.favicon_id IS NULL " - + "AND h_t.favicon_id IS NULL " + "LIMIT :limit_favicons " + ")", actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | @@ -329,12 +288,9 @@ const EXPIRATION_QUERIES = { sql: "DELETE FROM moz_annos WHERE id in ( " + "SELECT a.id FROM moz_annos a " + "LEFT JOIN moz_places h ON a.place_id = h.id " - + "LEFT JOIN moz_places_temp h_t ON a.place_id = h_t.id " + "LEFT JOIN moz_historyvisits v ON a.place_id = v.place_id " - + "LEFT JOIN moz_historyvisits_temp v_t ON a.place_id = v_t.place_id " - + "WHERE (h.id IS NULL AND h_t.id IS NULL) " - + "OR (v.id IS NULL AND v_t.id IS NULL AND " - + "a.expiration <> :expire_never) " + + "WHERE h.id IS NULL " + + "OR (v.id IS NULL AND a.expiration <> :expire_never) " + "LIMIT :limit_annos " + ")", actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | @@ -371,8 +327,6 @@ const EXPIRATION_QUERIES = { QUERY_EXPIRE_ANNOS_WITH_HISTORY: { sql: "DELETE FROM moz_annos " + "WHERE expiration = :expire_with_history " - + "AND NOT EXISTS (SELECT id FROM moz_historyvisits_temp " - + "WHERE place_id = moz_annos.place_id LIMIT 1) " + "AND NOT EXISTS (SELECT id FROM moz_historyvisits " + "WHERE place_id = moz_annos.place_id LIMIT 1)", actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | @@ -408,9 +362,7 @@ const EXPIRATION_QUERIES = { sql: "DELETE FROM moz_inputhistory WHERE place_id IN ( " + "SELECT i.place_id FROM moz_inputhistory i " + "LEFT JOIN moz_places h ON h.id = i.place_id " - + "LEFT JOIN moz_places_temp h_t ON h_t.id = i.place_id " + "WHERE h.id IS NULL " - + "AND h_t.id IS NULL " + "LIMIT :limit_inputhistory " + ")", actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG @@ -610,8 +562,7 @@ nsPlacesExpiration.prototype = { // Check if we are over history capacity, if so visits must be expired. if (!this._cachedStatements["LIMIT_COUNT"]) { this._cachedStatements["LIMIT_COUNT"] = this._db.createAsyncStatement( - "SELECT (SELECT COUNT(*) FROM moz_places_temp) + " - + "(SELECT COUNT(*) FROM moz_places)" + "SELECT COUNT(*) FROM moz_places" ); } let self = this; diff --git a/toolkit/components/places/src/nsPlacesIndexes.h b/toolkit/components/places/src/nsPlacesIndexes.h index 7b9bb896906..131cc7bd4dc 100644 --- a/toolkit/components/places/src/nsPlacesIndexes.h +++ b/toolkit/components/places/src/nsPlacesIndexes.h @@ -49,55 +49,31 @@ /** * moz_places */ -#define CREATE_IDX_MOZ_PLACES_TEMP_URL \ - CREATE_PLACES_IDX( \ - "url_uniqueindex", "moz_places_temp", "url", "UNIQUE" \ - ) #define CREATE_IDX_MOZ_PLACES_URL \ CREATE_PLACES_IDX( \ "url_uniqueindex", "moz_places", "url", "UNIQUE" \ ) -#define CREATE_IDX_MOZ_PLACES_TEMP_FAVICON \ - CREATE_PLACES_IDX( \ - "faviconindex", "moz_places_temp", "favicon_id", "" \ - ) #define CREATE_IDX_MOZ_PLACES_FAVICON \ CREATE_PLACES_IDX( \ "faviconindex", "moz_places", "favicon_id", "" \ ) -#define CREATE_IDX_MOZ_PLACES_TEMP_REVHOST \ - CREATE_PLACES_IDX( \ - "hostindex", "moz_places_temp", "rev_host", "" \ - ) #define CREATE_IDX_MOZ_PLACES_REVHOST \ CREATE_PLACES_IDX( \ "hostindex", "moz_places", "rev_host", "" \ ) -#define CREATE_IDX_MOZ_PLACES_TEMP_VISITCOUNT \ - CREATE_PLACES_IDX( \ - "visitcount", "moz_places_temp", "visit_count", "" \ - ) #define CREATE_IDX_MOZ_PLACES_VISITCOUNT \ CREATE_PLACES_IDX( \ "visitcount", "moz_places", "visit_count", "" \ ) -#define CREATE_IDX_MOZ_PLACES_TEMP_FRECENCY \ - CREATE_PLACES_IDX( \ - "frecencyindex", "moz_places_temp", "frecency", "" \ - ) #define CREATE_IDX_MOZ_PLACES_FRECENCY \ CREATE_PLACES_IDX( \ "frecencyindex", "moz_places", "frecency", "" \ ) -#define CREATE_IDX_MOZ_PLACES_TEMP_LASTVISITDATE \ - CREATE_PLACES_IDX( \ - "lastvisitdateindex", "moz_places_temp", "last_visit_date", "" \ - ) #define CREATE_IDX_MOZ_PLACES_LASTVISITDATE \ CREATE_PLACES_IDX( \ "lastvisitdateindex", "moz_places", "last_visit_date", "" \ @@ -107,28 +83,16 @@ * moz_historyvisits */ -#define CREATE_IDX_MOZ_HISTORYVISITS_TEMP_PLACEDATE \ - CREATE_PLACES_IDX( \ - "placedateindex", "moz_historyvisits_temp", "place_id, visit_date", "" \ - ) #define CREATE_IDX_MOZ_HISTORYVISITS_PLACEDATE \ CREATE_PLACES_IDX( \ "placedateindex", "moz_historyvisits", "place_id, visit_date", "" \ ) -#define CREATE_IDX_MOZ_HISTORYVISITS_TEMP_FROMVISIT \ - CREATE_PLACES_IDX( \ - "fromindex", "moz_historyvisits_temp", "from_visit", "" \ - ) #define CREATE_IDX_MOZ_HISTORYVISITS_FROMVISIT \ CREATE_PLACES_IDX( \ "fromindex", "moz_historyvisits", "from_visit", "" \ ) -#define CREATE_IDX_MOZ_HISTORYVISITS_TEMP_VISITDATE \ - CREATE_PLACES_IDX( \ - "dateindex", "moz_historyvisits_temp", "visit_date", "" \ - ) #define CREATE_IDX_MOZ_HISTORYVISITS_VISITDATE \ CREATE_PLACES_IDX( \ "dateindex", "moz_historyvisits", "visit_date", "" \ diff --git a/toolkit/components/places/src/nsPlacesTables.h b/toolkit/components/places/src/nsPlacesTables.h index e9f550fd23c..b80549e2177 100644 --- a/toolkit/components/places/src/nsPlacesTables.h +++ b/toolkit/components/places/src/nsPlacesTables.h @@ -40,8 +40,9 @@ #ifndef __nsPlacesTables_h__ #define __nsPlacesTables_h__ -#define CREATE_MOZ_PLACES_BASE(__name, __temporary) NS_LITERAL_CSTRING( \ - "CREATE " __temporary " TABLE " __name " ( " \ + +#define CREATE_MOZ_PLACES NS_LITERAL_CSTRING( \ + "CREATE TABLE moz_places ( " \ " id INTEGER PRIMARY KEY" \ ", url LONGVARCHAR" \ ", title LONGVARCHAR" \ @@ -54,22 +55,12 @@ ", last_visit_date INTEGER " \ ")" \ ) -#define CREATE_MOZ_PLACES CREATE_MOZ_PLACES_BASE("moz_places", "") -#define CREATE_MOZ_PLACES_TEMP CREATE_MOZ_PLACES_BASE("moz_places_temp", "TEMP") #define MOZ_PLACES_COLUMNS \ "id, url, title, rev_host, visit_count, hidden, typed, favicon_id, " \ "frecency, last_visit_date" -#define CREATE_MOZ_PLACES_VIEW NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY VIEW moz_places_view AS " \ - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places_temp " \ - "UNION ALL " \ - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places " \ - "WHERE id NOT IN (SELECT id FROM moz_places_temp) " \ -) - -#define CREATE_MOZ_HISTORYVISITS_BASE(__name, __temporary) NS_LITERAL_CSTRING( \ - "CREATE " __temporary " TABLE " __name " (" \ +#define CREATE_MOZ_HISTORYVISITS NS_LITERAL_CSTRING( \ + "CREATE TABLE moz_historyvisits (" \ " id INTEGER PRIMARY KEY" \ ", from_visit INTEGER" \ ", place_id INTEGER" \ @@ -78,19 +69,9 @@ ", session INTEGER" \ ")" \ ) -#define CREATE_MOZ_HISTORYVISITS \ - CREATE_MOZ_HISTORYVISITS_BASE("moz_historyvisits", "") -#define CREATE_MOZ_HISTORYVISITS_TEMP \ - CREATE_MOZ_HISTORYVISITS_BASE("moz_historyvisits_temp", "TEMP") #define MOZ_HISTORYVISITS_COLUMNS \ "id, from_visit, place_id, visit_date, visit_type, session" -#define CREATE_MOZ_HISTORYVISITS_VIEW NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY VIEW moz_historyvisits_view AS " \ - "SELECT " MOZ_HISTORYVISITS_COLUMNS " FROM moz_historyvisits_temp " \ - "UNION ALL " \ - "SELECT " MOZ_HISTORYVISITS_COLUMNS " FROM moz_historyvisits " \ - "WHERE id NOT IN (SELECT id FROM moz_historyvisits_temp) " \ -) + #define CREATE_MOZ_INPUTHISTORY NS_LITERAL_CSTRING( \ "CREATE TABLE moz_inputhistory (" \ diff --git a/toolkit/components/places/src/nsPlacesTriggers.h b/toolkit/components/places/src/nsPlacesTriggers.h index e9b2c939220..af13b863c20 100644 --- a/toolkit/components/places/src/nsPlacesTriggers.h +++ b/toolkit/components/places/src/nsPlacesTriggers.h @@ -73,198 +73,33 @@ ) /** - * This trigger allows for an insertion into moz_places_view. It enters the new - * data into the temporary table, ensuring that the new id is one greater than - * the largest id value found. - * We have both sync and async users for this trigger, so it could happen that - * an async statement tries to insert a page when a sync statement just added - * it. We should ignore the insertion in such a case, the async implementer - * will fetch the id of the existing entry. + * This triggers update visit_count and last_visit_date based on historyvisits + * table changes. */ -#define CREATE_PLACES_VIEW_INSERT_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_places_view_insert_trigger " \ - "INSTEAD OF INSERT " \ - "ON moz_places_view " \ +#define CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER NS_LITERAL_CSTRING( \ + "CREATE TRIGGER moz_historyvisits_afterinsert_v2_trigger " \ + "AFTER INSERT ON moz_historyvisits FOR EACH ROW " \ "BEGIN " \ - "INSERT OR IGNORE INTO moz_places_temp (" MOZ_PLACES_COLUMNS ") " \ - "VALUES (MAX(IFNULL((SELECT MAX(id) FROM moz_places_temp), 0), " \ - "IFNULL((SELECT MAX(id) FROM moz_places), 0)) + 1," \ - "NEW.url, NEW.title, NEW.rev_host, " \ - "IFNULL(NEW.visit_count, 0), " /* enforce having a value */ \ - "NEW.hidden, NEW.typed, NEW.favicon_id, NEW.frecency, " \ - "NEW.last_visit_date);" \ - "END" \ -) - -/** - * This trigger allows for the deletion of a record in moz_places_view. It - * removes any entry in the temporary table, the permanent table, and any - * associated entry in moz_openpages_temp. - */ -#define CREATE_PLACES_VIEW_DELETE_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_places_view_delete_trigger " \ - "INSTEAD OF DELETE " \ - "ON moz_places_view " \ - "BEGIN " \ - "DELETE FROM moz_places_temp " \ - "WHERE id = OLD.id; " \ - "DELETE FROM moz_places " \ - "WHERE id = OLD.id; " \ - "DELETE FROM moz_openpages_temp " \ - "WHERE url = OLD.url; " \ - "END" \ -) - -/** - * This trigger allows for updates to a record in moz_places_view. It first - * copies the row from the permanent table over to the temp table if it does not - * exist in the temporary table. Then, it will update the temporary table with - * the new data. - * We use INSERT OR IGNORE to avoid looking if the place already exists in the - * temp table. - */ -#define CREATE_PLACES_VIEW_UPDATE_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_places_view_update_trigger " \ - "INSTEAD OF UPDATE " \ - "ON moz_places_view " \ - "BEGIN " \ - "INSERT OR IGNORE INTO moz_places_temp (" MOZ_PLACES_COLUMNS ") " \ - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places " \ - "WHERE id = OLD.id; " \ - "UPDATE moz_places_temp " \ - "SET url = IFNULL(NEW.url, OLD.url), " \ - "title = IFNULL(NEW.title, OLD.title), " \ - "rev_host = IFNULL(NEW.rev_host, OLD.rev_host), " \ - "visit_count = IFNULL(NEW.visit_count, OLD.visit_count), " \ - "hidden = IFNULL(NEW.hidden, OLD.hidden), " \ - "typed = IFNULL(NEW.typed, OLD.typed), " \ - "favicon_id = IFNULL(NEW.favicon_id, OLD.favicon_id), " \ - "frecency = IFNULL(NEW.frecency, OLD.frecency), " \ - "last_visit_date = IFNULL(NEW.last_visit_date, OLD.last_visit_date) " \ - "WHERE id = OLD.id; " \ - "END" \ -) - -/** - * This trigger allows for an insertion into moz_historyvisits_view. It enters - * the new data into the temporary table, ensuring that the new id is one - * greater than the largest id value found. It then updates moz_places_view - * with the new visit count. - * We use INSERT OR IGNORE to avoid looking if the place already exists in the - * temp table. - * In this case when updating last_visit_date we can simply check the maximum - * between new visit date and the actual cached value (or 0 if it is NULL). - */ -#define CREATE_HISTORYVISITS_VIEW_INSERT_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_historyvisits_view_insert_trigger " \ - "INSTEAD OF INSERT " \ - "ON moz_historyvisits_view " \ - "BEGIN " \ - "INSERT INTO moz_historyvisits_temp (" MOZ_HISTORYVISITS_COLUMNS ") " \ - "VALUES (MAX(IFNULL((SELECT MAX(id) FROM moz_historyvisits_temp), 0), " \ - "IFNULL((SELECT MAX(id) FROM moz_historyvisits), 0)) + 1, " \ - "NEW.from_visit, NEW.place_id, NEW.visit_date, NEW.visit_type, " \ - "NEW.session); " \ - "INSERT OR IGNORE INTO moz_places_temp (" MOZ_PLACES_COLUMNS ") " \ - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places " \ - "WHERE id = NEW.place_id " \ - "AND NEW.visit_type NOT IN (" EXCLUDED_VISIT_TYPES "); " \ - "UPDATE moz_places_temp " \ - "SET visit_count = visit_count + 1 " \ - "WHERE id = NEW.place_id " \ - "AND NEW.visit_type NOT IN (" EXCLUDED_VISIT_TYPES "); " \ - "UPDATE moz_places_temp " \ - "SET last_visit_date = MAX(IFNULL(last_visit_date, 0), NEW.visit_date)" \ + "UPDATE moz_places SET " \ + "visit_count = visit_count + (SELECT NEW.visit_type NOT IN (" EXCLUDED_VISIT_TYPES ")), "\ + "last_visit_date = MAX(IFNULL(last_visit_date, 0), NEW.visit_date) " \ "WHERE id = NEW.place_id;" \ "END" \ ) -/** - * This trigger allows for the deletion of a record in moz_historyvisits_view. - * It removes any entry in the temporary table, and removes any entry in the - * permanent table as well. It then updates moz_places_view with the new visit - * count. - * We use INSERT OR IGNORE to avoid looking if the place already exists in the - * temp table. - */ -#define CREATE_HISTORYVISITS_VIEW_DELETE_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_historyvisits_view_delete_trigger " \ - "INSTEAD OF DELETE " \ - "ON moz_historyvisits_view " \ +#define CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER NS_LITERAL_CSTRING( \ + "CREATE TRIGGER moz_historyvisits_afterdelete_v2_trigger " \ + "AFTER DELETE ON moz_historyvisits FOR EACH ROW " \ "BEGIN " \ - "DELETE FROM moz_historyvisits_temp " \ - "WHERE id = OLD.id; " \ - "DELETE FROM moz_historyvisits " \ - "WHERE id = OLD.id; " \ - "INSERT OR IGNORE INTO moz_places_temp (" MOZ_PLACES_COLUMNS ") " \ - "SELECT " MOZ_PLACES_COLUMNS " FROM moz_places " \ - "WHERE id = OLD.place_id " \ - "AND OLD.visit_type NOT IN (" EXCLUDED_VISIT_TYPES "); " \ - "UPDATE moz_places_temp " \ - "SET visit_count = visit_count - 1 " \ - "WHERE id = OLD.place_id " \ - "AND OLD.visit_type NOT IN (" EXCLUDED_VISIT_TYPES "); " \ - "UPDATE moz_places_temp " \ - "SET last_visit_date = " \ - "(SELECT visit_date FROM moz_historyvisits_temp " \ - "WHERE place_id = OLD.place_id " \ - "UNION ALL " \ - "SELECT visit_date FROM moz_historyvisits " \ - "WHERE place_id = OLD.place_id " \ - "ORDER BY visit_date DESC LIMIT 1) " \ - "WHERE id = OLD.place_id; " \ + "UPDATE moz_places SET " \ + "visit_count = visit_count - (SELECT OLD.visit_type NOT IN (" EXCLUDED_VISIT_TYPES ")), "\ + "last_visit_date = (SELECT visit_date FROM moz_historyvisits " \ + "WHERE place_id = OLD.place_id " \ + "ORDER BY visit_date DESC LIMIT 1) " \ + "WHERE id = OLD.place_id;" \ "END" \ ) -/** - * This trigger allows for updates to a record in moz_historyvisits_view. It - * first copies the row from the permanent table over to the temp table if it - * does not exist in the temporary table. Then it will update the temporary - * table with the new data. - * We use INSERT OR IGNORE to avoid looking if the visit already exists in the - * temp table. - */ -#define CREATE_HISTORYVISITS_VIEW_UPDATE_TRIGGER NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER moz_historyvisits_view_update_trigger " \ - "INSTEAD OF UPDATE " \ - "ON moz_historyvisits_view " \ - "BEGIN " \ - "INSERT OR IGNORE INTO moz_historyvisits_temp (" MOZ_HISTORYVISITS_COLUMNS ") " \ - "SELECT " MOZ_HISTORYVISITS_COLUMNS " FROM moz_historyvisits " \ - "WHERE id = OLD.id; " \ - "UPDATE moz_historyvisits_temp " \ - "SET from_visit = IFNULL(NEW.from_visit, OLD.from_visit), " \ - "place_id = IFNULL(NEW.place_id, OLD.place_id), " \ - "visit_date = IFNULL(NEW.visit_date, OLD.visit_date), " \ - "visit_type = IFNULL(NEW.visit_type, OLD.visit_type), " \ - "session = IFNULL(NEW.session, OLD.session) " \ - "WHERE id = OLD.id; " \ - "END" \ -) - -/** - * This trigger moves the data out of a temporary table into the permanent one - * before deleting from the temporary table. - * - * Note - it's OK to use an INSERT OR REPLACE here because the only conflict - * that will happen is the primary key. As a result, the row will be deleted, - * and the replacement will be inserted with the same id. - */ -#define CREATE_TEMP_SYNC_TRIGGER_BASE(__table, __columns) NS_LITERAL_CSTRING( \ - "CREATE TEMPORARY TRIGGER " __table "_beforedelete_trigger " \ - "BEFORE DELETE ON " __table "_temp FOR EACH ROW " \ - "BEGIN " \ - "INSERT OR REPLACE INTO " __table " (" __columns ") " \ - "SELECT " __columns " FROM " __table "_temp " \ - "WHERE id = OLD.id;" \ - "END" \ -) -#define CREATE_MOZ_PLACES_SYNC_TRIGGER \ - CREATE_TEMP_SYNC_TRIGGER_BASE("moz_places", MOZ_PLACES_COLUMNS) -#define CREATE_MOZ_HISTORYVISITS_SYNC_TRIGGER \ - CREATE_TEMP_SYNC_TRIGGER_BASE("moz_historyvisits", MOZ_HISTORYVISITS_COLUMNS) - - /** * This trigger removes a row from moz_openpages_temp when open_count reaches 0. */ diff --git a/toolkit/components/places/src/toolkitplaces.manifest b/toolkit/components/places/src/toolkitplaces.manifest index 4c0cd46070b..5882bf3e6a4 100644 --- a/toolkit/components/places/src/toolkitplaces.manifest +++ b/toolkit/components/places/src/toolkitplaces.manifest @@ -8,12 +8,6 @@ contract @mozilla.org/browser/tagging-service;1 {bbc23860-2553-479d-8b78-94d9038 component {1dcc23b0-d4cb-11dc-9ad6-479d56d89593} nsTaggingService.js contract @mozilla.org/autocomplete/search;1?name=places-tag-autocomplete {1dcc23b0-d4cb-11dc-9ad6-479d56d89593} -# nsPlacesDBFlush.js -component {c1751cfc-e8f1-4ade-b0bb-f74edfb8ef6a} nsPlacesDBFlush.js -contract @mozilla.org/places/sync;1 {c1751cfc-e8f1-4ade-b0bb-f74edfb8ef6a} -category bookmark-observers nsPlacesDBFlush @mozilla.org/places/sync;1 -category history-observers nsPlacesDBFlush @mozilla.org/places/sync;1 - # nsPlacesExpiration.js component {705a423f-2f69-42f3-b9fe-1517e0dee56f} nsPlacesExpiration.js contract @mozilla.org/places/expiration;1 {705a423f-2f69-42f3-b9fe-1517e0dee56f} diff --git a/toolkit/components/places/tests/Makefile.in b/toolkit/components/places/tests/Makefile.in index 949d33a61e3..9531dbfdeee 100644 --- a/toolkit/components/places/tests/Makefile.in +++ b/toolkit/components/places/tests/Makefile.in @@ -49,7 +49,6 @@ MODULE = test_places XPCSHELL_TESTS = \ autocomplete \ expiration \ - sync \ bookmarks \ queries \ unit \ diff --git a/toolkit/components/places/tests/autocomplete/test_autocomplete_on_value_removed_479089.js b/toolkit/components/places/tests/autocomplete/test_autocomplete_on_value_removed_479089.js index ad63cecabb2..7c6ade89f64 100644 --- a/toolkit/components/places/tests/autocomplete/test_autocomplete_on_value_removed_479089.js +++ b/toolkit/components/places/tests/autocomplete/test_autocomplete_on_value_removed_479089.js @@ -72,7 +72,7 @@ function run_test() // open the result container queryRes.root.containerOpen = true; // debug queries - // dump_table("moz_places_view"); + // dump_table("moz_places"); do_check_eq(queryRes.root.childCount, 1); // call the untested code path listener.onValueRemoved(null, uri.spec, true); diff --git a/toolkit/components/places/tests/browser/browser_settitle.js b/toolkit/components/places/tests/browser/browser_settitle.js index 76e56b88bf9..ae38051a732 100644 --- a/toolkit/components/places/tests/browser/browser_settitle.js +++ b/toolkit/components/places/tests/browser/browser_settitle.js @@ -36,8 +36,6 @@ var conn = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnectio function getColumn(table, column, fromColumnName, fromColumnValue) { var stmt = conn.createStatement( - "SELECT " + column + " FROM " + table + "_temp WHERE " + fromColumnName + "=:val " + - "UNION ALL " + "SELECT " + column + " FROM " + table + " WHERE " + fromColumnName + "=:val " + "LIMIT 1"); try { diff --git a/toolkit/components/places/tests/browser/browser_visituri.js b/toolkit/components/places/tests/browser/browser_visituri.js index e21bb59aa0f..db6189535bd 100644 --- a/toolkit/components/places/tests/browser/browser_visituri.js +++ b/toolkit/components/places/tests/browser/browser_visituri.js @@ -51,8 +51,6 @@ var conn = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnectio function getColumn(table, column, fromColumnName, fromColumnValue) { var stmt = conn.createStatement( - "SELECT " + column + " FROM " + table + "_temp WHERE " + fromColumnName + "=:val " + - "UNION ALL " + "SELECT " + column + " FROM " + table + " WHERE " + fromColumnName + "=:val " + "LIMIT 1"); try { diff --git a/toolkit/components/places/tests/cpp/places_test_harness.h b/toolkit/components/places/tests/cpp/places_test_harness.h index 389cda76105..2d2114cefcc 100644 --- a/toolkit/components/places/tests/cpp/places_test_harness.h +++ b/toolkit/components/places/tests/cpp/places_test_harness.h @@ -182,16 +182,12 @@ do_get_place(nsIURI* aURI, PlaceRecord& result) do_check_success(rv); rv = dbConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, hidden, typed, visit_count FROM moz_places_temp " - "WHERE url=?1 " - "UNION ALL " "SELECT id, hidden, typed, visit_count FROM moz_places " "WHERE url=?1 " - "LIMIT 1" ), getter_AddRefs(stmt)); do_check_success(rv); - rv = stmt->BindUTF8StringParameter(0, spec); + rv = stmt->BindUTF8StringByIndex(0, spec); do_check_success(rv); PRBool hasResults; @@ -222,16 +218,13 @@ do_get_lastVisit(PRInt64 placeId, VisitRecord& result) nsCOMPtr stmt; nsresult rv = dbConn->CreateStatement(NS_LITERAL_CSTRING( - "SELECT id, from_visit, visit_type FROM moz_historyvisits_temp " - "WHERE place_id=?1 " - "UNION ALL " "SELECT id, from_visit, visit_type FROM moz_historyvisits " "WHERE place_id=?1 " "LIMIT 1" ), getter_AddRefs(stmt)); do_check_success(rv); - rv = stmt->BindInt64Parameter(0, placeId); + rv = stmt->BindInt64ByIndex(0, placeId); do_check_success(rv); PRBool hasResults; diff --git a/toolkit/components/places/tests/expiration/test_annos_expire_policy.js b/toolkit/components/places/tests/expiration/test_annos_expire_policy.js index 695a9382286..76385d43dec 100644 --- a/toolkit/components/places/tests/expiration/test_annos_expire_policy.js +++ b/toolkit/components/places/tests/expiration/test_annos_expire_policy.js @@ -90,7 +90,7 @@ function add_old_anno(aIdentifier, aName, aValue, aExpirePolicy, // Update dateAdded for the last added annotation. sql = "UPDATE moz_annos SET dateAdded = :expire_date, lastModified = :last_modified " + "WHERE id = (SELECT a.id FROM moz_annos a " + - "LEFT JOIN moz_places_view h on h.id = a.place_id " + + "LEFT JOIN moz_places h on h.id = a.place_id " + "WHERE h.url = :id " + "ORDER BY a.dateAdded DESC LIMIT 1)"; } diff --git a/toolkit/components/places/tests/expiration/test_removeAllPages.js b/toolkit/components/places/tests/expiration/test_removeAllPages.js index c2b84395d4a..8cd94f78629 100644 --- a/toolkit/components/places/tests/expiration/test_removeAllPages.js +++ b/toolkit/components/places/tests/expiration/test_removeAllPages.js @@ -88,7 +88,7 @@ function add_old_anno(aIdentifier, aName, aValue, aExpirePolicy, "WHERE id = ( " + "SELECT a.id FROM moz_annos a " + "JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id " + - "JOIN moz_places_view h on h.id = a.place_id " + + "JOIN moz_places h on h.id = a.place_id " + "WHERE h.url = :id " + "AND n.name = :anno_name " + "ORDER BY a.dateAdded DESC LIMIT 1 " + diff --git a/toolkit/components/places/tests/head_common.js b/toolkit/components/places/tests/head_common.js index 8e5a5866e20..ccfc6edab20 100644 --- a/toolkit/components/places/tests/head_common.js +++ b/toolkit/components/places/tests/head_common.js @@ -257,7 +257,7 @@ function dump_table(aName) function page_in_database(aUrl) { let stmt = DBConn().createStatement( - "SELECT id FROM moz_places_view WHERE url = :url" + "SELECT id FROM moz_places WHERE url = :url" ); stmt.params.url = aUrl; try { @@ -459,6 +459,57 @@ function check_JSON_backup() { } +/** + * Waits for a frecency update then calls back. + * + * @param aUrl + * Address of the page we are waiting frecency for. + * @param aValidator + * Validator function for the current frecency. If it returns true we + * have the expected frecency, otherwise we wait for next update. + * @param aCallback + * function invoked when frecency update finishes. + * @param aCbScope + * "this" scope for the callback + * @param aCbArguments + * array of arguments to be passed to the callback + * + * @note since frecency is something that can be changed by a bunch of stuff + * like adding and removing visits, bookmarks we use a polling strategy. + */ +function waitForFrecency(aUrl, aValidator, aCallback, aCbScope, aCbArguments) { + Services.obs.addObserver(function (aSubject, aTopic, aData) { + let frecency = frecencyForUrl(aUrl); + if (!aValidator(frecency)) { + return; + } + Services.obs.removeObserver(arguments.callee, aTopic); + aCallback.apply(aCbScope, aCbArguments); + }, "places-frecency-updated", false); +} + +/** + * Returns the frecency of a url. + * + * @param aURL + * the URL to get frecency for. + * @return the frecency value. + */ +function frecencyForUrl(aUrl) +{ + let stmt = DBConn().createStatement( + "SELECT frecency FROM moz_places WHERE url = ?1" + ); + stmt.bindUTF8StringParameter(0, aUrl); + if (!stmt.executeStep()) + throw "No result for frecency."; + let frecency = stmt.getInt32(0); + stmt.finalize(); + + return frecency; +} + + /** * Compares two times in usecs, considering eventual platform timers skews. * @@ -478,33 +529,3 @@ function is_time_ordered(before, after) { return after - before > -skew; } - -// These tests are known to randomly fail due to bug 507790 when database -// flushes are active, so we turn off syncing for them. -let (randomFailingSyncTests = [ - "test_multi_word_tags.js", - "test_removeVisitsByTimeframe.js", - "test_utils_getURLsForContainerNode.js", - "test_exclude_livemarks.js", - "test_402799.js", - "test_results-as-visit.js", - "test_sorting.js", - "test_redirectsMode.js", - "test_384228.js", - "test_395593.js", - "test_containersQueries_sorting.js", - "test_browserGlue_smartBookmarks.js", - "test_browserGlue_distribution.js", - "test_331487.js", - "test_tags.js", - "test_385829.js", - "test_405938_restore_queries.js", -]) { - let currentTestFilename = do_get_file(_TEST_FILE[0], true).leafName; - if (randomFailingSyncTests.indexOf(currentTestFilename) != -1) { - print("Test " + currentTestFilename + - " is known random due to bug 507790, disabling PlacesDBFlush."); - let sync = Cc["@mozilla.org/places/sync;1"].getService(Ci.nsIObserver); - sync.observe(null, "places-debug-stop-sync", null); - } -} diff --git a/toolkit/components/places/tests/mochitest/bug_411966/redirect.js b/toolkit/components/places/tests/mochitest/bug_411966/redirect.js index f141db08f3f..4a05d1c528a 100644 --- a/toolkit/components/places/tests/mochitest/bug_411966/redirect.js +++ b/toolkit/components/places/tests/mochitest/bug_411966/redirect.js @@ -158,11 +158,11 @@ function checkDB(data){ ghist.addURI(this.mChannel.URI, true, true, referrer); // Get all pages visited from the original typed one - var sql = "SELECT url FROM moz_historyvisits_view " + - "JOIN moz_places_view h ON h.id = place_id " + + var sql = "SELECT url FROM moz_historyvisits " + + "JOIN moz_places h ON h.id = place_id " + "WHERE from_visit IN " + - "(SELECT v.id FROM moz_historyvisits_view v " + - "JOIN moz_places_view p ON p.id = v.place_id " + + "(SELECT v.id FROM moz_historyvisits v " + + "JOIN moz_places p ON p.id = v.place_id " + "WHERE p.url = ?1)"; var stmt = mDBConn.createStatement(sql); stmt.bindUTF8StringParameter(0, typedURI.spec); diff --git a/toolkit/components/places/tests/network/test_history_redirects.js b/toolkit/components/places/tests/network/test_history_redirects.js index 016b4023dd6..0bdbc87b909 100644 --- a/toolkit/components/places/tests/network/test_history_redirects.js +++ b/toolkit/components/places/tests/network/test_history_redirects.js @@ -109,8 +109,8 @@ function run_test() { function continue_test() { let stmt = DBConn().createStatement( "SELECT v.id, h.url, v.from_visit, v.visit_date, v.visit_type, v.session " + - "FROM moz_historyvisits_view v " + - "JOIN moz_places_view h on h.id = v.place_id " + + "FROM moz_historyvisits v " + + "JOIN moz_places h on h.id = v.place_id " + "ORDER BY v.id ASC"); const EXPECTED = [ { id: 1, diff --git a/toolkit/components/places/tests/queries/head_queries.js b/toolkit/components/places/tests/queries/head_queries.js index eef43fa89cf..18bc7dbc979 100644 --- a/toolkit/components/places/tests/queries/head_queries.js +++ b/toolkit/components/places/tests/queries/head_queries.js @@ -85,7 +85,7 @@ function populateDB(aArray) { if (qdata.title && !qdata.isDetails) { // Set the page title synchronously, otherwise setPageTitle is LAZY. let stmt = DBConn().createStatement( - "UPDATE moz_places_view SET title = :title WHERE url = :url" + "UPDATE moz_places SET title = :title WHERE url = :url" ); stmt.params.title = qdata.title; stmt.params.url = qdata.uri; @@ -102,7 +102,7 @@ function populateDB(aArray) { // Set a fake visit_count, this is not a real count but can be used // to test sorting by visit_count. let stmt = DBConn().createStatement( - "UPDATE moz_places_view SET visit_count = :vc WHERE url = :url"); + "UPDATE moz_places SET visit_count = :vc WHERE url = :url"); stmt.params.vc = qdata.visitCount; stmt.params.url = qdata.uri; try { diff --git a/toolkit/components/places/tests/queries/test_transitions.js b/toolkit/components/places/tests/queries/test_transitions.js index f541693432c..e9f788928ce 100644 --- a/toolkit/components/places/tests/queries/test_transitions.js +++ b/toolkit/components/places/tests/queries/test_transitions.js @@ -100,10 +100,8 @@ function run_test() { testData[i].title = null; } - dump_table("moz_places"); - dump_table("moz_places_temp"); - dump_table("moz_historyvisits"); - dump_table("moz_historyvisits_temp"); + //dump_table("moz_places"); + //dump_table("moz_historyvisits"); var numSortFunc = function (a,b) { return (a - b); }; var arrs = testDataTyped.concat(testDataDownload).concat(testDataBookmark) diff --git a/toolkit/components/places/tests/sync/head_sync.js b/toolkit/components/places/tests/sync/head_sync.js deleted file mode 100644 index ceeb6211163..00000000000 --- a/toolkit/components/places/tests/sync/head_sync.js +++ /dev/null @@ -1,136 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is Places. - * - * The Initial Developer of the Original Code is - * Google Inc. - * Portions created by the Initial Developer are Copyright (C) 2005 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Brian Ryner - * Dietrich Ayala - * Shawn Wilsher - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -const Ci = Components.interfaces; -const Cc = Components.classes; -const Cr = Components.results; -const Cu = Components.utils; - -Cu.import("resource://gre/modules/Services.jsm"); - -// Import common head. -let (commonFile = do_get_file("../head_common.js", false)) { - let uri = Services.io.newFileURI(commonFile); - Services.scriptloader.loadSubScript(uri.spec, this); -} - -// Put any other stuff relative to this test folder below. - - -/** - * Function tests to see if the place associated with the bookmark with id - * aBookmarkId has the uri aExpectedURI. The event will call do_test_finished() - * if aFinish is true. - * - * @param aBookmarkId - * The bookmark to check against. - * @param aExpectedURI - * The URI we expect to be in moz_places. - * @param aExpected - * Indicates if we expect to get a result or not. - * @param [optional] aFinish - * Indicates if the test should be completed or not. - */ -function new_test_bookmark_uri_event(aBookmarkId, aExpectedURI, aExpected, aFinish) -{ - let stmt = DBConn().createStatement( - "SELECT moz_places.url " + - "FROM moz_bookmarks INNER JOIN moz_places " + - "ON moz_bookmarks.fk = moz_places.id " + - "WHERE moz_bookmarks.id = ?1" - ); - stmt.bindInt64Parameter(0, aBookmarkId); - - if (aExpected) { - do_check_true(stmt.executeStep()); - do_check_eq(stmt.getUTF8String(0), aExpectedURI); - } - else { - do_check_false(stmt.executeStep()); - } - stmt.reset(); - stmt.finalize(); - stmt = null; - - if (aFinish) - do_test_finished(); -} - - -/** - * Function tests to see if the place associated with the visit with id aVisitId - * has the uri aExpectedURI. The event will call do_test_finished() if aFinish is - * true. - * - * @param aVisitId - * The visit to check against. - * @param aExpectedURI - * The URI we expect to be in moz_places. - * @param aExpected - * Indicates if we expect to get a result or not. - * @param [optional] aFinish - * Indicates if the test should be completed or not. - */ -function new_test_visit_uri_event(aVisitId, aExpectedURI, aExpected, aFinish) -{ - let stmt = DBConn().createStatement( - "SELECT moz_places.url " + - "FROM moz_historyvisits INNER JOIN moz_places " + - "ON moz_historyvisits.place_id = moz_places.id " + - "WHERE moz_historyvisits.id = ?1" - ); - stmt.bindInt64Parameter(0, aVisitId); - - if (aExpected) { - do_check_true(stmt.executeStep()); - do_check_eq(stmt.getUTF8String(0), aExpectedURI); - } - else { - do_check_false(stmt.executeStep()); - } - stmt.reset(); - stmt.finalize(); - stmt = null; - - if (aFinish) - do_test_finished(); -} - diff --git a/toolkit/components/places/tests/sync/test_bookmarks_sorted_by_none.js b/toolkit/components/places/tests/sync/test_bookmarks_sorted_by_none.js deleted file mode 100644 index 9e1e030e356..00000000000 --- a/toolkit/components/places/tests/sync/test_bookmarks_sorted_by_none.js +++ /dev/null @@ -1,126 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2009 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/** - * Bug 493933 - When sorting bookmarks BY NONE we should take in count partitioning. - * - * When we have a bookmarks results sorted BY NONE, could be the order depends - * on current status of disk and temp tables due to how the query executes the - * UNION. We must ensure their order is correct. - */ - -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; -const kSyncFinished = "places-sync-finished"; - -var observer = { - _runCount: 0, - one: null, - two: null, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - // Skip the first sync. - if (++this._runCount < 2) - return; - - os.removeObserver(this, kSyncFinished); - - // Sanity check positions. - do_check_true(this.one < this.two); - do_check_eq(bs.getItemIndex(this.one), 0); - do_check_eq(bs.getItemIndex(this.two), 1); - - check_results(); - - // Now add a visit to the second bookmark, so that it is in temp table. - hs.addVisit(uri("http://2.mozilla.org/"), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - - // Nothing should change in results. - check_results(); - - // Cleanup. - bs.removeFolderChildren(bs.toolbarFolder); - do_test_finished(); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function check_results() { - // Create a bookmarks query. - var options = hs.getNewQueryOptions(); - options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS; - var query = hs.getNewQuery(); - query.setFolders([bs.toolbarFolder], 1); - query.searchTerms = "mozilla"; - var result = hs.executeQuery(query, options); - var root = result.root; - root.containerOpen = true; - do_check_eq(root.childCount, 2); - do_check_eq(root.getChild(0).title, "mozilla 1"); - do_check_eq(root.getChild(1).title, "mozilla 2"); - root.containerOpen = false; -} - -function run_test() -{ - // First set the preference for the timer to a large value so we don't sync - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add two bookmarks and check their index - observer.one = bs.insertBookmark(bs.toolbarFolder, - uri("http://1.mozilla.org/"), - bs.DEFAULT_INDEX, "mozilla 1"); - observer.two = bs.insertBookmark(bs.toolbarFolder, - uri("http://2.mozilla.org/"), - bs.DEFAULT_INDEX, "mozilla 2"); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark.js b/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark.js deleted file mode 100644 index 85033c5f683..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark.js +++ /dev/null @@ -1,89 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); - -const TEST_URI = "http://test.com/"; - -const SYNC_INTERVAL = 600; // ten minutes -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const kSyncFinished = "places-sync-finished"; - -// Used to update observer itemId -var bookmarksObserver = { - onItemAdded: function(aItemId, aNewParent, aNewIndex) { - observer.itemId = aItemId; - } -} -bs.addObserver(bookmarksObserver, false); - -var observer = { - itemId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - do_check_neq(this.itemId, -1); - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver); - // Check that moz_places table has been correctly synced - new_test_bookmark_uri_event(this.itemId, TEST_URI, true, true); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // First set the preference for the timer to a really large value so it won't - // run before the test finishes. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Insert a new bookmark - bs.insertBookmark(bs.unfiledBookmarksFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "test"); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark_batched.js b/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark_batched.js deleted file mode 100644 index f95227bd01b..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_addBookmark_batched.js +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); - -const TEST_URI = "http://test.com/"; - -const SYNC_INTERVAL = 600; // ten minutes -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const kSyncFinished = "places-sync-finished"; - -// Used to check if we are batching and to update observer itemId -var bookmarksObserver = { - _batching: false, - onBeginUpdateBatch: function() { - this._batching = true; - }, - onEndUpdateBatch: function() { - this._batching = false; - }, - onItemAdded: function(aItemId, aNewParent, aNewIndex) { - observer.itemId = aItemId; - } -} -bs.addObserver(bookmarksObserver, false); - -var observer = { - itemId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - dump(this.itemId); - // item id must be valid - do_check_neq(this.itemId, -1); - // Check that we are not in a batch - do_check_false(bookmarksObserver._batching); - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver); - // Check that tables have been correctly synced - new_test_bookmark_uri_event(this.itemId, TEST_URI, true, true); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // Set the preference for the timer to a really large value, so it won't - // run before the test finishes. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add a bookmark in batch mode - let id = -1; - bs.runInBatchMode({ - runBatched: function(aUserData) - { - id = bs.insertBookmark(bs.unfiledBookmarksFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "test"); - // We should not sync during a batch - new_test_bookmark_uri_event(id, TEST_URI, false); - } - }, null); - // Ensure the bookmark has been added - do_check_neq(id, -1); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_addVisit.js b/toolkit/components/places/tests/sync/test_database_sync_after_addVisit.js deleted file mode 100644 index a100df86ef0..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_addVisit.js +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 1; -const kSyncFinished = "places-sync-finished"; - -var observer = { - visitId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished && this.visitId != -1) { - // sanity check: visitId set by history observer should be the same we - // have added - do_check_eq(this.visitId, visitId); - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - // Check the visit - new_test_visit_uri_event(this.visitId, TEST_URI, true, true); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -// Used to update observer visitId -var historyObserver = { - onVisit: function(aURI, aVisitId, aTime, aSessionId, aReferringId, - aTransitionType, aAdded) { - observer.visitId = aVisitId; - hs.removeObserver(this, false); - } -} -hs.addObserver(historyObserver, false); - -// used for sanity check -var visitId = -1; - -function run_test() -{ - // First set the preference for the timer to a small value - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Now add the visit - visitId = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_addVisit_batched.js b/toolkit/components/places/tests/sync/test_database_sync_after_addVisit_batched.js deleted file mode 100644 index 6c905559469..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_addVisit_batched.js +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - - var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); - var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); - var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; - -const kSyncFinished = "places-sync-finished"; - -// Used to check if we are batching -var bookmarksObserver = { - _batching: false, - onBeginUpdateBatch: function() { - this._batching = true; - }, - onEndUpdateBatch: function() { - this._batching = false; - } -} -bs.addObserver(bookmarksObserver, false); - -// Used to update observer visitId -var historyObserver = { - onVisit: function(aURI, aVisitId, aTime, aSessionId, aReferringId, - aTransitionType, aAdded) { - observer.visitId = aVisitId; - } -} -hs.addObserver(historyObserver, false); - -var observer = { - visitId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - // visit id must be valid - do_check_neq(this.visitId, -1); - // Check that we are not in a batch - do_check_false(bookmarksObserver._batching); - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver); - hs.removeObserver(historyObserver); - // Check that tables have been correctly synced - new_test_visit_uri_event(this.visitId, TEST_URI, true, true); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // Add a visit in batch mode - let id = -1; - bs.runInBatchMode({ - runBatched: function(aUserData) - { - id = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - // We should not sync during a batch - new_test_visit_uri_event(id, TEST_URI, false); - } - }, null); - // Ensure the visit has been added - do_check_neq(id, -1); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_modifyBookmark.js b/toolkit/components/places/tests/sync/test_database_sync_after_modifyBookmark.js deleted file mode 100644 index 0b5f456a683..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_modifyBookmark.js +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); - -const TEST_URI = "http://test.com/"; -const MODIFIED_URI = "http://test.com/index.html"; - -const SYNC_INTERVAL = 600; // ten minutes -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const kSyncFinished = "places-sync-finished"; - -// Used to update observer itemId -var bookmarksObserver = { - onItemAdded: function(aItemId, aNewParent, aNewIndex, aItemType) { - observer.itemId = aItemId; - }, - onItemChanged: function(aItemId, aProperty, aNewValue, aLastModified, - aItemType) { - if (aProperty == "uri") - do_check_eq(observer.itemId, aItemId); - } -} -bs.addObserver(bookmarksObserver, false); - -var observer = { - itemId: -1, - _runCount: 0, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - // item id must be valid - do_check_neq(this.itemId, -1); - if (++this._runCount == 1) { - // First sync is fired by adding the bookmark - // Check that tables have been synced after insertBookmark - new_test_bookmark_uri_event(this.itemId, TEST_URI, true); - // Now modify the bookmark - bs.changeBookmarkURI(this.itemId, uri(MODIFIED_URI)); - } - else if (this._runCount == 2) { - // Second sync is fired by changing the bookmark's uri - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver); - // Check that tables have been synced after changeBookmarkURI - new_test_bookmark_uri_event(this.itemId, MODIFIED_URI, true, true); - } - else - do_throw("Too many places sync calls"); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // Set the preference for the timer to a really large value, so it won't - // run before the test finishes. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Insert a new bookmark - bs.insertBookmark(bs.unfiledBookmarksFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "test"); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_shutdown.js b/toolkit/components/places/tests/sync/test_database_sync_after_shutdown.js deleted file mode 100644 index d19aa6e31c8..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_shutdown.js +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); - -const TEST_URI = "http://test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; // ten minutes -const TOPIC_CONNECTION_CLOSED = "places-connection-closed"; - -var historyObserver = { - onVisit: function(aURI, aVisitId, aTime, aSessionId, aReferringId, - aTransitionType, aAdded) { - observer.visitId = aVisitId; - hs.removeObserver(this); - } -} -hs.addObserver(historyObserver, false); - -var observer = { - visitId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == TOPIC_CONNECTION_CLOSED) { - os.removeObserver(this, aTopic); - - // visit id must be valid - do_check_neq(this.visitId, -1); - // Check that tables have been correctly synced - new_test_visit_uri_event(this.visitId, TEST_URI, true, true); - } - } -} -os.addObserver(observer, TOPIC_CONNECTION_CLOSED, false); - -function run_test() -{ - // Set the preference for the timer to a really large value, so it won't - // run before the test finishes. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Now add a visit - hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - - // Notify that we are quitting the app - we should sync! - shutdownPlaces(); - - // Test will continue on sync notification. - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_after_shutdown_with_removeAllPages.js b/toolkit/components/places/tests/sync/test_database_sync_after_shutdown_with_removeAllPages.js deleted file mode 100644 index 4c68a2325db..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_after_shutdown_with_removeAllPages.js +++ /dev/null @@ -1,152 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var bh = hs.QueryInterface(Ci.nsIBrowserHistory); -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); - -const TEST_URI = "http://test.com/"; - -const PREF_SYNC_INTERVAL = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; // ten minutes -const TOPIC_SYNC_FINISHED = "places-sync-finished"; -const TOPIC_CONNECTION_CLOSED = "places-connection-closed"; - -var historyObserver = { - visitId: -1, - cleared: false, - onVisit: function(aURI, aVisitId, aTime, aSessionId, aReferringId, - aTransitionType, aAdded) { - this.visitId = aVisitId; - }, - onClearHistory: function() { - // check browserHistory returns no entries - do_check_eq(0, bh.count); - this.cleared = true; - hs.removeObserver(this); - } -} -hs.addObserver(historyObserver, false); - -var observer = { - _runCount: 0, - observe: function(aSubject, aTopic, aData) { - if (aTopic == TOPIC_SYNC_FINISHED) { - if (++this._runCount == 1) { - // The first sync is due to the insert bookmark. - // Simulate a clear private data just before shutdown. - bh.removeAllPages(); - os.addObserver(shutdownObserver, TOPIC_CONNECTION_CLOSED, false); - // Immediately notify shutdown. - shutdownPlaces(); - return; - } - - // Remove the observer, we don't need it anymore. - os.removeObserver(this, TOPIC_SYNC_FINISHED); - - // Visit id must be valid. - do_check_neq(historyObserver.visitId, -1); - // History must have been cleared. - do_check_true(historyObserver.cleared); - } - } -} -os.addObserver(observer, TOPIC_SYNC_FINISHED, false); - -let shutdownObserver = { - observe: function(aSubject, aTopic, aData) { - os.removeObserver(this, aTopic); - do_check_false(PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase) - .DBConnection.connectionReady); - - let dbConn = DBConn(); - do_check_true(dbConn.connectionReady); - - // Check that frecency for not cleared items (bookmarks) has been - // converted to -MAX(visit_count, 1), so we will be able to - // recalculate frecency starting from most frecent bookmarks. - let stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE frecency > 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - stmt = dbConn.createStatement( - "SELECT h.id FROM moz_places h WHERE h.frecency = -2 " + - "AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1"); - do_check_true(stmt.executeStep()); - stmt.finalize(); - - // Check that all visit_counts have been brought to 0 - stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE visit_count <> 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - dbConn.asyncClose(function() { - do_check_false(dbConn.connectionReady); - - do_test_finished(); - }); - } -} - -function run_test() -{ - do_test_pending(); - - // Set the preference for the timer to a really large value, so it won't - // run before the test finishes. - prefs.setIntPref(PREF_SYNC_INTERVAL, SYNC_INTERVAL); - - // Now add a visit before creating bookmark, and one later - hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - bs.insertBookmark(bs.unfiledBookmarksFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "bookmark"); - hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_embed_visits.js b/toolkit/components/places/tests/sync/test_database_sync_embed_visits.js deleted file mode 100644 index 21720a9b321..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_embed_visits.js +++ /dev/null @@ -1,173 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - - /* - * This test checks that embed visits are not synced down to disk, we hold - * them in memory since they're going to be purged at session close - * - * We instead sync to disk FRAMED_LINK visits. - */ - -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var dbConn = hs.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection; -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; -const EMBED_URI = "http://embed.test.com/"; -const PLACE_URI = "place:test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 1; -const kSyncFinished = "places-sync-finished"; - -var transitions = [ hs.TRANSITION_LINK, - hs.TRANSITION_TYPED, - hs.TRANSITION_BOOKMARK, - hs.TRANSITION_EMBED, - hs.TRANSITION_FRAMED_LINK, - hs.TRANSITION_REDIRECT_PERMANENT, - hs.TRANSITION_REDIRECT_TEMPORARY, - hs.TRANSITION_DOWNLOAD ]; - -var observer = { - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished && this.visitId != -1) { - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - - // Check that moz_places table has been correctly synced - var stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE url = :url"); - stmt.params["url"] = TEST_URI; - do_check_true(stmt.executeStep()); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_places_temp WHERE url = :url"); - stmt.params["url"] = TEST_URI; - do_check_false(stmt.executeStep()); - stmt.finalize(); - - stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE url = :url"); - stmt.params["url"] = EMBED_URI; - do_check_false(stmt.executeStep()); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_places_temp WHERE url = :url"); - stmt.params["url"] = EMBED_URI; - do_check_true(stmt.executeStep()); - stmt.finalize(); - - stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE url = :url"); - stmt.params["url"] = PLACE_URI; - do_check_true(stmt.executeStep()); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_places_temp WHERE url = :url"); - stmt.params["url"] = PLACE_URI; - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that all visits but embed ones are in disk table - stmt = dbConn.createStatement( - "SELECT count(*) FROM moz_historyvisits h WHERE visit_type <> :t_embed"); - stmt.params["t_embed"] = hs.TRANSITION_EMBED; - do_check_true(stmt.executeStep()); - do_check_eq(stmt.getInt32(0), (transitions.length - 1) * 2); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_historyvisits h WHERE visit_type = :t_embed"); - stmt.params["t_embed"] = hs.TRANSITION_EMBED; - do_check_false(stmt.executeStep()); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_historyvisits_temp h WHERE visit_type = :t_embed"); - stmt.params["t_embed"] = hs.TRANSITION_EMBED; - do_check_true(stmt.executeStep()); - stmt.finalize(); - stmt = dbConn.createStatement( - "SELECT id FROM moz_historyvisits_temp h WHERE visit_type <> :t_embed"); - stmt.params["t_embed"] = hs.TRANSITION_EMBED; - do_check_false(stmt.executeStep()); - stmt.finalize(); - - do_test_finished(); - } - } -} - -function run_test() -{ - // First set the preference for the timer to a small value - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add a visit for every transition type on TEST_URI - // Embed visit should stay in temp table, while other should be synced - // Place entry for this uri should be synced to disk table - transitions.forEach(function addVisit(aTransition) { - hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - aTransition, false, 0); - }); - - // Add an embed visit for EMBED_URI - // Embed visit should stay in temp table - // Place entry for this uri should stay in temp table - hs.addVisit(uri(EMBED_URI), Date.now() * 1000, null, - hs.TRANSITION_EMBED, false, 0); - - // Add a visit for every transition type on PLACE_URI - // Embed visit should stay in temp table - // Place entry for this uri should be synced to disk table - transitions.forEach(function addVisit(aTransition) { - hs.addVisit(uri(PLACE_URI), Date.now() * 1000, null, - aTransition, false, 0); - }); - - os.addObserver(observer, kSyncFinished, false); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_expireAllFavicons.js b/toolkit/components/places/tests/sync/test_database_sync_expireAllFavicons.js deleted file mode 100644 index e6218b448a5..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_expireAllFavicons.js +++ /dev/null @@ -1,134 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2009 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/* - * This test ensures favicons are correctly expired by expireAllFavicons API, - * both synced and unsynced favicons should be expired. - */ -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var dbConn = hs.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection; -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var icons = Cc["@mozilla.org/browser/favicon-service;1"]. - getService(Ci.nsIFaviconService); - -const TEST_URI = "http://test.com/"; -const TEST_ICON_URI = "http://test.com/favicon.ico"; -const TEST_NOSYNC_URI = "http://nosync.test.com/"; -const TEST_NOSYNC_ICON_URI = "http://nosync.test.com/favicon.ico"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; -const kSyncFinished = "places-sync-finished"; - -const kExpirationFinished = "places-favicons-expired"; - -var observer = { - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - os.removeObserver(this, kSyncFinished); - - // Add a not synced page with a visit. - hs.addVisit(uri(TEST_NOSYNC_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - // Set a favicon for the page. - icons.setFaviconUrlForPage(uri(TEST_NOSYNC_URI), uri(TEST_NOSYNC_ICON_URI)); - - // Check both a synced and a not-synced favicons exist. - let stmt = dbConn.createStatement( - "SELECT id FROM moz_places WHERE url = :url AND favicon_id NOT NULL"); - stmt.params["url"] = TEST_URI; - do_check_true(stmt.executeStep()); - - stmt.finalize(); - - stmt = dbConn.createStatement( - "SELECT id FROM moz_places_temp WHERE url = :url AND favicon_id NOT NULL"); - stmt.params["url"] = TEST_NOSYNC_URI; - do_check_true(stmt.executeStep()); - - stmt.finalize(); - - // Expire all favicons. - icons.expireAllFavicons(); - } - else if (aTopic == kExpirationFinished) { - os.removeObserver(this, kExpirationFinished); - // Check all favicons have been removed. - let stmt = dbConn.createStatement( - "SELECT id FROM moz_favicons"); - do_check_false(stmt.executeStep()); - - stmt.finalize(); - - stmt = dbConn.createStatement( - "SELECT id FROM moz_places_view WHERE favicon_id NOT NULL"); - do_check_false(stmt.executeStep()); - - stmt.finalize(); - - do_test_finished(); - } - } -} -os.addObserver(observer, kSyncFinished, false); -os.addObserver(observer, kExpirationFinished, false); - -function run_test() -{ - // First set the preference for the timer to a large value so we don't sync. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add a page with a visit. - let visitId = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - // Set a favicon for the page. - icons.setFaviconUrlForPage(uri(TEST_URI), uri(TEST_ICON_URI)); - - // Now add a bookmark to force a sync. - bs.insertBookmark(bs.toolbarFolder, uri(TEST_URI), bs.DEFAULT_INDEX, "visited"); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_onitemadded.js b/toolkit/components/places/tests/sync/test_database_sync_onitemadded.js deleted file mode 100644 index f0e3b9ffa0c..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_onitemadded.js +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - - /* - * This test checks that when we add an item we sync only if its type is - * TYPE_BOOKMARKS - */ - -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; -const kSyncFinished = "places-sync-finished"; - -var syncObserver = { - _numSyncs: 0, - finalSync: false, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - if (++this._numSyncs > 1 || !this.finalSync) - do_throw("We synced too many times: " + this._numSyncs); - - // remove the observers - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver, false); - - do_test_finished(); - } - } -} -os.addObserver(syncObserver, kSyncFinished, false); - -var bookmarksObserver = { - onItemAdded: function(aItemId, aNewParent, aNewIndex) { - if (bs.getItemType(aItemId) == bs.TYPE_BOOKMARK) - syncObserver.finalSync = true; - } -} -bs.addObserver(bookmarksObserver, false); - -function run_test() -{ - // dynamic container sample - do_load_manifest("../unit/nsDynamicContainerServiceSample.manifest"); - - // First set the preference for the timer to a large value, so it won't sync - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add a folder - bs.createFolder(bs.toolbarFolder, "folder", bs.DEFAULT_INDEX); - - // Add a dynamic container - bs.createDynamicContainer(bs.toolbarFolder, "dynamic", - "@mozilla.org/browser/remote-container-sample;1", - bs.DEFAULT_INDEX); - - // Add a separator - bs.insertSeparator(bs.toolbarFolder, bs.DEFAULT_INDEX); - - // Add a bookmark, this will trigger final sync - bs.insertBookmark(bs.toolbarFolder, uri(TEST_URI), bs.DEFAULT_INDEX, "bookmark"); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_database_sync_with_specialHistoryQueries.js b/toolkit/components/places/tests/sync/test_database_sync_with_specialHistoryQueries.js deleted file mode 100644 index 5b9efaa2fb6..00000000000 --- a/toolkit/components/places/tests/sync/test_database_sync_with_specialHistoryQueries.js +++ /dev/null @@ -1,116 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Marco Bonardo (Original Author) - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 600; -const kSyncFinished = "places-sync-finished"; - -var observer = { - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - // Set the preference for the timer to a large value so we don't sync. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Now add another visit, be sure to use a different session, so we - // will also test grouping by uri. - hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 1); - - // Create the history menu query. - var options = hs.getNewQueryOptions(); - options.maxResults = 10; - options.resultType = options.RESULTS_AS_URI; - options.sortingMode = options.SORT_BY_DATE_DESCENDING; - var query = hs.getNewQuery(); - var result = hs.executeQuery(query, options); - var root = result.root; - root.containerOpen = true; - do_check_eq(root.childCount, 1); - root.containerOpen = false; - - // Create the most visited query. - options = hs.getNewQueryOptions(); - options.maxResults = 10; - options.resultType = options.RESULTS_AS_URI; - options.sortingMode = options.SORT_BY_VISITCOUNT_DESCENDING; - query = hs.getNewQuery(); - result = hs.executeQuery(query, options); - root = result.root; - root.containerOpen = true; - do_check_eq(root.childCount, 1); - root.containerOpen = false; - - // Create basic uri query. - options = hs.getNewQueryOptions(); - query = hs.getNewQuery(); - result = hs.executeQuery(query, options); - root = result.root; - root.containerOpen = true; - do_check_eq(root.childCount, 1); - root.containerOpen = false; - - os.removeObserver(this, kSyncFinished); - do_test_finished(); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // First set the preference for the timer to a small value so we sync - prefs.setIntPref(kSyncPrefName, 1); - - // Now add the visit - let visitId = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_multiple_bookmarks_around_sync.js b/toolkit/components/places/tests/sync/test_multiple_bookmarks_around_sync.js deleted file mode 100644 index 5e518b497bb..00000000000 --- a/toolkit/components/places/tests/sync/test_multiple_bookmarks_around_sync.js +++ /dev/null @@ -1,150 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/** - * This test ensures that adding a bookmark (which has an implicit sync), then - * adding another one that has the same place, we end up with only one entry in - * moz_places. - */ - -var bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var db = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsPIPlacesDatabase). - DBConnection; -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); - -const TEST_URI = "http://test.com/"; - -const SYNC_INTERVAL = 600; // ten minutes -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const kSyncFinished = "places-sync-finished"; - -// Used to update observer itemId -var bookmarksObserver = { - onItemAdded: function(aItemId, aNewParent, aNewIndex) { - observer.itemIds.push(aItemId); - } -} -bs.addObserver(bookmarksObserver, false); - -var observer = { - itemIds: [], - _placeId: -1, - _runCount: 0, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished) { - if (++this._runCount == 1) { - let itemId = this.itemIds[this._runCount - 1]; - // item id must be valid - do_check_neq(itemId, null); - // Ensure tables have been synced - new_test_bookmark_uri_event(itemId, TEST_URI, true); - - // Get the place_id - let stmt = db.createStatement( - "SELECT fk " + - "FROM moz_bookmarks " + - "WHERE id = ?" - ); - stmt.bindInt64Parameter(0, itemId); - do_check_true(stmt.executeStep()); - this._placeId = stmt.getInt64(0); - stmt.finalize(); - stmt = null; - // place id must be valid - do_check_true(this._placeId > 0); - } - else if (this._runCount == 2) { - let itemId = this.itemIds[this._runCount - 1]; - // item id must be valid - do_check_neq(itemId, null); - // Ensure it was added - new_test_bookmark_uri_event(itemId, TEST_URI, true); - - // Check to make sure we have the same place_id - stmt = db.createStatement( - "SELECT * " + - "FROM moz_bookmarks " + - "WHERE id = ?1 " + - "AND fk = ?2" - ); - stmt.bindInt64Parameter(0, itemId); - stmt.bindInt64Parameter(1, this._placeId); - do_check_true(stmt.executeStep()); - stmt.finalize(); - stmt = null; - - // remove the observer, we don't need to observe sync on quit - os.removeObserver(this, kSyncFinished); - bs.removeObserver(bookmarksObserver); - // test ends here - do_test_finished(); - } - else - do_throw("Too many places sync calls"); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -function run_test() -{ - // Set the preference for the timer to a really large value, so it won't - // run before the test finishes. - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Add the first bookmark - let id1 = bs.insertBookmark(bs.unfiledBookmarksFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "test"); - - // Now we add another bookmark to a different folder - let id2 = bs.insertBookmark(bs.toolbarFolder, uri(TEST_URI), - bs.DEFAULT_INDEX, "test"); - do_check_neq(id1, id2); - - do_test_pending(); -} diff --git a/toolkit/components/places/tests/sync/test_multiple_visits_around_sync.js b/toolkit/components/places/tests/sync/test_multiple_visits_around_sync.js deleted file mode 100644 index 1b99b09a923..00000000000 --- a/toolkit/components/places/tests/sync/test_multiple_visits_around_sync.js +++ /dev/null @@ -1,152 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * vim: sw=2 ts=2 sts=2 expandtab - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2008 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Shawn Wilsher (Original Author) - * Marco Bonardo - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -/** - * This test ensures that when adding a visit, then syncing, and adding another - * visit to the same url creates two visits and that we only end up with one - * entry in moz_places. - */ - -var hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -var db = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsPIPlacesDatabase). - DBConnection; -var prefs = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefService). - getBranch("places."); -var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - -const TEST_URI = "http://test.com/"; - -const kSyncPrefName = "syncDBTableIntervalInSecs"; -const SYNC_INTERVAL = 1; -const kSyncFinished = "places-sync-finished"; - -var observer = { - visitId: -1, - _runCount: 0, - _placeId: -1, - _lastVisitId: -1, - observe: function(aSubject, aTopic, aData) { - if (aTopic == kSyncFinished && this.visitId != -1) { - // Check the visit - new_test_visit_uri_event(this.visitId, TEST_URI, true); - - // sanity check: visitId set by history observer should be the same we - // have added - do_check_eq(this.visitId, visitIds[this._runCount]); - - if (++this._runCount == 1) { - // Get the place_id and pass it on - let stmt = db.createStatement( - "SELECT place_id " + - "FROM moz_historyvisits " + - "WHERE id = ?1" - ); - stmt.bindInt64Parameter(0, this.visitId); - do_check_true(stmt.executeStep()); - this._placeId = stmt.getInt64(0); - this._lastVisitId = this.visitId; - stmt.finalize(); - stmt = null; - - // clear cached value before continue test - this.visitId = -1; - continue_test(); - } - else if (this._runCount == 2) { - do_check_neq(this.visitId, this._lastVisitId); - // Get the place_id and check for equality - let stmt = db.createStatement( - "SELECT place_id " + - "FROM moz_historyvisits " + - "WHERE id = ?1" - ); - stmt.bindInt64Parameter(0, this.visitId); - do_check_true(stmt.executeStep()); - do_check_eq(this._placeId, stmt.getInt64(0)); - stmt.finalize(); - stmt = null; - - // remove the observers - os.removeObserver(this, kSyncFinished); - hs.removeObserver(historyObserver, false); - do_test_finished(); - } - else - do_throw("bad runCount!"); - } - } -} -os.addObserver(observer, kSyncFinished, false); - -// Used to update observer visitId -var historyObserver = { - onVisit: function(aURI, aVisitId, aTime, aSessionId, aReferringId, - aTransitionType, aAdded) { - do_check_true(aVisitId > 0); - observer.visitId = aVisitId; - } -} -hs.addObserver(historyObserver, false); - -// used for sanity checks -var visitIds = []; - -function run_test() -{ - // First set the preference for the timer to a small value - prefs.setIntPref(kSyncPrefName, SYNC_INTERVAL); - - // Now add the first visit - visitIds[0] = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); - - do_test_pending(); -} - -function continue_test() -{ - // Now we add another visit - visitIds[1] = hs.addVisit(uri(TEST_URI), Date.now() * 1000, null, - hs.TRANSITION_TYPED, false, 0); -} diff --git a/toolkit/components/places/tests/unit/test_248970.js b/toolkit/components/places/tests/unit/test_248970.js index ad51d92746b..ce4f2307b35 100644 --- a/toolkit/components/places/tests/unit/test_248970.js +++ b/toolkit/components/places/tests/unit/test_248970.js @@ -45,8 +45,6 @@ var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"]. var bhist = histsvc.QueryInterface(Ci.nsIBrowserHistory); var bmsvc = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. getService(Ci.nsINavBookmarksService); -var ios = Cc["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); /** * Function prohibits an attempt to pop up a confirmation @@ -249,22 +247,13 @@ function run_test() { var pb = get_PBSvc(); if (pb) { // Private Browsing might not be available - // need to catch places sync notifications - var os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - const kSyncFinished = "places-sync-finished"; do_test_pending(); - var prefBranch = Cc["@mozilla.org/preferences-service;1"]. - getService(Ci.nsIPrefBranch); - prefBranch.setBoolPref("browser.privatebrowsing.keep_current_session", true); - - var bookmark_A_URI = ios.newURI("http://google.com/", null, null); - var bookmark_B_URI = ios.newURI("http://bugzilla.mozilla.org/", null, null); - var onBookmarkAAdded = { - observe: function (aSubject, aTopic, aData) { - os.removeObserver(this, kSyncFinished); + Services.prefs.setBoolPref("browser.privatebrowsing.keep_current_session", true); + var bookmark_A_URI = NetUtil.newURI("http://google.com/"); + var bookmark_B_URI = NetUtil.newURI("http://bugzilla.mozilla.org/"); + var onBookmarkAAdded = function() { check_placesItem_Count(); // Bookmark-A should be bookmarked, data should be retrievable @@ -298,16 +287,11 @@ function run_test() { do_check_true(bmsvc.isBookmarked(bookmark_A_URI)); do_check_eq("google",bmsvc.getKeywordForURI(bookmark_A_URI)); - os.addObserver(onBookmarkBAdded, kSyncFinished, false); - // Create Bookmark-B myBookmarks[1] = create_bookmark(bookmark_B_URI,"title 2", "bugzilla"); - } + onBookmarkBAdded(); }; - var onBookmarkBAdded = { - observe: function (aSubject, aTopic, aData) { - os.removeObserver(this, kSyncFinished); - + var onBookmarkBAdded = function() { // A check on the history count should be same as before, 7 history entries with // now 2 bookmark items (A) and bookmark (B), so we set num_places_entries to 9 num_places_entries = 10; // Bookmark-B successfully added but not the history entries. @@ -330,9 +314,8 @@ function run_test() { do_check_true(bhist.isVisited(uri(visited_uri))); } - prefBranch.clearUserPref("browser.privatebrowsing.keep_current_session"); + Services.prefs.clearUserPref("browser.privatebrowsing.keep_current_session"); do_test_finished(); - } }; // History database should be empty @@ -344,8 +327,6 @@ function run_test() { // History database should have entries do_check_true(histsvc.hasHistoryEntries); - os.addObserver(onBookmarkAAdded, kSyncFinished, false); - // Create Bookmark-A myBookmarks[0] = create_bookmark(bookmark_A_URI,"title 1", "google"); @@ -354,5 +335,7 @@ function run_test() { do_check_true(bhist.isVisited(uri(visited_uri))); do_check_true(uri_in_db(uri(visited_uri))); } + + onBookmarkAAdded(); } } diff --git a/toolkit/components/places/tests/unit/test_412132.js b/toolkit/components/places/tests/unit/test_412132.js index 8a73511e5be..308653c7b2e 100644 --- a/toolkit/components/places/tests/unit/test_412132.js +++ b/toolkit/components/places/tests/unit/test_412132.js @@ -44,275 +44,230 @@ * https://bugzilla.mozilla.org/show_bug.cgi?id=412132 */ -const bmServ = - Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -const histServ = - Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -const lmServ = - Cc["@mozilla.org/browser/livemark-service;2"]. - getService(Ci.nsILivemarkService); +const bmServ = PlacesUtils.bookmarks; +const histServ = PlacesUtils.history; +const lmServ = PlacesUtils.livemarks; -var defaultBookmarksMaxId; -var dbConn; -var tests = []; - -tests.push({ +let tests = [ +{ desc: ["Frecency of unvisited, separately bookmarked livemark item's URI ", "should be zero after bookmark's URI changed."].join(""), run: function () { - var lmItemURL; - var lmItemURI; - var bmId; - var frecencyBefore; - var frecencyAfter; - // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. - lmItemURL = "http://example.com/livemark-item"; - lmItemURI = uri(lmItemURL); + let lmItemURL = "http://example.com/livemark-item"; + let lmItemURI = uri(lmItemURL); createLivemark(lmItemURI); - bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - lmItemURI, - bmServ.DEFAULT_INDEX, - "bookmark title"); + let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + lmItemURI, + bmServ.DEFAULT_INDEX, + "bookmark title"); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0."); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - frecencyBefore = getFrecency(lmItemURL); - do_check_neq(frecencyBefore, 0); - - bmServ.changeBookmarkURI(bmId, uri("http://example.com/new-uri")); - - // URI's only "bookmark" is now unvisited livemark item => frecency = 0. - frecencyAfter = getFrecency(lmItemURL); - do_check_eq(frecencyAfter, 0); + bmServ.changeBookmarkURI(aItemId, uri("http://example.com/new-uri")); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now unvisited livemark item => frecency = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["Frecency of visited, separately bookmarked livemark item's URI ", "should not be zero after bookmark's URI changed."].join(""), run: function () { - var lmItemURL; - var lmItemURI; - var bmId; - var frecencyBefore; - var frecencyAfter; - // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. - lmItemURL = "http://example.com/livemark-item"; - lmItemURI = uri(lmItemURL); + let lmItemURL = "http://example.com/livemark-item"; + let lmItemURI = uri(lmItemURL); createLivemark(lmItemURI); - bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - lmItemURI, - bmServ.DEFAULT_INDEX, - "bookmark title"); + let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + lmItemURI, + bmServ.DEFAULT_INDEX, + "bookmark title"); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - frecencyBefore = getFrecency(lmItemURL); - do_check_neq(frecencyBefore, 0); + visit(uri(aUrl)); - visit(lmItemURI); - - bmServ.changeBookmarkURI(bmId, uri("http://example.com/new-uri")); - - // URI's only "bookmark" is now *visited* livemark item => frecency != 0. - frecencyAfter = getFrecency(lmItemURL); - do_check_neq(frecencyAfter, 0); + bmServ.changeBookmarkURI(aItemId, uri("http://example.com/new-uri")); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now *visited* livemark item => frecency != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After changing URI of bookmark, frecency of bookmark's original URI ", "should be zero if original URI is unvisited and no longer ", "bookmarked."].join(""), run: function () { - var url; - var bmId; - var frecencyBefore; - var frecencyAfter; + let url = "http://example.com/1"; + let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + uri(url), + bmServ.DEFAULT_INDEX, + "bookmark title"); + waitForFrecency(url, function(aFrecency) aFrecency > 0, + this.check1, this, [url, bmId]); + }, + check1: function (aUrl, aItemId) { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - url = "http://example.com/1"; - bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - uri(url), - bmServ.DEFAULT_INDEX, - "bookmark title"); + bmServ.changeBookmarkURI(aItemId, uri("http://example.com/2")); - // Bookmarked => frecency of URI should be != 0. - frecencyBefore = getFrecency(url); - do_check_neq(frecencyBefore, 0); - - bmServ.changeBookmarkURI(bmId, uri("http://example.com/2")); - - // Unvisited URI no longer bookmarked => frecency should = 0. - frecencyAfter = getFrecency(url); - do_check_eq(frecencyAfter, 0); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("Unvisited URI no longer bookmarked => frecency should = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After changing URI of bookmark, frecency of bookmark's original URI ", "should not be zero if original URI is visited."].join(""), run: function () { - var bmURL; - var bmURI; - var bmId; - var frecencyBefore; - var frecencyAfter; + let bmURL = "http://example.com/1"; + let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + uri(bmURL), + bmServ.DEFAULT_INDEX, + "bookmark title"); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - bmURL = "http://example.com/1"; - bmURI = uri(bmURL); - - bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - bmURI, - bmServ.DEFAULT_INDEX, - "bookmark title"); - - // Bookmarked => frecency of URI should be != 0. - frecencyBefore = getFrecency(bmURL); - do_check_neq(frecencyBefore, 0); - - visit(bmURI); - bmServ.changeBookmarkURI(bmId, uri("http://example.com/2")); - - // *Visited* URI no longer bookmarked => frecency should != 0. - frecencyAfter = getFrecency(bmURL); - do_check_neq(frecencyAfter, 0); + visit(uri(aUrl)); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl, aItemId]); + }, + check2: function (aUrl, aItemId) + { + bmServ.changeBookmarkURI(aItemId, uri("http://example.com/2")); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check3, this, [aUrl]); + }, + check3: function (aUrl) + { + print("*Visited* URI no longer bookmarked => frecency should != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After changing URI of bookmark, frecency of bookmark's original URI ", "should not be zero if original URI is still bookmarked."].join(""), run: function () { - var bmURL; - var bmURI; - var bm1Id; - var bm2Id; - var frecencyBefore; - var frecencyAfter; + let bmURL = "http://example.com/1"; + let bm1Id = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + uri(bmURL), + bmServ.DEFAULT_INDEX, + "bookmark 1 title"); - bmURL = "http://example.com/1"; - bmURI = uri(bmURL); + let bm2Id = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + uri(bmURL), + bmServ.DEFAULT_INDEX, + "bookmark 2 title"); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL, bm1Id]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - bm1Id = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - bmURI, - bmServ.DEFAULT_INDEX, - "bookmark 1 title"); - - bm2Id = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - bmURI, - bmServ.DEFAULT_INDEX, - "bookmark 2 title"); - - - // Bookmarked => frecency of URI should be != 0. - frecencyBefore = getFrecency(bmURL); - do_check_neq(frecencyBefore, 0); - - bmServ.changeBookmarkURI(bm1Id, uri("http://example.com/2")); - - // URI still bookmarked => frecency should != 0. - frecencyAfter = getFrecency(bmURL); - do_check_neq(frecencyAfter, 0); + bmServ.changeBookmarkURI(aItemId, uri("http://example.com/2")); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI still bookmarked => frecency should != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: "Changing the URI of a nonexistent bookmark should fail.", run: function () { - var stmt; - var maxId; - var bmId; - function tryChange(itemId) { - var failed; - - failed= false; - try - { + try { bmServ.changeBookmarkURI(itemId + 1, uri("http://example.com/2")); + do_throw("Nonexistent bookmark should throw."); } - catch (exc) - { - failed= true; - } - do_check_true(failed); + catch (exc) {} } // First try a straight-up bogus item ID, one greater than the current max // ID. - stmt = dbConn.createStatement("SELECT MAX(id) FROM moz_bookmarks"); + let stmt = DBConn().createStatement("SELECT MAX(id) FROM moz_bookmarks"); stmt.executeStep(); - maxId = stmt.getInt32(0); + let maxId = stmt.getInt32(0); stmt.finalize(); tryChange(maxId + 1); // Now add a bookmark, delete it, and check. - bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, - uri("http://example.com/"), - bmServ.DEFAULT_INDEX, - "bookmark title"); + let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, + uri("http://example.com/"), + bmServ.DEFAULT_INDEX, + "bookmark title"); bmServ.removeItem(bmId); tryChange(bmId); + run_next_test(); } -}); +}, + +]; /******************************************************************************/ function createLivemark(lmItemURI) { - var lmId; - var lmItemId; - - lmId = lmServ.createLivemarkFolderOnly(bmServ.unfiledBookmarksFolder, - "livemark title", - uri("http://www.mozilla.org/"), - uri("http://www.mozilla.org/news.rdf"), - -1); - lmItemId = bmServ.insertBookmark(lmId, - lmItemURI, - bmServ.DEFAULT_INDEX, - "livemark item title"); - return lmItemId; -} - -function getFrecency(url) -{ - var sql; - var stmt; - var frecency; - - sql = "SELECT frecency FROM moz_places_view WHERE url = ?1"; - stmt = dbConn.createStatement(sql); - stmt.bindUTF8StringParameter(0, url); - do_check_true(stmt.executeStep()); - frecency = stmt.getInt32(0); - print("frecency=" + frecency); - stmt.finalize(); - - return frecency; -} - -function prepTest(testName, callback) -{ - print("Test: " + testName); - waitForClearHistory(function() { - dbConn.executeSimpleSQL("DELETE FROM moz_places_view"); - dbConn.executeSimpleSQL("DELETE FROM moz_bookmarks WHERE id > " + - defaultBookmarksMaxId); - callback(); - runNextTest(); - }); + let lmId = lmServ.createLivemarkFolderOnly(bmServ.unfiledBookmarksFolder, + "livemark title", + uri("http://www.mozilla.org/"), + uri("http://www.mozilla.org/news.rdf"), + -1); + return bmServ.insertBookmark(lmId, + lmItemURI, + bmServ.DEFAULT_INDEX, + "livemark item title"); } function visit(uri) @@ -330,27 +285,19 @@ function visit(uri) function run_test() { do_test_pending(); - var stmt; - - dbConn = - Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsPIPlacesDatabase). - DBConnection; - - stmt = dbConn.createStatement("SELECT MAX(id) FROM moz_bookmarks"); - stmt.executeStep(); - defaultBookmarksMaxId = stmt.getInt32(0); - stmt.finalize(); - do_check_true(defaultBookmarksMaxId > 0); - - runNextTest(); + run_next_test(); } -let currentTest; -function runNextTest() { +function run_next_test() +{ if (tests.length) { - currentTest = tests.shift(); - prepTest(currentTest.desc, currentTest.run); + let test = tests.shift(); + print("\n ***Test: " + test.desc); + waitForClearHistory(function() { + DBConn().executeSimpleSQL("DELETE FROM moz_places"); + remove_all_bookmarks(); + test.run.call(test); + }); } else { do_test_finished(); diff --git a/toolkit/components/places/tests/unit/test_454977.js b/toolkit/components/places/tests/unit/test_454977.js index fa8f61a638a..7919502b03d 100644 --- a/toolkit/components/places/tests/unit/test_454977.js +++ b/toolkit/components/places/tests/unit/test_454977.js @@ -57,7 +57,7 @@ function add_visit(aURI, aVisitDate, aVisitType) { aVisitType != hs.TRANSITION_DOWNLOAD) visit_count ++; // Get the place id - var sql = "SELECT place_id FROM moz_historyvisits_view WHERE id = ?1"; + var sql = "SELECT place_id FROM moz_historyvisits WHERE id = ?1"; var stmt = mDBConn.createStatement(sql); stmt.bindInt64Parameter(0, visitId); do_check_true(stmt.executeStep()); diff --git a/toolkit/components/places/tests/unit/test_doSetAndLoadFaviconForPage_failures.js b/toolkit/components/places/tests/unit/test_doSetAndLoadFaviconForPage_failures.js index d1ea2d67f86..e8c5ea0b072 100644 --- a/toolkit/components/places/tests/unit/test_doSetAndLoadFaviconForPage_failures.js +++ b/toolkit/components/places/tests/unit/test_doSetAndLoadFaviconForPage_failures.js @@ -138,8 +138,8 @@ let historyObserver = { return; // dump tables, useful if the test fails. - dump_table("moz_places_temp"); - dump_table("moz_favicons"); + //dump_table("moz_places"); + //dump_table("moz_favicons"); // Ensure we have been called by the last test. do_check_true(pageURI.equals(uri("http://test4.bar/"))); diff --git a/toolkit/components/places/tests/unit/test_history_removeAllPages.js b/toolkit/components/places/tests/unit/test_history_removeAllPages.js index 48aa75a8138..64ce0d7d312 100644 --- a/toolkit/components/places/tests/unit/test_history_removeAllPages.js +++ b/toolkit/components/places/tests/unit/test_history_removeAllPages.js @@ -37,245 +37,201 @@ * * ***** END LICENSE BLOCK ***** */ -// Get services. -let hs = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -let bh = hs.QueryInterface(Ci.nsIBrowserHistory); -let mDBConn = hs.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection; -let bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -let as = Cc["@mozilla.org/browser/annotation-service;1"]. - getService(Ci.nsIAnnotationService); -let os = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); -let lms = Cc["@mozilla.org/browser/livemark-service;2"]. - getService(Ci.nsILivemarkService); - -const kSyncFinished = "places-sync-finished"; -const kExpirationFinished = "places-expiration-finished"; - -// Number of expected sync notifications, we expect one per bookmark. -const EXPECTED_SYNCS = 4; +let mDBConn = DBConn(); function add_fake_livemark() { - let lmId = lms.createLivemarkFolderOnly(bs.toolbarFolder, - "Livemark", - uri("http://www.mozilla.org/"), - uri("http://www.mozilla.org/test.xml"), - bs.DEFAULT_INDEX); - // add a visited child - bs.insertBookmark(lmId, uri("http://visited.livemark.com/"), - bs.DEFAULT_INDEX, "visited"); - hs.addVisit(uri("http://visited.livemark.com/"), Date.now(), null, - hs.TRANSITION_BOOKMARK, false, 0); - // add an unvisited child - bs.insertBookmark(lmId, uri("http://unvisited.livemark.com/"), - bs.DEFAULT_INDEX, "unvisited"); + let lmId = PlacesUtils.livemarks.createLivemarkFolderOnly( + PlacesUtils.toolbarFolderId, "Livemark", + uri("http://www.mozilla.org/"), uri("http://www.mozilla.org/test.xml"), + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + // Add a visited child. + PlacesUtils.bookmarks.insertBookmark(lmId, + uri("http://visited.livemark.com/"), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "visited"); + PlacesUtils.history.addVisit(uri("http://visited.livemark.com/"), + Date.now(), null, + Ci.nsINavHistoryService.TRANSITION_BOOKMARK, + false, 0); + // Add an unvisited child. + PlacesUtils.bookmarks.insertBookmark(lmId, + uri("http://unvisited.livemark.com/"), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "unvisited"); } -let observer = { - onBeginUpdateBatch: function() { - }, - onEndUpdateBatch: function() { - }, - onVisit: function(aURI, aVisitID, aTime, aSessionID, aReferringID, aTransitionType) { - }, - onTitleChanged: function(aURI, aPageTitle) { - }, - onBeforeDeleteURI: function(aURI) { - }, - onDeleteURI: function(aURI) { - }, +let historyObserver = { + onBeginUpdateBatch: function() {}, + onEndUpdateBatch: function() {}, + onVisit: function() {}, + onTitleChanged: function() {}, + onBeforeDeleteURI: function() {}, + onDeleteURI: function(aURI) {}, + onPageChanged: function() {}, + onDeleteVisits: function() {}, onClearHistory: function() { + PlacesUtils.history.removeObserver(this, false); + // check browserHistory returns no entries - do_check_eq(0, bh.count); + do_check_eq(0, PlacesUtils.bhistory.count); let expirationObserver = { observe: function (aSubject, aTopic, aData) { - os.removeObserver(this, kExpirationFinished, false); - + Services.obs.removeObserver(this, aTopic, false); + // Check that frecency for not cleared items (bookmarks) has been converted // to -MAX(visit_count, 1), so we will be able to recalculate frecency - // starting from most frecent bookmarks. - // Memory table has been updated, disk table has not + // starting from most frecent bookmarks. stmt = mDBConn.createStatement( - "SELECT id FROM moz_places_temp WHERE frecency > 0 LIMIT 1"); + "SELECT h.id FROM moz_places h WHERE h.frecency > 0 "); do_check_false(stmt.executeStep()); stmt.finalize(); stmt = mDBConn.createStatement( - "SELECT h.id FROM moz_places_temp h WHERE h.frecency = -2 " + + "SELECT h.id FROM moz_places h WHERE h.frecency = -2 " + "AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1"); do_check_true(stmt.executeStep()); stmt.finalize(); // Check that all visit_counts have been brought to 0 stmt = mDBConn.createStatement( - "SELECT id FROM moz_places_temp WHERE visit_count <> 0 LIMIT 1"); + "SELECT id FROM moz_places WHERE visit_count <> 0 LIMIT 1"); do_check_false(stmt.executeStep()); stmt.finalize(); // Check that history tables are empty stmt = mDBConn.createStatement( - "SELECT * FROM (SELECT id FROM moz_historyvisits_temp LIMIT 1) " + - "UNION ALL " + "SELECT * FROM (SELECT id FROM moz_historyvisits LIMIT 1)"); do_check_false(stmt.executeStep()); stmt.finalize(); - // force a sync and check again disk tables, insertBookmark will do that - bs.insertBookmark(bs.unfiledBookmarksFolder, uri("place:folder=4"), - bs.DEFAULT_INDEX, "shortcut"); + // Check that all moz_places entries except bookmarks and place: have been removed + stmt = mDBConn.createStatement( + "SELECT h.id FROM moz_places h WHERE SUBSTR(h.url, 1, 6) <> 'place:' "+ + "AND NOT EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + // Check that we only have favicons for retained places + stmt = mDBConn.createStatement( + "SELECT f.id FROM moz_favicons f WHERE NOT EXISTS " + + "(SELECT id FROM moz_places WHERE favicon_id = f.id) LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + // Check that we only have annotations for retained places + stmt = mDBConn.createStatement( + "SELECT a.id FROM moz_annos a WHERE NOT EXISTS " + + "(SELECT id FROM moz_places WHERE id = a.place_id) LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + // Check that we only have inputhistory for retained places + stmt = mDBConn.createStatement( + "SELECT i.place_id FROM moz_inputhistory i WHERE NOT EXISTS " + + "(SELECT id FROM moz_places WHERE id = i.place_id) LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + // Check that place:uris have frecency 0 + stmt = mDBConn.createStatement( + "SELECT h.id FROM moz_places h " + + "WHERE SUBSTR(h.url, 1, 6) = 'place:' AND h.frecency <> 0 LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + // Check that livemarks children don't have frecency <> 0 + stmt = mDBConn.createStatement( + "SELECT h.id FROM moz_places h " + + "JOIN moz_bookmarks b ON h.id = b.fk " + + "JOIN moz_bookmarks bp ON bp.id = b.parent " + + "JOIN moz_items_annos t ON t.item_id = bp.id " + + "JOIN moz_anno_attributes n ON t.anno_attribute_id = n.id " + + "WHERE n.name = 'livemark/feedURI' AND h.frecency <> 0 LIMIT 1"); + do_check_false(stmt.executeStep()); + stmt.finalize(); + + do_test_finished(); } } - os.addObserver(expirationObserver, kExpirationFinished, false); + Services.obs.addObserver(expirationObserver, + PlacesUtils.TOPIC_EXPIRATION_FINISHED, + false); }, - onPageChanged: function(aURI, aWhat, aValue) { - }, - onDeleteVisits: function() { - }, - - QueryInterface: function(iid) { - if (iid.equals(Ci.nsINavHistoryObserver) || - iid.equals(Ci.nsISupports)) { - return this; - } - throw Cr.NS_ERROR_NO_INTERFACE; - } + QueryInterface: XPCOMUtils.generateQI([ + Ci.nsINavHistoryObserver, + ]), } -hs.addObserver(observer, false); +PlacesUtils.history.addObserver(historyObserver, false); -let syncObserver = { - _runCount: 0, - observe: function (aSubject, aTopic, aData) { - if (++this._runCount < EXPECTED_SYNCS) - return; - if (this._runCount == EXPECTED_SYNCS) { - bh.removeAllPages(); - return; - } - os.removeObserver(this, kSyncFinished, false); - - // Sanity: check that places temp table is empty - stmt = mDBConn.createStatement( - "SELECT id FROM moz_places_temp LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that frecency for not cleared items (bookmarks) has been converted - // to -MAX(visit_count, 1), so we will be able to recalculate frecency - // starting from most frecent bookmarks. - stmt = mDBConn.createStatement( - "SELECT id FROM moz_places WHERE frecency > 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - stmt = mDBConn.createStatement( - "SELECT h.id FROM moz_places h WHERE h.frecency = -2 " + - "AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1"); - do_check_true(stmt.executeStep()); - stmt.finalize(); - - // Check that all visit_counts have been brought to 0 - stmt = mDBConn.createStatement( - "SELECT id FROM moz_places WHERE visit_count <> 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that all moz_places entries except bookmarks and place: have been removed - stmt = mDBConn.createStatement( - "SELECT h.id FROM moz_places h WHERE SUBSTR(h.url, 1, 6) <> 'place:' "+ - "AND NOT EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that we only have favicons for retained places - stmt = mDBConn.createStatement( - "SELECT f.id FROM moz_favicons f WHERE NOT EXISTS " + - "(SELECT id FROM moz_places WHERE favicon_id = f.id) LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that we only have annotations for retained places - stmt = mDBConn.createStatement( - "SELECT a.id FROM moz_annos a WHERE NOT EXISTS " + - "(SELECT id FROM moz_places WHERE id = a.place_id) LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that we only have inputhistory for retained places - stmt = mDBConn.createStatement( - "SELECT i.place_id FROM moz_inputhistory i WHERE NOT EXISTS " + - "(SELECT id FROM moz_places WHERE id = i.place_id) LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that place:uris have frecency 0 - stmt = mDBConn.createStatement( - "SELECT h.id FROM moz_places h " + - "WHERE SUBSTR(h.url, 1, 6) = 'place:' AND h.frecency <> 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - // Check that livemarks children don't have frecency <> 0 - stmt = mDBConn.createStatement( - "SELECT h.id FROM moz_places h " + - "JOIN moz_bookmarks b ON h.id = b.fk " + - "JOIN moz_bookmarks bp ON bp.id = b.parent " + - "JOIN moz_items_annos t ON t.item_id = bp.id " + - "JOIN moz_anno_attributes n ON t.anno_attribute_id = n.id " + - "WHERE n.name = 'livemark/feedURI' AND h.frecency <> 0 LIMIT 1"); - do_check_false(stmt.executeStep()); - stmt.finalize(); - - do_test_finished(); - } -} -os.addObserver(syncObserver, kSyncFinished, false); - -// main function run_test() { + // places-init-complete is notified after run_test, and it will + // run a first frecency fix through async statements. + // To avoid random failures we have to run after all of this. + Services.obs.addObserver(function(aSubject, aTopic, aData) { + Services.obs.removeObserver(arguments.callee, aTopic, false); + do_execute_soon(continue_test); + }, PlacesUtils.TOPIC_INIT_COMPLETE, false); + + do_test_pending(); +} + +function continue_test() { // Add a livemark with a visited and an unvisited child add_fake_livemark(); - - // Add a bunch of visits - hs.addVisit(uri("http://typed.mozilla.org"), Date.now(), null, - hs.TRANSITION_TYPED, false, 0); - - hs.addVisit(uri("http://link.mozilla.org"), Date.now(), null, - hs.TRANSITION_LINK, false, 0); - hs.addVisit(uri("http://download.mozilla.org"), Date.now(), null, - hs.TRANSITION_DOWNLOAD, false, 0); - hs.addVisit(uri("http://invalid.mozilla.org"), Date.now(), null, - 0, false, 0); // invalid transition - hs.addVisit(uri("http://redir_temp.mozilla.org"), Date.now(), - uri("http://link.mozilla.org"), hs.TRANSITION_REDIRECT_TEMPORARY, - true, 0); - hs.addVisit(uri("http://redir_perm.mozilla.org"), Date.now(), - uri("http://link.mozilla.org"), hs.TRANSITION_REDIRECT_PERMANENT, - true, 0); + PlacesUtils.history.addVisit(uri("http://typed.mozilla.org/"), Date.now(), + null, Ci.nsINavHistoryService.TRANSITION_TYPED, + false, 0); + PlacesUtils.history.addVisit(uri("http://link.mozilla.org/"), Date.now(), + null, Ci.nsINavHistoryService.TRANSITION_LINK, + false, 0); + PlacesUtils.history.addVisit(uri("http://download.mozilla.org/"), Date.now(), + null, Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + false, 0); + PlacesUtils.history.addVisit(uri("http://invalid.mozilla.org/"), Date.now(), + null, 0, false, 0); // Use an invalid transition. + PlacesUtils.history.addVisit(uri("http://redir_temp.mozilla.org/"), Date.now(), + uri("http://link.mozilla.org/"), + Ci.nsINavHistoryService.TRANSITION_REDIRECT_TEMPORARY, + true, 0); + PlacesUtils.history.addVisit(uri("http://redir_perm.mozilla.org/"), Date.now(), + uri("http://link.mozilla.org/"), + Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT, + true, 0); // add a place: bookmark - bs.insertBookmark(bs.unfiledBookmarksFolder, uri("place:folder=4"), - bs.DEFAULT_INDEX, "shortcut"); + PlacesUtils.bookmarks.insertBookmark(PlacesUtils.unfiledBookmarksFolderId, + uri("place:folder=4"), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "shortcut"); // Add an expire never annotation // Actually expire never annotations are removed as soon as a page is removed // from the database, so this should act as a normal visit. - as.setPageAnnotation(uri("http://download.mozilla.org"), "never", "never", 0, - as.EXPIRE_NEVER); + PlacesUtils.annotations.setPageAnnotation(uri("http://download.mozilla.org/"), + "never", "never", 0, + PlacesUtils.annotations.EXPIRE_NEVER); // Add a bookmark // Bookmarked page should have history cleared and frecency = -old_visit_count - // This will also finally sync temp tables to disk - bs.insertBookmark(bs.unfiledBookmarksFolder, uri("http://typed.mozilla.org"), - bs.DEFAULT_INDEX, "bookmark"); + PlacesUtils.bookmarks.insertBookmark(PlacesUtils.unfiledBookmarksFolderId, + uri("http://typed.mozilla.org/"), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "bookmark"); - // this visit is not synced to disk - hs.addVisit(uri("http://typed.mozilla.org"), Date.now(), null, - hs.TRANSITION_BOOKMARK, false, 0); + PlacesUtils.history.addVisit(uri("http://typed.mozilla.org/"), Date.now(), + null, PlacesUtils.history.TRANSITION_BOOKMARK, + false, 0); - do_test_pending(); + // Add a last page and wait for its frecency. + PlacesUtils.history.addVisit(uri("http://frecency.mozilla.org/"), Date.now(), + null, Ci.nsINavHistoryService.TRANSITION_LINK, + false, 0); + waitForFrecency("http://frecency.mozilla.org/", function (aFrecency) aFrecency > 0, + function () { + PlacesUtils.bhistory.removeAllPages(); + }, this, []); } diff --git a/toolkit/components/places/tests/unit/test_migrateFrecency.js b/toolkit/components/places/tests/unit/test_migrateFrecency.js index 9750cfea2d8..91cc5272efb 100644 --- a/toolkit/components/places/tests/unit/test_migrateFrecency.js +++ b/toolkit/components/places/tests/unit/test_migrateFrecency.js @@ -72,7 +72,7 @@ function run_test() { _("Now that places has migrated, check that it calculated frecencies"); var stmt = places.DBConnection.createStatement( - "SELECT COUNT(*) FROM moz_places_view WHERE frecency < 0"); + "SELECT COUNT(*) FROM moz_places WHERE frecency < 0"); stmt.executeAsync({ handleResult: function(results) { _("Should always get a result from COUNT(*)"); diff --git a/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js b/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js index bc01033f389..9cb418f147e 100644 --- a/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js +++ b/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js @@ -36,18 +36,14 @@ * * ***** END LICENSE BLOCK ***** */ -const bmsvc = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -const histsvc = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); - -const dbConn = Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsPIPlacesDatabase). - DBConnection; +const bmsvc = PlacesUtils.bookmarks; +const histsvc = PlacesUtils.history; const NOW = Date.now() * 1000; -const TEST_URI = uri("http://example.com/"); -const PLACE_URI = uri("place:queryType=0&sort=8&maxResults=10"); +const TEST_URL = "http://example.com/"; +const TEST_URI = uri(TEST_URL); +const PLACE_URL = "place:queryType=0&sort=8&maxResults=10"; +const PLACE_URI = uri(PLACE_URL); var gTests = [ { @@ -62,16 +58,16 @@ var gTests = [ false, 0); } - - print("Get frecency."); - var frecency = getFrecencyForURI(TEST_URI); - + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + this.continue_run, this); + }, + continue_run: function () { print("Remove visits using timerange outside the URI's visits."); histsvc.QueryInterface(Ci.nsIBrowserHistory). removeVisitsByTimeframe(NOW - 10, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(TEST_URI)); + do_check_true(page_in_database(TEST_URL)); print("Run a history query and check that all visits still exist."); var query = histsvc.getNewQuery(); @@ -92,7 +88,8 @@ var gTests = [ isVisited(TEST_URI)); print("Frecency should be unchanged."); - do_check_eq(getFrecencyForURI(TEST_URI), frecency); + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + run_next_test, Components.utils.getGlobalForObject(this)); } }, @@ -115,15 +112,16 @@ var gTests = [ bmsvc.DEFAULT_INDEX, "bookmark title"); - print("Get frecency."); - var frecency = getFrecencyForURI(TEST_URI); - + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + this.continue_run, this); + }, + continue_run: function () { print("Remove visits using timerange outside the URI's visits."); histsvc.QueryInterface(Ci.nsIBrowserHistory). removeVisitsByTimeframe(NOW - 10, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(TEST_URI)); + do_check_true(page_in_database(TEST_URL)); print("Run a history query and check that all visits still exist."); var query = histsvc.getNewQuery(); @@ -144,7 +142,8 @@ var gTests = [ isVisited(TEST_URI)); print("Frecency should be unchanged."); - do_check_eq(getFrecencyForURI(TEST_URI), frecency); + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + run_next_test, Components.utils.getGlobalForObject(this)); } }, @@ -160,16 +159,16 @@ var gTests = [ false, 0); } - - print("Get frecency."); - var frecency = getFrecencyForURI(TEST_URI); - + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + this.continue_run, this); + }, + continue_run: function () { print("Remove the 5 most recent visits."); histsvc.QueryInterface(Ci.nsIBrowserHistory). removeVisitsByTimeframe(NOW - 4, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(TEST_URI)); + do_check_true(page_in_database(TEST_URL)); print("Run a history query and check that only the older 5 visits " + "still exist."); @@ -191,7 +190,8 @@ var gTests = [ isVisited(TEST_URI)); print("Frecency should be unchanged."); - do_check_eq(getFrecencyForURI(TEST_URI), frecency); + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + run_next_test, Components.utils.getGlobalForObject(this)); } }, @@ -213,16 +213,16 @@ var gTests = [ TEST_URI, bmsvc.DEFAULT_INDEX, "bookmark title"); - - print("Get frecency."); - var frecency = getFrecencyForURI(TEST_URI); - + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + this.continue_run, this); + }, + continue_run: function () { print("Remove the 5 most recent visits."); histsvc.QueryInterface(Ci.nsIBrowserHistory). removeVisitsByTimeframe(NOW - 4, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(TEST_URI)); + do_check_true(page_in_database(TEST_URL)); print("Run a history query and check that only the older 5 visits " + "still exist."); @@ -244,7 +244,8 @@ var gTests = [ isVisited(TEST_URI)); print("Frecency should be unchanged."); - do_check_eq(getFrecencyForURI(TEST_URI), frecency); + waitForFrecency(TEST_URL, function (aFrecency) aFrecency > 0, + run_next_test, Components.utils.getGlobalForObject(this)); } }, @@ -266,7 +267,7 @@ var gTests = [ removeVisitsByTimeframe(NOW - 10, NOW); print("URI should no longer exist in moz_places."); - do_check_false(uriExistsInMozPlaces(TEST_URI)); + do_check_false(page_in_database(TEST_URL)); print("Run a history query and check that no visits exist."); var query = histsvc.getNewQuery(); @@ -281,6 +282,7 @@ var gTests = [ print("nsIGlobalHistory2.isVisited should return false."); do_check_false(histsvc.QueryInterface(Ci.nsIGlobalHistory2). isVisited(TEST_URI)); + run_next_test(); } }, @@ -302,7 +304,7 @@ var gTests = [ removeVisitsByTimeframe(NOW - 10, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(PLACE_URI)); + do_check_true(page_in_database(PLACE_URL)); print("Run a history query and check that no visits exist."); var query = histsvc.getNewQuery(); @@ -319,7 +321,8 @@ var gTests = [ isVisited(PLACE_URI)); print("Frecency should be 0."); - do_check_eq(getFrecencyForURI(PLACE_URI), 0); + waitForFrecency(PLACE_URL, function (aFrecency) aFrecency == 0, + run_next_test, Components.utils.getGlobalForObject(this)); } }, @@ -347,7 +350,7 @@ var gTests = [ removeVisitsByTimeframe(NOW - 10, NOW); print("URI should still exist in moz_places."); - do_check_true(uriExistsInMozPlaces(TEST_URI)); + do_check_true(page_in_database(TEST_URL)); print("Run a history query and check that no visits exist."); var query = histsvc.getNewQuery(); @@ -367,62 +370,31 @@ var gTests = [ do_check_true(bmsvc.isBookmarked(TEST_URI)); print("Frecency should be -visit_count == -10."); - do_check_eq(getFrecencyForURI(TEST_URI), -10); + waitForFrecency(TEST_URL, function (aFrecency) aFrecency == -10, + run_next_test, Components.utils.getGlobalForObject(this)); } } ]; /////////////////////////////////////////////////////////////////////////////// -/** - * Removes history and bookmarks. - */ -function deleteAllHistoryAndBookmarks() { - histsvc.QueryInterface(Ci.nsIBrowserHistory).removeAllPages(); - remove_all_bookmarks(); +function run_test() +{ + do_test_pending(); + run_next_test(); } -/** - * Returns the frecency of a URI. - * - * @param aURI - * the URI of a place - * @return the frecency of aURI - */ -function getFrecencyForURI(aURI) { - let sql = "SELECT frecency FROM moz_places_view WHERE url = :url"; - let stmt = dbConn.createStatement(sql); - stmt.params.url = aURI.spec; - do_check_true(stmt.executeStep()); - let frecency = stmt.getInt32(0); - stmt.finalize(); - - return frecency; -} - -/** - * Returns true if the URI exists in moz_places and false otherwise. - * - * @param aURI - * the URI of a place - */ -function uriExistsInMozPlaces(aURI) { - let sql = "SELECT id FROM moz_places_view WHERE url = :url"; - let stmt = dbConn.createStatement(sql); - stmt.params.url = aURI.spec; - var exists = stmt.executeStep(); - stmt.finalize(); - - return exists; -} - -/////////////////////////////////////////////////////////////////////////////// - -function run_test() { - gTests.forEach(function (t) { - deleteAllHistoryAndBookmarks(); - print("------ RUNNING TEST: " + t.desc); - t.run(); - }); - deleteAllHistoryAndBookmarks(); +function run_next_test() { + if (gTests.length) { + let test = gTests.shift(); + print("\n ***Test: " + test.desc); + waitForClearHistory(function() { + DBConn().executeSimpleSQL("DELETE FROM moz_places"); + remove_all_bookmarks(); + test.run.call(test); + }); + } + else { + do_test_finished(); + } } diff --git a/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js b/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js index a6b7658ebd8..ff3be812d25 100644 --- a/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js +++ b/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js @@ -45,27 +45,17 @@ * bookmark is deleted. */ -const bmServ = - Cc["@mozilla.org/browser/nav-bookmarks-service;1"]. - getService(Ci.nsINavBookmarksService); -const histServ = - Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsINavHistoryService); -const lmServ = - Cc["@mozilla.org/browser/livemark-service;2"]. - getService(Ci.nsILivemarkService); +const bmServ = PlacesUtils.bookmarks; +const histServ = PlacesUtils.history; +const lmServ = PlacesUtils.livemarks; -const dbConn = - Cc["@mozilla.org/browser/nav-history-service;1"]. - getService(Ci.nsPIPlacesDatabase). - DBConnection; +let tests = [ -var tests = []; - -tests.push({ +{ desc: ["Frecency of unvisited, separately bookmarked livemark item's URI ", "should be zero after bookmark removed."].join(""), - run: function () { + run: function () + { // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. let lmItemURL = "http://example.com/livemark-item"; @@ -75,22 +65,31 @@ tests.push({ lmItemURI, bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(lmItemURL), 0); - - bmServ.removeItem(bmId); - - // URI's only "bookmark" is now unvisited livemark item => frecency = 0. - do_check_eq(getFrecency(lmItemURL), 0); - runNextTest(); + bmServ.removeItem(aItemId); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now unvisited livemark item => frecency = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["Frecency of visited, separately bookmarked livemark item's URI ", "should not be zero after bookmark removed."].join(""), - run: function () { + run: function () + { // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. let lmItemURL = "http://example.com/livemark-item"; @@ -100,44 +99,62 @@ tests.push({ lmItemURI, bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(lmItemURL), 0); - - visit(lmItemURI); - bmServ.removeItem(bmId); - - // URI's only "bookmark" is now *visited* livemark item => frecency != 0. - do_check_neq(getFrecency(lmItemURL), 0); - runNextTest(); + visit(uri(aUrl)); + bmServ.removeItem(aItemId); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now *visited* livemark item => frecency != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing bookmark, frecency of bookmark's URI should be zero ", "if URI is unvisited and no longer bookmarked."].join(""), - run: function () { + run: function () + { let url = "http://example.com/1"; let bmId = bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, uri(url), bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(url, function(aFrecency) aFrecency > 0, + this.check1, this, [url, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(url), 0); - - bmServ.removeItem(bmId); - - // Unvisited URI no longer bookmarked => frecency should = 0. - do_check_eq(getFrecency(url), 0); - runNextTest(); + bmServ.removeItem(aItemId); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("Unvisited URI no longer bookmarked => frecency should = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing bookmark, frecency of bookmark's URI should not be ", "zero if URI is visited."].join(""), - run: function () { + run: function () + { let bmURL = "http://example.com/1"; let bmURI = uri(bmURL); @@ -145,23 +162,32 @@ tests.push({ bmURI, bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL, bmId]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(bmURL), 0); - - visit(bmURI); - bmServ.removeItem(bmId); - - // *Visited* URI no longer bookmarked => frecency should != 0. - do_check_neq(getFrecency(bmURL), 0); - runNextTest(); + visit(uri(aUrl)); + bmServ.removeItem(aItemId); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("*Visited* URI no longer bookmarked => frecency should != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing bookmark, frecency of bookmark's URI should not be ", "zero if URI is still bookmarked."].join(""), - run: function () { + run: function () + { let bmURL = "http://example.com/1"; let bmURI = uri(bmURL); @@ -174,23 +200,32 @@ tests.push({ bmURI, bmServ.DEFAULT_INDEX, "bookmark 2 title"); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL, bm1Id]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(bmURL), 0); - - bmServ.removeItem(bm1Id); - - // URI still bookmarked => frecency should != 0. - do_check_neq(getFrecency(bmURL), 0); - runNextTest(); + bmServ.removeItem(aItemId); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI still bookmarked => frecency should != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["Frecency of unvisited, separately bookmarked livemark item's URI ", "should be zero after all children removed from bookmark's ", "parent."].join(""), - run: function () { + run: function () + { // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. let lmItemURL = "http://example.com/livemark-item"; @@ -201,23 +236,32 @@ tests.push({ lmItemURI, bmServ.DEFAULT_INDEX, "bookmark title"); - - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(lmItemURL), 0); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL]); + }, + check1: function (aUrl) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); bmServ.removeFolderChildren(bmServ.unfiledBookmarksFolder); - - // URI's only "bookmark" is now unvisited livemark item => frecency = 0. - do_check_eq(getFrecency(lmItemURL), 0); - runNextTest(); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now unvisited livemark item => frecency = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["Frecency of visited, separately bookmarked livemark item's URI ", "should not be zero after all children removed from bookmark's ", "parent."].join(""), - run: function () { + run: function () + { // Add livemark and bookmark. Bookmark's URI is the URI of the livemark's // only item. let lmItemURL = "http://example.com/livemark-item"; @@ -227,45 +271,63 @@ tests.push({ lmItemURI, bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(lmItemURL, function(aFrecency) aFrecency > 0, + this.check1, this, [lmItemURL]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(lmItemURL), 0); - - visit(lmItemURI); + visit(uri(aUrl)); bmServ.removeFolderChildren(bmServ.unfiledBookmarksFolder); - - // URI's only "bookmark" is now *visited* livemark item => frecency != 0. - do_check_neq(getFrecency(lmItemURL), 0); - runNextTest(); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("URI's only bookmark is now *visited* livemark item => frecency != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing all children from bookmark's parent, frecency of ", "bookmark's URI should be zero if URI is unvisited and no longer ", "bookmarked."].join(""), - run: function () { + run: function () + { let url = "http://example.com/1"; bmServ.insertBookmark(bmServ.unfiledBookmarksFolder, uri(url), bmServ.DEFAULT_INDEX, "bookmark title"); - - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(url), 0); + waitForFrecency(url, function(aFrecency) aFrecency > 0, + this.check1, this, [url]); + }, + check1: function (aUrl, aItemId) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); bmServ.removeFolderChildren(bmServ.unfiledBookmarksFolder); - - // Unvisited URI no longer bookmarked => frecency should = 0. - do_check_eq(getFrecency(url), 0); - runNextTest(); + waitForFrecency(aUrl, function(aFrecency) aFrecency == 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("Unvisited URI no longer bookmarked => frecency should = 0"); + do_check_eq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing all children from bookmark's parent, frecency of ", "bookmark's URI should not be zero if URI is visited."].join(""), - run: function () { + run: function () + { let bmURL = "http://example.com/1"; let bmURI = uri(bmURL); @@ -273,24 +335,33 @@ tests.push({ bmURI, bmServ.DEFAULT_INDEX, "bookmark title"); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL]); + }, + check1: function (aUrl) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(bmURL), 0); - - visit(bmURI); + visit(uri(aUrl)); bmServ.removeFolderChildren(bmServ.unfiledBookmarksFolder); - - // *Visited* URI no longer bookmarked => frecency should != 0. - do_check_neq(getFrecency(bmURL), 0); - runNextTest(); + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { + print("*Visited* URI no longer bookmarked => frecency should != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, -tests.push({ +{ desc: ["After removing all children from bookmark's parent, frecency of ", "bookmark's URI should not be zero if URI is still ", "bookmarked."].join(""), - run: function () { + run: function () + { let bmURL = "http://example.com/1"; let bmURI = uri(bmURL); @@ -303,17 +374,27 @@ tests.push({ bmURI, bmServ.DEFAULT_INDEX, "bookmark 2 title"); - - // Bookmarked => frecency of URI should be != 0. - do_check_neq(getFrecency(bmURL), 0); + waitForFrecency(bmURL, function(aFrecency) aFrecency > 0, + this.check1, this, [bmURL]); + }, + check1: function (aUrl) + { + print("Bookmarked => frecency of URI should be != 0"); + do_check_neq(frecencyForUrl(aUrl), 0); bmServ.removeFolderChildren(bmServ.unfiledBookmarksFolder); - + waitForFrecency(aUrl, function(aFrecency) aFrecency > 0, + this.check2, this, [aUrl]); + }, + check2: function (aUrl) + { // URI still bookmarked => frecency should != 0. - do_check_neq(getFrecency(bmURL), 0); - runNextTest(); + do_check_neq(frecencyForUrl(aUrl), 0); + run_next_test(); } -}); +}, + +]; /////////////////////////////////////////////////////////////////////////////// @@ -330,30 +411,10 @@ function createLivemark(aLmChildItemURI) { uri("http://example.com/"), uri("http://example.com/rdf"), -1); - let lmChildItemId = bmServ.insertBookmark(lmItemId, - aLmChildItemURI, - bmServ.DEFAULT_INDEX, - "livemark item title"); - return lmChildItemId; -} - -/** - * Returns the frecency of a Place. - * - * @param aURL - * the URL of a Place - * @return the frecency of aURL - */ -function getFrecency(aURL) { - let sql = "SELECT frecency FROM moz_places_view WHERE url = :url"; - let stmt = dbConn.createStatement(sql); - stmt.params.url = aURL; - do_check_true(stmt.executeStep()); - let frecency = stmt.getInt32(0); - print("frecency=" + frecency); - stmt.finalize(); - - return frecency; + return bmServ.insertBookmark(lmItemId, + aLmChildItemURI, + bmServ.DEFAULT_INDEX, + "livemark item title"); } /** @@ -375,15 +436,18 @@ function visit(aURI) { function run_test() { do_test_pending(); - runNextTest(); + run_next_test(); } -function runNextTest() { +function run_next_test() { if (tests.length) { let test = tests.shift(); - print("Test " + + ": " + test.desc); - remove_all_bookmarks(); - waitForClearHistory(test.run); + print("\n ***Test: " + test.desc); + waitForClearHistory(function() { + DBConn().executeSimpleSQL("DELETE FROM moz_places"); + remove_all_bookmarks(); + test.run.call(test); + }); } else { do_test_finished();