diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f4517ac4..d7722c86 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -36,7 +36,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible. # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. # SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4") +# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/database/common.c b/src/database/common.c index 4fd30b6c..a081fc2e 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -251,9 +251,15 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) generate_backtrace(); if(iErrCode == SQLITE_WARNING) - log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode); + log_warn("SQLite3: %s (%d)", zMsg, iErrCode); + else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) + // SQLITE_SCHEMA is returned when the database schema has changed + // This is not necessarily an error, as sqlite3_step() will re-prepare + // the statement and try again. If it cannot, it will return an error + // and this will be handled over there. + log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode); else - log_err("SQLite3 message: %s (%d)", zMsg, iErrCode); + log_err("SQLite3: %s (%d)", zMsg, iErrCode); } void db_init(void) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 40c10853..d4e39267 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -56,10 +56,23 @@ static bool delete_old_queries_in_DB(sqlite3 *db) return true; } -static bool optimize_database(sqlite3 *db) +static bool analyze_database(sqlite3 *db) { - // Optimize the database by running PRAGMA optimize - SQL_bool(db, "PRAGMA optimize;"); + // Optimize the database by running ANALYZE + // The ANALYZE command gathers statistics about tables and indices and + // stores the collected information in internal tables of the database + // where the query optimizer can access the information and use it to + // help make better query planning choices. + + // Measure time + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); + SQL_bool(db, "ANALYZE;"); + clock_gettime(CLOCK_MONOTONIC, &end); + + // Print final message + log_info("Optimized database in %.3f seconds", + (double)(end.tv_sec - start.tv_sec) + 1e-9*(end.tv_nsec - start.tv_nsec)); return true; } @@ -81,7 +94,7 @@ void *DB_thread(void *val) time_t lastDBsave = before - before%config.database.DBinterval.v.ui; // Other timestamps - time_t lastOptimize = before; + time_t lastAnalyze = before; time_t lastMACVendor = before; // This thread runs until shutdown of the process. We keep this thread @@ -151,11 +164,12 @@ void *DB_thread(void *val) if(killed) break; - // Optimize database every 24 hours - if(now - lastOptimize >= 86400) + // Optimize database once per week + if(now - lastAnalyze >= 604800) { DBOPEN_OR_AGAIN(); - optimize_database(db); + analyze_database(db); + lastAnalyze = now; DBCLOSE_OR_BREAK(); } @@ -169,6 +183,7 @@ void *DB_thread(void *val) { DBOPEN_OR_AGAIN(); updateMACVendorRecords(db); + lastMACVendor = now; DBCLOSE_OR_BREAK(); } diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index a47d5db7..10a0d934 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -953,22 +953,6 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&gravity_stmt); free_sqlite3_stmt_vec(&antigravity_stmt); - // Run PRAMGMA optimize to optimize database file - // It is recommended to run this command on a regular basis - // when closing the database connection - // See https://www.sqlite.org/pragma.html#pragma_optimize - // We set a small value of analysis_limit=1000 to ensure the - // command returns quickly (it limits the number of rows - // analyzed by the query planner to 1000) - const char *querystr = "PRAGMA analysis_limit = 1000;"; - int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); - querystr = "PRAGMA optimize;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); - // Close table sqlite3_close(gravity_db); gravity_db = NULL;