Run ANALYZE instead of PRAMGA optimize after some discussion with the SQlite3 developers. Also ensure notices and mere messages are not always logged as errors in FTL's log. Furhtermore, reduce the frequency of running ANALYZE from once per day to once per week.

Signed-off-by: DL6ER <dl6er@dl6er.de>

src/CMakeLists.txt

@@ -36,7 +36,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
 # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
 # SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4")
+# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")
 
 # Code hardening and debugging improvements
 # -fstack-protector-strong: The program will be resistant to having its stack overflowed

src/database/common.c

@@ -251,9 +251,15 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg)
 		generate_backtrace();
 
 	if(iErrCode == SQLITE_WARNING)
-		log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode);
+		log_warn("SQLite3: %s (%d)", zMsg, iErrCode);
+	else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA)
+		// SQLITE_SCHEMA is returned when the database schema has changed
+		// This is not necessarily an error, as sqlite3_step() will re-prepare
+		// the statement and try again. If it cannot, it will return an error
+		// and this will be handled over there.
+		log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode);
 	else
-		log_err("SQLite3 message: %s (%d)", zMsg, iErrCode);
+		log_err("SQLite3: %s (%d)", zMsg, iErrCode);
 }
 
 void db_init(void)

src/database/database-thread.c

@@ -56,10 +56,23 @@ static bool delete_old_queries_in_DB(sqlite3 *db)
 	return true;
 }
 
-static bool optimize_database(sqlite3 *db)
+static bool analyze_database(sqlite3 *db)
 {
-	// Optimize the database by running PRAGMA optimize
-	SQL_bool(db, "PRAGMA optimize;");
+	// Optimize the database by running ANALYZE
+	// The ANALYZE command gathers statistics about tables and indices and
+	// stores the collected information in internal tables of the database
+	// where the query optimizer can access the information and use it to
+	// help make better query planning choices.
+
+	// Measure time
+	struct timespec start, end;
+	clock_gettime(CLOCK_MONOTONIC, &start);
+	SQL_bool(db, "ANALYZE;");
+	clock_gettime(CLOCK_MONOTONIC, &end);
+
+	// Print final message
+	log_info("Optimized database in %.3f seconds",
+	         (double)(end.tv_sec - start.tv_sec) + 1e-9*(end.tv_nsec - start.tv_nsec));
 
 	return true;
 }
@@ -81,7 +94,7 @@ void *DB_thread(void *val)
 	time_t lastDBsave = before - before%config.database.DBinterval.v.ui;
 
 	// Other timestamps
-	time_t lastOptimize = before;
+	time_t lastAnalyze = before;
 	time_t lastMACVendor = before;
 
 	// This thread runs until shutdown of the process. We keep this thread
@@ -151,11 +164,12 @@ void *DB_thread(void *val)
 		if(killed)
 			break;
 
-		// Optimize database every 24 hours
-		if(now - lastOptimize >= 86400)
+		// Optimize database once per week
+		if(now - lastAnalyze >= 604800)
 		{
 			DBOPEN_OR_AGAIN();
-			optimize_database(db);
+			analyze_database(db);
+			lastAnalyze = now;
 			DBCLOSE_OR_BREAK();
 		}
 
@@ -169,6 +183,7 @@ void *DB_thread(void *val)
 		{
 			DBOPEN_OR_AGAIN();
 			updateMACVendorRecords(db);
+			lastMACVendor = now;
 			DBCLOSE_OR_BREAK();
 		}
 

src/database/gravity-db.c

@@ -953,22 +953,6 @@ void gravityDB_close(void)
 	free_sqlite3_stmt_vec(&gravity_stmt);
 	free_sqlite3_stmt_vec(&antigravity_stmt);
 
-	// Run PRAMGMA optimize to optimize database file
-	// It is recommended to run this command on a regular basis
-	// when closing the database connection
-	// See https://www.sqlite.org/pragma.html#pragma_optimize
-	// We set a small value of analysis_limit=1000 to ensure the
-	// command returns quickly (it limits the number of rows
-	// analyzed by the query planner to 1000)
-	const char *querystr = "PRAGMA analysis_limit = 1000;";
-	int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
-	if(rc != SQLITE_OK)
-		log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc));
-	querystr = "PRAGMA optimize;";
-	rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
-	if(rc != SQLITE_OK)
-		log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc));
-
 	// Close table
 	sqlite3_close(gravity_db);
 	gravity_db = NULL;