From 0fa1def3bb0eb3f770feb2a789bfe6b79168ca88 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Feb 2021 11:51:41 +0100 Subject: [PATCH] Add a second in-memory database to ensure non-blocking operation at all times Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 +- src/api/ftl.c | 2 +- src/database/database-thread.c | 15 ++- src/database/query-table.c | 172 +++++++++++++++++++++++++-------- src/database/query-table.h | 5 +- src/gc.c | 4 +- src/main.c | 8 +- 7 files changed, 159 insertions(+), 50 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c38b0763..570bb5cf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,7 +28,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime). # SQLITE_DEFAULT_CACHE_SIZE=-8000: Set SQLite3's cache to 8MB (instead of 2MB which is the default) -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=-8000") +# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=-8000 -DSQLITE_USE_URI=1") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/api/ftl.c b/src/api/ftl.c index a229308b..c8e16a14 100644 --- a/src/api/ftl.c +++ b/src/api/ftl.c @@ -184,7 +184,7 @@ int api_ftl_dbinfo(struct ftl_conn *api) JSON_OBJ_ADD_ITEM(json, "owner", owner); // Add number of queries in on-disk database - const int queries_in_database = get_number_of_queries_in_DB(true); + const int queries_in_database = get_number_of_queries_in_DB(NULL, true); JSON_OBJ_ADD_NUMBER(json, "queries", queries_in_database); // Add SQLite library version diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 9c3b8386..40d2395e 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -60,17 +60,28 @@ void *DB_thread(void *val) // Save timestamp as we do not want to store immediately // to the database - time_t lastDBsave = time(NULL) - time(NULL)%config.DBinterval; + time_t before = time(NULL); + time_t lastDBsave = before - before%config.DBinterval; while(!killed) { if(FTL_DB_avail()) { time_t now = time(NULL); + + // Move queries from non-blocking newdb into the larger memdb + // Do this once per second + if(now > before) + { + mv_newdb_memdb(); + before = now; + } + + // Store queries in on-disk database if(now - lastDBsave >= config.DBinterval) { // Update lastDBsave timer - lastDBsave = time(NULL) - time(NULL)%config.DBinterval; + lastDBsave = now - now%config.DBinterval; // Save data to database (if enabled) if(config.DBexport) diff --git a/src/database/query-table.c b/src/database/query-table.c index 2f207f97..fa568a73 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -22,67 +22,114 @@ #include "common.h" #include "../timers.h" -static sqlite3 *memdb = NULL; +static sqlite3 *memdb = NULL, *newdb = NULL; static double new_last_timestamp = 0; static unsigned int new_total = 0, new_blocked = 0; static long last_mem_db_idx = 0, last_disk_db_idx = 0; -// Initialize in-memory database and add queries table -bool init_memory_database(void) +// Initialize in-memory database, add queries table and indices +static bool init_memory_database(sqlite3 **db, const char *name, const int busy) { int rc; - // Try to open database - rc = sqlite3_open_v2(":memory:", &memdb, SQLITE_OPEN_READWRITE, NULL); + // Try to open in-memory database + rc = sqlite3_open_v2(name, db, SQLITE_OPEN_READWRITE, NULL); if( rc != SQLITE_OK ) { - logg("init_memory_database(): Step error while trying to open in-memory database: %s", - sqlite3_errstr(rc)); + logg("init_memory_database(): Step error while trying to open %s database: %s", + name, sqlite3_errstr(rc)); return false; } // Explicitly set busy handler to value defined in FTL.h - rc = sqlite3_busy_timeout(memdb, DATABASE_BUSY_TIMEOUT); + rc = sqlite3_busy_timeout(*db, busy); if( rc != SQLITE_OK ) { - logg("init_memory_database(): Step error while trying to set busy timeout (%d ms) on in-memory database: %s", - DATABASE_BUSY_TIMEOUT, sqlite3_errstr(rc)); - sqlite3_close(memdb); + logg("init_memory_database(): Step error while trying to set busy timeout (%d ms) on %s database: %s", + DATABASE_BUSY_TIMEOUT, name, sqlite3_errstr(rc)); + sqlite3_close(*db); return false; } // Create queries table in the database - rc = sqlite3_exec(memdb, CREATE_QUERIES_TABLE_V7, NULL, NULL, NULL); + rc = sqlite3_exec(*db, CREATE_QUERIES_TABLE_V7, NULL, NULL, NULL); if( rc != SQLITE_OK ){ - logg("init_memory_database(\"%s\") failed: %s", - CREATE_QUERIES_TABLE_V7, sqlite3_errstr(rc)); - sqlite3_close(memdb); + logg("init_memory_database(%s: \"%s\") failed: %s", + name, CREATE_QUERIES_TABLE_V7, sqlite3_errstr(rc)); + sqlite3_close(*db); return false; } // Add indices on all columns of the in-memory database for(unsigned int i = 0; i < ArraySize(index_creation); i++) { - rc = sqlite3_exec(memdb, index_creation[i], NULL, NULL, NULL); + rc = sqlite3_exec(*db, index_creation[i], NULL, NULL, NULL); if( rc != SQLITE_OK ){ - logg("init_memory_database(\"%s\") failed: %s", - index_creation[i], sqlite3_errstr(rc)); - sqlite3_close(memdb); + logg("init_memory_database(%s: \"%s\") failed: %s", + name, index_creation[i], sqlite3_errstr(rc)); + sqlite3_close(*db); return false; } } + // Everything went well + return true; +} + +// Initialize in-memory databases +// The flow of queries is as follows: +// 1. A new query is always added to the special new.queries table This table +// is only used for storing new queries and does never block because of +// not allowing (externally triggered) SELECT statements. This ensures we +// can always add new queries even when the in-memory queries table is +// currently busy (e.g., a complex SELECT statement is running from the API) +// 2. Every second, we try to copy all queries from new.queries into queries. +// When successful, we delete the queries in new.queries afterwards. This +// operation may fail if either of the tables is currently busy. This isn't +// an issue as the queries are simply preserved and we try again on the next +// second. This ensures the in-memory database isn't updated midway when an +// API query is running. Furthermore, it ensures that new queries are not +// blocked when the database is busy and INSERTions aren't currently possible. +// 3. At user-configured intervals, the in-memory database is dumped on-disk. +// For this, we +// 3.1. Attach the on-disk database +// 3.2. INSERT the queries that came in since the last dumping +// 3.3. Detach the on-disk database +// 4. At the end of their lifetime (that is after 24 hours), queries are DELETEd +// from the in-memory database to make room for new queries in the rolling +// window. The queries are not removed from the on-disk database. +bool init_memory_databases(void) +{ + // Initialize in-memory database for all queries + if(!init_memory_database(&memdb, "file:memdb?mode=memory", DATABASE_BUSY_TIMEOUT)) + return false; + // Initialize in-memory database for new queries + if(!init_memory_database(&newdb, "file:newdb?mode=memory&cache=shared", 0)) + return false; + + logg("memdb: %p, newdb: %p", memdb, newdb); + + // ATTACH newdb to memdb + const char *querystr = "ATTACH 'file:newdb?mode=memory&cache=shared' AS new"; + int rc = sqlite3_exec(memdb, querystr, NULL, NULL, NULL); + if( rc != SQLITE_OK ){ + logg("init_memory_databases(\"%s\") failed: %s", + querystr, sqlite3_errstr(rc)); + return false; + } + return true; } -static bool get_memdb_size(size_t *memsize, int *queries) +// Get memory usage and size of in-memory tables +static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries) { int rc; sqlite3_stmt* stmt = NULL; size_t page_count, page_size; // PRAGMA page_count - rc = sqlite3_prepare_v2(memdb, "PRAGMA page_count", -1, &stmt, NULL); + rc = sqlite3_prepare_v2(db, "PRAGMA page_count", -1, &stmt, NULL); if( rc != SQLITE_OK ) { if( rc != SQLITE_BUSY ) @@ -103,7 +150,7 @@ static bool get_memdb_size(size_t *memsize, int *queries) sqlite3_finalize(stmt); // PRAGMA page_size - rc = sqlite3_prepare_v2(memdb, "PRAGMA page_size", -1, &stmt, NULL); + rc = sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &stmt, NULL); if( rc != SQLITE_OK ) { if( rc != SQLITE_BUSY ) @@ -126,26 +173,36 @@ static bool get_memdb_size(size_t *memsize, int *queries) *memsize = page_count * page_size; // Get number of queries in the memory table - if((*queries = get_number_of_queries_in_DB(false)) == DB_FAILED) + if((*queries = get_number_of_queries_in_DB(db, false)) == DB_FAILED) return false; return true; } +// Log the memory usage of in-memory databases static void log_in_memory_usage(void) { size_t memsize = 0; int queries = 0; - if(get_memdb_size(&memsize, &queries)) + if(get_memdb_size(newdb, &memsize, &queries)) { char prefix[2] = { 0 }; double num = 0.0; format_memory_size(prefix, memsize, &num); - logg("In-memory database size: %.1f%s (%d queries)", + logg("new database size: %.1f%s (%d queries)", + num, prefix, queries); + } + if(get_memdb_size(memdb, &memsize, &queries)) + { + char prefix[2] = { 0 }; + double num = 0.0; + format_memory_size(prefix, memsize, &num); + logg("mem database size: %.1f%s (%d queries)", num, prefix, queries); } } +// Attach disk database to in-memory database static bool attach_disk_database(void) { int rc; @@ -153,7 +210,7 @@ static bool attach_disk_database(void) sqlite3_stmt* stmt = NULL; // ATTACH database file on-disk - rc = sqlite3_prepare_v2(memdb, "ATTACH :path AS disk", -1, &stmt, NULL); + rc = sqlite3_prepare_v2(memdb, "ATTACH ? AS disk", -1, &stmt, NULL); if( rc != SQLITE_OK ) { if( rc != SQLITE_BUSY ) @@ -161,9 +218,8 @@ static bool attach_disk_database(void) return false; } - // Bind type to prepared statement (if requested) - const int path_idx = sqlite3_bind_parameter_index(stmt, ":path"); - if((rc = sqlite3_bind_text(stmt, path_idx, FTLfiles.FTL_db, -1, SQLITE_STATIC)) != SQLITE_OK) + // Bind path to prepared index + if((rc = sqlite3_bind_text(stmt, 1, FTLfiles.FTL_db, -1, SQLITE_STATIC)) != SQLITE_OK) { logg("attach_disk_database(): Failed to bind path: %s", sqlite3_errstr(rc)); @@ -185,6 +241,7 @@ static bool attach_disk_database(void) return okay; } +// Detach disk database to in-memory database static bool detach_disk_database(void) { int rc; @@ -201,9 +258,9 @@ static bool detach_disk_database(void) return true; } -// Get number of queries either in the in-memory or in the on-diks database +// Get number of queries either in the temp or in the on-diks database // This routine is used by the API routines. -int get_number_of_queries_in_DB(bool disk) +int get_number_of_queries_in_DB(sqlite3 *db, bool disk) { int rc = 0, num = 0; sqlite3_stmt *stmt = NULL; @@ -211,16 +268,22 @@ int get_number_of_queries_in_DB(bool disk) if(disk && !attach_disk_database()) return DB_FAILED; - // Count number of rows using the index timestamp is faster than select(*) - const char *querystr = disk ? "SELECT COUNT(timestamp) FROM disk.queries" : "SELECT COUNT(timestamp) FROM queries"; + // Count number of rows + const char *querystr = disk ? + "SELECT COUNT(*) FROM disk.queries" : + "SELECT COUNT(*) FROM queries"; + + // The database pointer may be NULL, meaning we want the memdb + if(db == NULL) + db = memdb; // PRAGMA page_size - rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); + rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ) { if( rc != SQLITE_BUSY ) logg("get_number_of_queries_in_DB(): Prepare error: %s", - sqlite3_errstr(rc)); + sqlite3_errstr(rc)); return false; } @@ -230,7 +293,7 @@ int get_number_of_queries_in_DB(bool disk) else { logg("get_number_of_queries_in_DB(): Step error: %s", - sqlite3_errstr(rc)); + sqlite3_errstr(rc)); return false; } sqlite3_finalize(stmt); @@ -241,6 +304,8 @@ int get_number_of_queries_in_DB(bool disk) return num; } +// Read queries from the on-disk database into the in-memory database (after +// restart, etc.) bool import_queries_from_disk(void) { // Get time stamp 24 hours (or what was configured) in the past @@ -285,6 +350,8 @@ bool import_queries_from_disk(void) return okay; } +// Export in-memory queries to disk - either due to periodic dumping (final = +// false) or because of a sutdown (final = true) bool export_queries_to_disk(bool final) { // Get time stamp 24 hours (or what was configured) in the past @@ -339,7 +406,7 @@ bool export_queries_to_disk(bool final) if(!detach_disk_database()) return false; - // All in-memory queries were stored to disk, update the IDs + // All temp queries were stored to disk, update the IDs unsigned int saved = last_mem_db_idx - last_disk_db_idx; last_disk_db_idx = last_mem_db_idx; @@ -358,6 +425,7 @@ bool export_queries_to_disk(bool final) return okay; } +// Delete query with given ID from database. Used by garbage collection bool delete_query_from_db(const sqlite3_int64 id) { // Get time stamp 24 hours (or what was configured) in the past @@ -393,7 +461,31 @@ bool delete_query_from_db(const sqlite3_int64 id) return okay; } -// Get most recent 24 hours data from in-memory long-term database +// Move queries from newdb.queries into memdb.queries +// If the database is busy, no moving is happening and queries are retained in +// here until the next try. This ensures we cannot loose queries. +bool mv_newdb_memdb(void) +{ + const char *querystr[] = { "BEGIN TRANSACTION EXCLUSIVE", + "REPLACE INTO queries SELECT * FROM new.queries", + "DELETE FROM new.queries", + "END TRANSACTION" }; + + // Run queries against the database + for(unsigned int i = 0; i < ArraySize(querystr); i++) + { + const int rc = sqlite3_exec(memdb, querystr[i], NULL, NULL, NULL); + if( rc != SQLITE_OK ){ + logg("mv_newdb_memdb(%s) failed: %s", + querystr[i], sqlite3_errstr(rc)); + return false; + } + } + + return true; +} + +// Get most recent 24 hours data from temp long-term database void DB_read_queries(void) { // Prepare request @@ -664,8 +756,8 @@ bool query_to_database(queriesData* query) return true; } - // Start preparing INSERT query - rc = sqlite3_prepare_v2(FTL_db, "REPLACE INTO queries VALUES (?,?,?,?,?,?,?,?)", -1, &stmt, NULL); + // Start preparing query + rc = sqlite3_prepare_v2(newdb, "REPLACE INTO queries VALUES (?,?,?,?,?,?,?,?)", -1, &stmt, NULL); if( rc != SQLITE_OK ) { logg("query_to_database() - SQL error step: %s", sqlite3_errstr(rc)); diff --git a/src/database/query-table.h b/src/database/query-table.h index 056be14a..9c67dc23 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -30,11 +30,12 @@ const char *index_creation[] = { CREATE_QUERIES_TIMESTAMP_INDEX, CREATE_FORWARD_DOMAIN_INDEX }; #endif -bool init_memory_database(void); +bool init_memory_databases(void); bool import_queries_from_disk(void); -int get_number_of_queries_in_DB(bool disk); +int get_number_of_queries_in_DB(sqlite3 *db, bool disk); bool export_queries_to_disk(bool final); bool delete_query_from_db(const sqlite3_int64 id); +bool mv_newdb_memdb(void); void DB_read_queries(void); bool query_to_database(queriesData* query); diff --git a/src/gc.c b/src/gc.c index a7abf42d..509d119a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -201,8 +201,8 @@ void *GC_thread(void *val) // Count removed queries removed++; - // Remove query from queries table (in-memory), - // we can release the lock for this action to + // Remove query from queries table (temp), we + // can release the lock for this action to // prevent blocking the DNS service too long unlock_shm(); delete_query_from_db(query->db); diff --git a/src/main.c b/src/main.c index 5dfd3c35..9d72d5ca 100644 --- a/src/main.c +++ b/src/main.c @@ -77,8 +77,12 @@ int main (int argc, char* argv[]) // Initialize query database (pihole-FTL.db) db_init(); - // Initialize in-memory database - init_memory_database(); + // Initialize in-memory databases + if(!init_memory_databases()) + { + logg("FATAL: Cannot initialize in-memory database."); + return EXIT_FAILURE; + } // Try to import queries from long-term database if available if(config.DBimport)