Move query table related routines into a new module database/query-table.o

Signed-off-by: DL6ER <dl6er@dl6er.de>

FTL.h

@@ -139,10 +139,10 @@ typedef struct {
 	char* log;
 	char* pid;
 	char* port;
-	char* db;
 	char* socketfile;
-	char* gravitydb;
-	char* macvendordb;
+	char* FTL_db;
+	char* gravity_db;
+	char* macvendor_db;
 } FTLFileNamesStruct;
 
 typedef struct {

Makefile

@@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
 FTLDEPS = *.h
-FTLDBOBJ = database/common.o database/gravity.o database/networktable.o
+FTLDBOBJ = database/common.o database/query-table.o database/network-table.o database/gravity.o
 FTLOBJ = $(FTLDBOBJ) main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o files.o setupVars.o args.o gc.o config.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o overTime.o timers.o
 
 DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h

api.c

@@ -19,6 +19,7 @@
 #include "request.h"
 #include "config.h"
 #include "database/common.h"
+#include "database/query-table.h"
 #include "overTime.h"
 #include "api.h"
 #include "version.h"
@@ -1029,13 +1030,7 @@ void getVersion(const int *sock)
 void getDBstats(const int *sock)
 {
 	// Get file details
-	struct stat st;
-	long int filesize = 0;
-	if(stat(FTLfiles.db, &st) != 0)
-		// stat() failed (maybe the file does not exist?)
-		filesize = -1;
-	else
-		filesize = st.st_size;
+	long int filesize = ((long int)1e6*get_FTL_db_filesize());
 
 	char *prefix = calloc(2, sizeof(char));
 	if(prefix == NULL) return;

config.c

@@ -170,20 +170,20 @@ void read_FTLconf(void)
 
 	errno = 0;
 	// Use sscanf() to obtain filename from config file parameter only if buffer != NULL
-	if(!(buffer != NULL && sscanf(buffer, "%127ms", &FTLfiles.db)))
+	if(!(buffer != NULL && sscanf(buffer, "%127ms", &FTLfiles.FTL_db)))
 	{
 		// Use standard path if no custom path was obtained from the config file
-		FTLfiles.db = strdup("/etc/pihole/pihole-FTL.db");
+		FTLfiles.FTL_db = strdup("/etc/pihole/pihole-FTL.FTL_db");
 	}
 
 	// Test if memory allocation was successful
-	if(FTLfiles.db == NULL && errno != 0)
+	if(FTLfiles.FTL_db == NULL && errno != 0)
 	{
-		logg("FATAL: Allocating memory for FTLfiles.db failed (%s, %i). Exiting.", strerror(errno), errno);
+		logg("FATAL: Allocating memory for FTLfiles.FTL_db failed (%s, %i). Exiting.", strerror(errno), errno);
 		exit(EXIT_FAILURE);
 	}
-	else if(FTLfiles.db != NULL && strlen(FTLfiles.db) > 0)
-		logg("   DBFILE: Using %s", FTLfiles.db);
+	else if(FTLfiles.FTL_db != NULL && strlen(FTLfiles.FTL_db) > 0)
+		logg("   DBFILE: Using %s", FTLfiles.FTL_db);
 	else
 		logg("   DBFILE: Not using database due to empty filename");
 
@@ -299,10 +299,10 @@ void read_FTLconf(void)
 	getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist);
 
 	// MACVENDORDB
-	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb);
+	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendor_db);
 
 	// GRAVITYDB
-	getpath(fp, "GRAVITYDB", "/etc/pihole/gravity.db", &FTLfiles.gravitydb);
+	getpath(fp, "GRAVITYDB", "/etc/pihole/gravity.db", &FTLfiles.gravity_db);
 
 	// PARSE_ARP_CACHE
 	// defaults to: true

database/common.c

@@ -3,17 +3,18 @@
 *  Network-wide ad blocking via your own hardware.
 *
 *  FTL Engine
-*  Database routines
+*  Common database routines
 *
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
 #include "common.h"
+#include "sqlite3.h"
 #include "shmem.h"
 #include "overTime.h"
-#include "sqlite3.h"
-#include "networktable.h"
+#include "network-table.h"
+#include "query-table.h"
 #include "datastructure.h"
 #include "memory.h"
 #include "config.h"
@@ -23,20 +24,14 @@
 // global variable killed
 #include "signals.h"
 
-static sqlite3 *db;
+sqlite3 *FTL_db;
 bool database = false;
 bool DBdeleteoldqueries = false;
 long int lastdbindex = 0;
 
 static pthread_mutex_t dblock;
 
-static bool db_set_counter(const unsigned int ID, const int value);
-static int db_get_FTL_property(const unsigned int ID);
-
-// defined in networktable.c
-extern bool unify_hwaddr(sqlite3 *db);
-
-static bool check_database(int rc)
+bool check_database(int rc)
 {
 	// We will retry if the database is busy at the moment
 	// However, we won't retry if any other error happened
@@ -56,38 +51,21 @@ static bool check_database(int rc)
 
 void dbclose(void)
 {
-	int rc = sqlite3_close(db);
+	int rc = sqlite3_close(FTL_db);
 	// Report any error
 	if( rc )
-		logg("dbclose() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+		logg("dbclose() - SQL error (%i): %s", rc, sqlite3_errmsg(FTL_db));
 
 	// Unlock mutex on the database
 	pthread_mutex_unlock(&dblock);
 }
 
-static double get_db_filesize(void)
-{
-	struct stat st;
-	if(stat(FTLfiles.db, &st) != 0)
-	{
-		// stat() failed (maybe the DB file does not exist?)
-		return 0;
-	}
-	return 1e-6*st.st_size;
-}
-
-static bool file_exists(const char *filename)
-{
-	struct stat st;
-	return stat(filename, &st) == 0;
-}
-
 bool dbopen(void)
 {
 	pthread_mutex_lock(&dblock);
-	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
+	int rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE, NULL);
 	if( rc ){
-		logg("dbopen() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+		logg("dbopen() - SQL error (%i): %s", rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return false;
@@ -113,7 +91,7 @@ bool dbquery(const char *format, ...)
 
 	if(config.debug & DEBUG_DATABASE) logg("dbquery: %s", query);
 
-	int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg);
+	int rc = sqlite3_exec(FTL_db, query, NULL, NULL, &zErrMsg);
 
 	if( rc != SQLITE_OK ){
 		logg("dbquery(%s) - SQL error (%i): %s", query, rc, zErrMsg);
@@ -157,9 +135,9 @@ static bool create_counter_table(void)
 static bool db_create(void)
 {
 	bool ret;
-	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
+	int rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
 	if( rc ){
-		logg("db_create() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+		logg("db_create() - SQL error (%i): %s", rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return false;
@@ -174,7 +152,7 @@ static bool db_create(void)
 	ret = dbquery("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );");
 	if(!ret){ dbclose(); return false; }
 
-	// Set DB version 1
+	// Set FTL_db version 1
 	ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION);
 	if(!ret){ dbclose(); return false; }
 
@@ -183,12 +161,12 @@ static bool db_create(void)
 	if(!ret){ dbclose(); return false; }
 
 	// Create counter table
-	// Will update DB version to 2
+	// Will update FTL_db version to 2
 	if(!create_counter_table())
 		return false;
 
 	// Create network table
-	// Will update DB version to 3
+	// Will update FTL_db version to 3
 	if(!create_network_table())
 		return false;
 
@@ -199,7 +177,7 @@ static bool db_create(void)
 	// Explicitly set permissions to 0644
 	// 644 =            u+w       u+r       g+r       o+r
 	const mode_t mode = S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH;
-	chmod_file(FTLfiles.db, mode);
+	chmod_file(FTLfiles.FTL_db, mode);
 
 	return true;
 }
@@ -216,7 +194,7 @@ void db_init(void)
 {
 	// First check if the user doesn't want to use the database and set an
 	// empty string as file name in FTL's config file
-	if(FTLfiles.db == NULL || strlen(FTLfiles.db) == 0)
+	if(FTLfiles.FTL_db == NULL || strlen(FTLfiles.FTL_db) == 0)
 	{
 		database = false;
 		return;
@@ -229,7 +207,7 @@ void db_init(void)
 	sqlite3_config(SQLITE_CONFIG_LOG, SQLite3LogCallback, NULL);
 
 	// Check if database exists, if not create empty database
-	if(!file_exists(FTLfiles.db))
+	if(!file_exists(FTLfiles.FTL_db))
 	{
 		logg("No database file found, creating new (empty) database");
 		if (!db_create())
@@ -240,16 +218,16 @@ void db_init(void)
 		}
 	}
 
-	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
+	int rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE, NULL);
 	if( rc ){
-		logg("db_init() - Cannot open database (%i): %s", rc, sqlite3_errmsg(db));
+		logg("db_init() - Cannot open database (%i): %s", rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 
 		database = false;
 		return;
 	}
 
-	// Test DB version and see if we need to upgrade the database file
+	// Test FTL_db version and see if we need to upgrade the database file
 	int dbversion = db_get_FTL_property(DB_VERSION);
 	logg("Database version is %i", dbversion);
 	if(dbversion < 1)
@@ -294,18 +272,18 @@ void db_init(void)
 	{
 		// Update to version 4: Unify clients in network table
 		logg("Updating long-term database to version 4");
-		unify_hwaddr(db);
+		unify_hwaddr();
 		// Get updated version
 		dbversion = db_get_FTL_property(DB_VERSION);
 	}
 
 	// Close database to prevent having it opened all time
 	// we already closed the database when we returned earlier
-	sqlite3_close(db);
+	sqlite3_close(FTL_db);
 
 	if (pthread_mutex_init(&dblock, NULL) != 0)
 	{
-		logg("FATAL: DB mutex init failed\n");
+		logg("FATAL: FTL_db mutex init failed\n");
 		// Return failure
 		exit(EXIT_FAILURE);
 	}
@@ -314,7 +292,7 @@ void db_init(void)
 	database = true;
 }
 
-static int db_get_FTL_property(const unsigned int ID)
+int db_get_FTL_property(const unsigned int ID)
 {
 	// Prepare SQL statement
 	char* querystr = NULL;
@@ -337,12 +315,12 @@ bool db_set_FTL_property(const unsigned int ID, const int value)
 	return dbquery("INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %i );", ID, value);
 }
 
-static bool db_set_counter(const unsigned int ID, const int value)
+bool db_set_counter(const unsigned int ID, const int value)
 {
 	return dbquery("INSERT OR REPLACE INTO counters (id, value) VALUES ( %u, %i );", ID, value);
 }
 
-static bool db_update_counters(const int total, const int blocked)
+bool db_update_counters(const int total, const int blocked)
 {
 	if(!dbquery("UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES))
 		return false;
@@ -354,9 +332,9 @@ static bool db_update_counters(const int total, const int blocked)
 int db_query_int(const char* querystr)
 {
 	sqlite3_stmt* stmt;
-	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL);
 	if( rc ){
-		logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return DB_FAILED;
@@ -376,7 +354,7 @@ int db_query_int(const char* querystr)
 	}
 	else
 	{
-		logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return DB_FAILED;
@@ -387,14 +365,13 @@ int db_query_int(const char* querystr)
 	return result;
 }
 
-static int number_of_queries_in_DB(void)
+long int last_ID_in_DB(void)
 {
 	sqlite3_stmt* stmt;
 
-	// Count number of rows using the index timestamp is faster than select(*)
-	int rc = sqlite3_prepare_v2(db, "SELECT COUNT(timestamp) FROM queries", -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(FTL_db, "SELECT MAX(ID) FROM queries", -1, &stmt, NULL);
 	if( rc ){
-		logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
+		logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return DB_FAILED;
@@ -402,34 +379,7 @@ static int number_of_queries_in_DB(void)
 
 	rc = sqlite3_step(stmt);
 	if( rc != SQLITE_ROW ){
-		logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return DB_FAILED;
-	}
-
-	int result = sqlite3_column_int(stmt, 0);
-
-	sqlite3_finalize(stmt);
-
-	return result;
-}
-
-static sqlite3_int64 last_ID_in_DB(void)
-{
-	sqlite3_stmt* stmt;
-
-	int rc = sqlite3_prepare_v2(db, "SELECT MAX(ID) FROM queries", -1, &stmt, NULL);
-	if( rc ){
-		logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return DB_FAILED;
-	}
-
-	rc = sqlite3_step(stmt);
-	if( rc != SQLITE_ROW ){
-		logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+		logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(FTL_db));
 		dbclose();
 		check_database(rc);
 		return DB_FAILED;
@@ -442,226 +392,10 @@ static sqlite3_int64 last_ID_in_DB(void)
 	return result;
 }
 
-int get_number_of_queries_in_DB(void)
-{
-	int result = DB_NODATA;
-
-	if(!dbopen())
-	{
-		logg("Failed to open DB in get_number_of_queries_in_DB()");
-		return DB_FAILED;
-	}
-
-	result = number_of_queries_in_DB();
-
-	// Close database
-	dbclose();
-
-	return result;
-}
-
-void save_to_DB(void)
-{
-	// Don't save anything to the database if in PRIVACY_NOSTATS mode
-	if(config.privacylevel >= PRIVACY_NOSTATS)
-		return;
-
-	// Start database timer
-	if(config.debug & DEBUG_DATABASE) timer_start(DATABASE_WRITE_TIMER);
-
-	// Open database
-	if(!dbopen())
-	{
-		logg("save_to_DB() - failed to open DB");
-		return;
-	}
-
-	unsigned int saved = 0, saved_error = 0;
-	sqlite3_stmt* stmt = NULL;
-
-	// Get last ID stored in the database
-	sqlite3_int64 lastID = last_ID_in_DB();
-
-	bool ret = dbquery("BEGIN TRANSACTION");
-	if(!ret)
-	{
-		logg("save_to_DB() - unable to begin transaction (%i): %s", ret, sqlite3_errmsg(db));
-		dbclose();
-		return;
-	}
-
-	int rc = sqlite3_prepare_v2(db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL);
-	if( rc )
-	{
-		logg("save_to_DB() - error in preparing SQL statement (%i): %s", ret, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return;
-	}
-
-	int total = 0, blocked = 0;
-	time_t currenttimestamp = time(NULL);
-	time_t newlasttimestamp = 0;
-	long int queryID;
-	for(queryID = MAX(0, lastdbindex); queryID < counters->queries; queryID++)
-	{
-		queriesData* query = getQuery(queryID, true);
-		if(query->db != 0)
-		{
-			// Skip, already saved in database
-			continue;
-		}
-
-		if(!query->complete && query->timestamp > currenttimestamp-2)
-		{
-			// Break if a brand new query (age < 2 seconds) is not yet completed
-			// giving it a chance to be stored next time
-			break;
-		}
-
-		if(query->privacylevel >= PRIVACY_MAXIMUM)
-		{
-			// Skip, we never store nor count queries recorded
-			// while have been in maximum privacy mode in the database
-			continue;
-		}
-
-		// TIMESTAMP
-		sqlite3_bind_int(stmt, 1, query->timestamp);
-
-		// TYPE
-		sqlite3_bind_int(stmt, 2, query->type);
-
-		// STATUS
-		sqlite3_bind_int(stmt, 3, query->status);
-
-		// DOMAIN
-		const char *domain = getDomainString(queryID);
-		sqlite3_bind_text(stmt, 4, domain, -1, SQLITE_TRANSIENT);
-
-		// CLIENT
-		const char *client = getClientIPString(queryID);
-		sqlite3_bind_text(stmt, 5, client, -1, SQLITE_TRANSIENT);
-
-		// FORWARD
-		if(query->status == QUERY_FORWARDED && query->forwardID > -1)
-		{
-			// Get forward pointer
-			const forwardedData* forward = getForward(query->forwardID, true);
-			sqlite3_bind_text(stmt, 6, getstr(forward->ippos), -1, SQLITE_TRANSIENT);
-		}
-		else
-		{
-			sqlite3_bind_null(stmt, 6);
-		}
-
-		// Step and check if successful
-		rc = sqlite3_step(stmt);
-		sqlite3_clear_bindings(stmt);
-		sqlite3_reset(stmt);
-
-		if( rc != SQLITE_DONE ){
-			logg("save_to_DB() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
-			saved_error++;
-			if(saved_error < 3)
-			{
-				continue;
-			}
-			else
-			{
-				logg("save_to_DB() - exiting due to too many errors");
-				break;
-			}
-			// Check this error message
-			check_database(rc);
-		}
-
-		saved++;
-		// Mark this query as saved in the database by setting the corresponding ID
-		query->db = ++lastID;
-
-		// Total counter information (delta computation)
-		total++;
-		if(query->status == QUERY_GRAVITY ||
-		   query->status == QUERY_BLACKLIST ||
-		   query->status == QUERY_WILDCARD ||
-		   query->status == QUERY_EXTERNAL_BLOCKED_IP ||
-		   query->status == QUERY_EXTERNAL_BLOCKED_NULL ||
-		   query->status == QUERY_EXTERNAL_BLOCKED_NXRA)
-			blocked++;
-
-		// Update lasttimestamp variable with timestamp of the latest stored query
-		if(query->timestamp > newlasttimestamp)
-			newlasttimestamp = query->timestamp;
-	}
-
-	// Finish prepared statement
-	ret = dbquery("END TRANSACTION");
-	int ret2 = sqlite3_finalize(stmt);
-	if(!ret || ret2 != SQLITE_OK){ dbclose(); return; }
-
-	// Store index for next loop interation round and update last time stamp
-	// in the database only if all queries have been saved successfully
-	if(saved > 0 && saved_error == 0)
-	{
-		lastdbindex = queryID;
-		db_set_FTL_property(DB_LASTTIMESTAMP, newlasttimestamp);
-	}
-
-	// Update total counters in DB
-	if(saved > 0 && !db_update_counters(total, blocked))
-	{
-		dbclose();
-		return;
-	}
-
-	// Close database
-	dbclose();
-
-	if(config.debug & DEBUG_DATABASE)
-	{
-		logg("Notice: Queries stored in DB: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID);
-		if(saved_error > 0)
-			logg("        There are queries that have not been saved");
-	}
-}
-
-static void delete_old_queries_in_DB(void)
-{
-	// Open database
-	if(!dbopen())
-	{
-		logg("Failed to open DB in delete_old_queries_in_DB()");
-		return;
-	}
-
-	int timestamp = time(NULL) - config.maxDBdays * 86400;
-
-	if(!dbquery("DELETE FROM queries WHERE timestamp <= %i", timestamp))
-	{
-		dbclose();
-		logg("delete_old_queries_in_DB(): Deleting queries due to age of entries failed!");
-		database = true;
-		return;
-	}
-
-	// Get how many rows have been affected (deleted)
-	const int affected = sqlite3_changes(db);
-
-	// Print final message only if there is a difference
-	if((config.debug & DEBUG_DATABASE) || affected)
-		logg("Notice: Database size is %.2f MB, deleted %i rows", get_db_filesize(), affected);
-
-	// Close database
-	dbclose();
-
-	// Re-enable database actions
-	database = true;
-}
-
-int lastDBsave = 0;
 void *DB_thread(void *val)
 {
+	int lastDBsave = 0;
+
 	// Set thread name
 	prctl(PR_SET_NAME,"database",0,0,0);
 
@@ -703,221 +437,3 @@ void *DB_thread(void *val)
 
 	return NULL;
 }
-
-// Get most recent 24 hours data from long-term database
-void read_data_from_DB(void)
-{
-	// Don't try to load anything to the database if in PRIVACY_NOSTATS mode
-	if(config.privacylevel >= PRIVACY_NOSTATS)
-		return;
-
-	// Open database file
-	if(!dbopen())
-	{
-		logg("read_data_from_DB() - Failed to open DB");
-		return;
-	}
-
-	// Prepare request
-	char *rstr = NULL;
-	// Get time stamp 24 hours in the past
-	const time_t now = time(NULL);
-	const time_t mintime = now - config.maxlogage;
-	int rc = asprintf(&rstr, "SELECT * FROM queries WHERE timestamp >= %li", mintime);
-	if(rc < 1)
-	{
-		logg("read_data_from_DB() - Allocation error (%i): %s", rc, sqlite3_errmsg(db));
-		return;
-	}
-	// Log DB query string in debug mode
-	if(config.debug & DEBUG_DATABASE) logg("%s", rstr);
-
-	// Prepare SQLite3 statement
-	sqlite3_stmt* stmt = NULL;
-	rc = sqlite3_prepare_v2(db, rstr, -1, &stmt, NULL);
-	if( rc ){
-		logg("read_data_from_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return;
-	}
-
-	// Loop through returned database rows
-	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
-	{
-		const sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0);
-		const time_t queryTimeStamp = sqlite3_column_int(stmt, 1);
-		// 1483228800 = 01/01/2017 @ 12:00am (UTC)
-		if(queryTimeStamp < 1483228800)
-		{
-			logg("DB warn: TIMESTAMP should be larger than 01/01/2017 but is %li", queryTimeStamp);
-			continue;
-		}
-		if(queryTimeStamp > now)
-		{
-			if(config.debug & DEBUG_DATABASE) logg("DB warn: Skipping query logged in the future (%li)", queryTimeStamp);
-			continue;
-		}
-
-		const int type = sqlite3_column_int(stmt, 2);
-		if(type < TYPE_A || type >= TYPE_MAX)
-		{
-			logg("DB warn: TYPE should not be %i", type);
-			continue;
-		}
-		// Don't import AAAA queries from database if the user set
-		// AAAA_QUERY_ANALYSIS=no in pihole-FTL.conf
-		if(type == TYPE_AAAA && !config.analyze_AAAA)
-		{
-			continue;
-		}
-
-		const int status = sqlite3_column_int(stmt, 3);
-		if(status < QUERY_UNKNOWN || status > QUERY_EXTERNAL_BLOCKED_NXRA)
-		{
-			logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_EXTERNAL_BLOCKED_NXRA, status);
-			continue;
-		}
-
-		const char * domainname = (const char *)sqlite3_column_text(stmt, 4);
-		if(domainname == NULL)
-		{
-			logg("DB warn: DOMAIN should never be NULL, %li", queryTimeStamp);
-			continue;
-		}
-
-		const char * clientIP = (const char *)sqlite3_column_text(stmt, 5);
-		if(clientIP == NULL)
-		{
-			logg("DB warn: CLIENT should never be NULL, %li", queryTimeStamp);
-			continue;
-		}
-
-		// Check if user wants to skip queries coming from localhost
-		if(config.ignore_localhost &&
-		   (strcmp(clientIP, "127.0.0.1") == 0 || strcmp(clientIP, "::1") == 0))
-		{
-			continue;
-		}
-
-		const char *forwarddest = (const char *)sqlite3_column_text(stmt, 6);
-		int forwardID = 0;
-		// Determine forwardID only when status == 2 (forwarded) as the
-		// field need not to be filled for other query status types
-		if(status == QUERY_FORWARDED)
-		{
-			if(forwarddest == NULL)
-			{
-				logg("DB warn: FORWARD should not be NULL with status QUERY_FORWARDED, %li", queryTimeStamp);
-				continue;
-			}
-			forwardID = findForwardID(forwarddest, true);
-		}
-
-		// Obtain IDs only after filtering which queries we want to keep
-		const int timeidx = getOverTimeID(queryTimeStamp);
-		const int domainID = findDomainID(domainname);
-		const int clientID = findClientID(clientIP, true);
-
-		// Ensure we have enough space in the queries struct
-		memory_check(QUERIES);
-
-		// Set index for this query
-		const int queryIndex = counters->queries;
-
-		// Store this query in memory
-		queriesData* query = getQuery(queryIndex, false);
-		query->magic = MAGICBYTE;
-		query->timestamp = queryTimeStamp;
-		query->type = type;
-		query->status = status;
-		query->domainID = domainID;
-		query->clientID = clientID;
-		query->forwardID = forwardID;
-		query->timeidx = timeidx;
-		query->db = dbid;
-		query->id = 0;
-		query->complete = true; // Mark as all information is available
-		query->response = 0;
-		query->dnssec = DNSSEC_UNKNOWN;
-		query->reply = REPLY_UNKNOWN;
-
-		// Set lastQuery timer and add one query for network table
-		clientsData* client = getClient(clientID, true);
-		client->lastQuery = queryTimeStamp;
-		client->numQueriesARP++;
-
-		// Handle type counters
-		if(type >= TYPE_A && type < TYPE_MAX)
-		{
-			counters->querytype[type-1]++;
-			overTime[timeidx].querytypedata[type-1]++;
-		}
-
-		// Update overTime data
-		overTime[timeidx].total++;
-		// Update overTime data structure with the new client
-		client->overTime[timeidx]++;
-
-		// Increase DNS queries counter
-		counters->queries++;
-
-		// Increment status counters
-		switch(status)
-		{
-			case QUERY_UNKNOWN: // Unknown
-				counters->unknown++;
-				break;
-
-			case QUERY_GRAVITY: // Blocked by gravity.list
-			case QUERY_WILDCARD: // Blocked by regex filter
-			case QUERY_BLACKLIST: // Blocked by black.list
-			case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider
-			case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider
-			case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider
-				counters->blocked++;
-				// Get domain pointer
-				domainsData* domain = getDomain(domainID, true);
-				domain->blockedcount++;
-				client->blockedcount++;
-				// Update overTime data structure
-				overTime[timeidx].blocked++;
-				break;
-
-			case QUERY_FORWARDED: // Forwarded
-				counters->forwardedqueries++;
-				// Update overTime data structure
-				overTime[timeidx].forwarded++;
-				break;
-
-			case QUERY_CACHE: // Cached or local config
-				counters->cached++;
-				// Update overTime data structure
-				overTime[timeidx].cached++;
-				break;
-
-			default:
-				logg("Error: Found unknown status %i in long term database!", status);
-				logg("       Timestamp: %li", queryTimeStamp);
-				logg("       Continuing anyway...");
-				break;
-		}
-	}
-	logg("Imported %i queries from the long-term database", counters->queries);
-
-	// Update lastdbindex so that the next call to save_to_DB()
-	// skips the queries that we just imported from the database
-	lastdbindex = counters->queries;
-
-	if( rc != SQLITE_DONE ){
-		logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return;
-	}
-
-	// Finalize SQLite3 statement
-	sqlite3_finalize(stmt);
-	dbclose();
-	free(rstr);
-}

database/common.h

@@ -7,23 +7,28 @@
 *
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
-#ifndef DATABASE_H
-#define DATABASE_H
+#ifndef DATABASE_COMMON_H
+#define DATABASE_COMMON_H
 
+#include "sqlite3.h"
+
+bool check_database(int rc);
 void db_init(void);
 void *DB_thread(void *val);
-int get_number_of_queries_in_DB(void);
-void save_to_DB(void);
-void read_data_from_DB(void);
+int db_get_FTL_property(const unsigned int ID);
 bool db_set_FTL_property(const unsigned int ID, const int value);
 bool dbquery(const char *format, ...);
 bool dbopen(void);
 void dbclose(void);
 int db_query_int(const char*);
 void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg);
+long int last_ID_in_DB(void);
+bool db_set_counter(const unsigned int ID, const int value);
+bool db_update_counters(const int total, const int blocked);
 
+extern sqlite3 *FTL_db;
 extern bool database;
 extern long int lastdbindex;
 extern bool DBdeleteoldqueries;
 
-#endif //DATABASE_H
+#endif //DATABASE_COMMON_H

database/gravity.c

@@ -17,7 +17,7 @@
 #include <sqlite3.h>
 
 // Private variables
-static sqlite3 *gravitydb = NULL;
+static sqlite3 *gravity_db = NULL;
 static sqlite3_stmt* stmt = NULL;
 static sqlite3_stmt* whitelist_stmt = NULL;
 bool gravity_database_avail = false;
@@ -29,17 +29,17 @@ void rehash(int size);
 bool gravityDB_open(void)
 {
 	struct stat st;
-	if(stat(FTLfiles.gravitydb, &st) != 0)
+	if(stat(FTLfiles.gravity_db, &st) != 0)
 	{
 		// File does not exist
-		logg("gravityDB_open(): %s does not exist", FTLfiles.gravitydb);
+		logg("gravityDB_open(): %s does not exist", FTLfiles.gravity_db);
 		return false;
 	}
 
-	int rc = sqlite3_open_v2(FTLfiles.gravitydb, &gravitydb, SQLITE_OPEN_READONLY, NULL);
+	int rc = sqlite3_open_v2(FTLfiles.gravity_db, &gravity_db, SQLITE_OPEN_READONLY, NULL);
 	if( rc )
 	{
-		logg("gravityDB_open() - SQL error (%i): %s", rc, sqlite3_errmsg(gravitydb));
+		logg("gravityDB_open() - SQL error (%i): %s", rc, sqlite3_errmsg(gravity_db));
 		gravityDB_close();
 		return false;
 	}
@@ -47,7 +47,7 @@ bool gravityDB_open(void)
 	// Tell SQLite3 to store temporary tables in memory. This speeds up read operations on
 	// temporary tables, indices, and views.
 	char *zErrMsg = NULL;
-	rc = sqlite3_exec(gravitydb, "PRAGMA temp_store = MEMORY", NULL, NULL, &zErrMsg);
+	rc = sqlite3_exec(gravity_db, "PRAGMA temp_store = MEMORY", NULL, NULL, &zErrMsg);
 	if( rc != SQLITE_OK )
 	{
 		logg("gravityDB_open(PRAGMA temp_store) - SQL error (%i): %s", rc, zErrMsg);
@@ -62,10 +62,10 @@ bool gravityDB_open(void)
 	// list but don't case about duplicates or similar. SELECT EXISTS(...)
 	// returns true as soon as it sees the first row from the query inside
 	// of EXISTS().
-	rc = sqlite3_prepare_v2(gravitydb, "SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ?);", -1, &whitelist_stmt, NULL);
+	rc = sqlite3_prepare_v2(gravity_db, "SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ?);", -1, &whitelist_stmt, NULL);
 	if( rc )
 	{
-		logg("gravityDB_open(\"SELECT EXISTS(...)\") - SQL error prepare (%i): %s", rc, sqlite3_errmsg(gravitydb));
+		logg("gravityDB_open(\"SELECT EXISTS(...)\") - SQL error prepare (%i): %s", rc, sqlite3_errmsg(gravity_db));
 		gravityDB_close();
 		return false;
 	}
@@ -88,7 +88,7 @@ void gravityDB_close(void)
 	sqlite3_finalize(whitelist_stmt);
 
 	// Close table
-	sqlite3_close(gravitydb);
+	sqlite3_close(gravity_db);
 	gravity_database_avail = false;
 }
 
@@ -122,10 +122,10 @@ bool gravityDB_getTable(const unsigned char list)
 	}
 
 	// Prepare SQLite3 statement
-	int rc = sqlite3_prepare_v2(gravitydb, querystr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
 	if( rc )
 	{
-		logg("readGravity(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravitydb));
+		logg("readGravity(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
 		gravityDB_close();
 		return false;
 	}
@@ -158,7 +158,7 @@ inline const char* gravityDB_getDomain(void)
 	// SQLITE_DONE (we are finished reading the table)
 	if(rc != SQLITE_DONE)
 	{
-		logg("gravityDB_getDomain() - SQL error step (%i): %s", rc, sqlite3_errmsg(gravitydb));
+		logg("gravityDB_getDomain() - SQL error step (%i): %s", rc, sqlite3_errmsg(gravity_db));
 		gravityDB_finalizeTable();
 		return NULL;
 	}
@@ -210,9 +210,9 @@ int gravityDB_count(const unsigned char list)
 	}
 
 	// Prepare query
-	int rc = sqlite3_prepare_v2(gravitydb, querystr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
 	if( rc ){
-		logg("gravityDB_count(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravitydb));
+		logg("gravityDB_count(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
 		sqlite3_finalize(stmt);
 		gravityDB_close();
 		return DB_FAILED;
@@ -221,7 +221,7 @@ int gravityDB_count(const unsigned char list)
 	// Perform query
 	rc = sqlite3_step(stmt);
 	if( rc != SQLITE_ROW ){
-		logg("gravityDB_count(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(gravitydb));
+		logg("gravityDB_count(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
 		sqlite3_finalize(stmt);
 		gravityDB_close();
 		return DB_FAILED;
@@ -245,7 +245,7 @@ bool in_whitelist(const char *domain)
 	if((retval = sqlite3_bind_text(whitelist_stmt, 1, domain, -1, SQLITE_STATIC)) != SQLITE_OK)
 	{
 		logg("in_whitelist(\"%s\"): Failed to bind domain (error %d) - %s",
-		     domain, retval, sqlite3_errmsg(gravitydb));
+		     domain, retval, sqlite3_errmsg(gravity_db));
 		sqlite3_reset(whitelist_stmt);
 		return false;
 	}
@@ -266,7 +266,7 @@ bool in_whitelist(const char *domain)
 		// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
 		// is a real error we should log
 		logg("in_whitelist(\"%s\"): Failed to perform step (error %d) - %s",
-		     domain, retval, sqlite3_errmsg(gravitydb));
+		     domain, retval, sqlite3_errmsg(gravity_db));
 		sqlite3_reset(whitelist_stmt);
 		sqlite3_clear_bindings(whitelist_stmt);
 		return false;

database/network-table.c

@@ -9,7 +9,7 @@
 *  Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
-#include "networktable.h"
+#include "network-table.h"
 #include "common.h"
 #include "shmem.h"
 #include "sqlite3.h"
@@ -22,7 +22,6 @@
 
 // Private prototypes
 static char* getMACVendor(const char* hwaddr);
-bool unify_hwaddr(sqlite3 *db);
 
 bool create_network_table(void)
 {
@@ -61,7 +60,7 @@ void parse_arp_cache(void)
 	// Open database file
 	if(!dbopen())
 	{
-		logg("read_arp_cache() - Failed to open DB");
+		logg("read_arp_cache() - Failed to open FTL_db");
 		fclose(arpfp);
 		return;
 	}
@@ -219,7 +218,7 @@ void parse_arp_cache(void)
 // If we find duplicates, we keep the most recent entry, while
 // - we replace the first-seen date by the earliest across all rows
 // - we sum up the number of queries of all clients with the same hwaddr
-bool unify_hwaddr(sqlite3 *db)
+bool unify_hwaddr(void)
 {
 	// We request sets of (id,hwaddr). They are GROUPed BY hwaddr to make
 	// the set unique in hwaddr.
@@ -237,9 +236,9 @@ bool unify_hwaddr(sqlite3 *db)
 
 	// Perform SQL query
 	sqlite3_stmt* stmt;
-	ret = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	ret = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL);
 	if( ret ){
-		logg("unify_hwaddr(%s) - SQL error prepare (%i): %s", querystr, ret, sqlite3_errmsg(db));
+		logg("unify_hwaddr(%s) - SQL error prepare (%i): %s", querystr, ret, sqlite3_errmsg(FTL_db));
 		dbclose();
 		return false;
 	}
@@ -250,7 +249,7 @@ bool unify_hwaddr(sqlite3 *db)
 		// Check if we ran into an error
 		if(ret != SQLITE_ROW)
 		{
-			logg("unify_hwaddr(%s) - SQL error step (%i): %s", querystr, ret, sqlite3_errmsg(db));
+			logg("unify_hwaddr(%s) - SQL error step (%i): %s", querystr, ret, sqlite3_errmsg(FTL_db));
 			dbclose();
 			return false;
 		}
@@ -305,11 +304,11 @@ bool unify_hwaddr(sqlite3 *db)
 static char* getMACVendor(const char* hwaddr)
 {
 	struct stat st;
-	if(stat(FTLfiles.macvendordb, &st) != 0)
+	if(stat(FTLfiles.macvendor_db, &st) != 0)
 	{
 		// File does not exist
 		if(config.debug & DEBUG_ARP)
-			logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb);
+			logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendor_db);
 		return strdup("");
 	}
 	else if(strlen(hwaddr) != 17)
@@ -320,11 +319,11 @@ static char* getMACVendor(const char* hwaddr)
 		return strdup("");
 	}
 
-	sqlite3 *macdb;
-	int rc = sqlite3_open_v2(FTLfiles.macvendordb, &macdb, SQLITE_OPEN_READONLY, NULL);
+	sqlite3 *macvendor_db;
+	int rc = sqlite3_open_v2(FTLfiles.macvendor_db, &macvendor_db, SQLITE_OPEN_READONLY, NULL);
 	if( rc ){
-		logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
-		sqlite3_close(macdb);
+		logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macvendor_db));
+		sqlite3_close(macvendor_db);
 		return strdup("");
 	}
 
@@ -336,16 +335,16 @@ static char* getMACVendor(const char* hwaddr)
 	if(rc < 1)
 	{
 		logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc);
-		sqlite3_close(macdb);
+		sqlite3_close(macvendor_db);
 		return strdup("");
 	}
 	free(hwaddrshort);
 
 	sqlite3_stmt* stmt;
-	rc = sqlite3_prepare_v2(macdb, querystr, -1, &stmt, NULL);
+	rc = sqlite3_prepare_v2(macvendor_db, querystr, -1, &stmt, NULL);
 	if( rc ){
-		logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb));
-		sqlite3_close(macdb);
+		logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macvendor_db));
+		sqlite3_close(macvendor_db);
 		return strdup("");
 	}
 	free(querystr);
@@ -365,40 +364,34 @@ static char* getMACVendor(const char* hwaddr)
 	if(rc != SQLITE_DONE && rc != SQLITE_ROW)
 	{
 		// Error
-		logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
+		logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macvendor_db));
 	}
 
 	sqlite3_finalize(stmt);
-	sqlite3_close(macdb);
+	sqlite3_close(macvendor_db);
 
 	return vendor;
 }
 
-void updateMACVendorRecords()
+void updateMACVendorRecords(void)
 {
 	struct stat st;
-	if(stat(FTLfiles.macvendordb, &st) != 0)
+	if(stat(FTLfiles.macvendor_db, &st) != 0)
 	{
 		// File does not exist
 		if(config.debug & DEBUG_ARP)
-			logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb);
+			logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendor_db);
 		return;
 	}
 
-	sqlite3 *db;
-	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
-	if( rc ){
-		logg("updateMACVendorRecords() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
-		sqlite3_close(db);
-		return;
-	}
+	dbopen();
 
 	sqlite3_stmt* stmt;
 	const char* selectstr = "SELECT id,hwaddr FROM network;";
-	rc = sqlite3_prepare_v2(db, selectstr, -1, &stmt, NULL);
+	int rc = sqlite3_prepare_v2(FTL_db, selectstr, -1, &stmt, NULL);
 	if( rc ){
-		logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", selectstr, rc, sqlite3_errmsg(db));
-		sqlite3_close(db);
+		logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", selectstr, rc, sqlite3_errmsg(FTL_db));
+		sqlite3_close(FTL_db);
 		return;
 	}
 
@@ -423,7 +416,7 @@ void updateMACVendorRecords()
 
 		// Execute prepared statement
 		char *zErrMsg = NULL;
-		rc = sqlite3_exec(db, updatestr, NULL, NULL, &zErrMsg);
+		rc = sqlite3_exec(FTL_db, updatestr, NULL, NULL, &zErrMsg);
 		if( rc != SQLITE_OK ){
 			logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", updatestr, rc, zErrMsg);
 			sqlite3_free(zErrMsg);
@@ -439,9 +432,9 @@ void updateMACVendorRecords()
 	if(rc != SQLITE_DONE)
 	{
 		// Error
-		logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+		logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(FTL_db));
 	}
 
 	sqlite3_finalize(stmt);
-	sqlite3_close(db);
+	dbclose();
 }

database/network-table.h

@@ -13,5 +13,6 @@
 bool create_network_table(void);
 void parse_arp_cache(void);
 void updateMACVendorRecords(void);
+bool unify_hwaddr(void);
 
 #endif //NETWORKTABLE_H

database/query-table.c

@@ -0,0 +1,485 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Query table database routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "query-table.h"
+#include "common.h"
+#include "sqlite3.h"
+#include "datastructure.h"
+#include "overTime.h"
+#include "files.h"
+#include "memory.h"
+#include "timers.h"
+#include "log.h"
+#include "config.h"
+#include "shmem.h"
+
+static int number_of_queries_in_DB(void)
+{
+	sqlite3_stmt* stmt;
+
+	// Count number of rows using the index timestamp is faster than select(*)
+	int rc = sqlite3_prepare_v2(FTL_db, "SELECT COUNT(timestamp) FROM queries", -1, &stmt, NULL);
+	if( rc ){
+		logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(FTL_db));
+		dbclose();
+		check_database(rc);
+		return DB_FAILED;
+	}
+
+	rc = sqlite3_step(stmt);
+	if( rc != SQLITE_ROW ){
+		logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(FTL_db));
+		dbclose();
+		check_database(rc);
+		return DB_FAILED;
+	}
+
+	int result = sqlite3_column_int(stmt, 0);
+
+	sqlite3_finalize(stmt);
+
+	return result;
+}
+
+int get_number_of_queries_in_DB(void)
+{
+	int result = DB_NODATA;
+
+	if(!dbopen())
+	{
+		logg("Failed to open FTL_db in get_number_of_queries_in_DB()");
+		return DB_FAILED;
+	}
+
+	result = number_of_queries_in_DB();
+
+	// Close database
+	dbclose();
+
+	return result;
+}
+
+void save_to_DB(void)
+{
+	// Don't save anything to the database if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
+	// Start database timer
+	if(config.debug & DEBUG_DATABASE) timer_start(DATABASE_WRITE_TIMER);
+
+	// Open database
+	if(!dbopen())
+	{
+		logg("save_to_DB() - failed to open FTL_db");
+		return;
+	}
+
+	unsigned int saved = 0, saved_error = 0;
+	sqlite3_stmt* stmt = NULL;
+
+	// Get last ID stored in the database
+	sqlite3_int64 lastID = last_ID_in_DB();
+
+	bool ret = dbquery("BEGIN TRANSACTION");
+	if(!ret)
+	{
+		logg("save_to_DB() - unable to begin transaction (%i): %s", ret, sqlite3_errmsg(FTL_db));
+		dbclose();
+		return;
+	}
+
+	int rc = sqlite3_prepare_v2(FTL_db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL);
+	if( rc )
+	{
+		logg("save_to_DB() - error in preparing SQL statement (%i): %s", ret, sqlite3_errmsg(FTL_db));
+		dbclose();
+		check_database(rc);
+		return;
+	}
+
+	int total = 0, blocked = 0;
+	time_t currenttimestamp = time(NULL);
+	time_t newlasttimestamp = 0;
+	long int queryID;
+	for(queryID = MAX(0, lastdbindex); queryID < counters->queries; queryID++)
+	{
+		queriesData* query = getQuery(queryID, true);
+		if(query->db != 0)
+		{
+			// Skip, already saved in database
+			continue;
+		}
+
+		if(!query->complete && query->timestamp > currenttimestamp-2)
+		{
+			// Break if a brand new query (age < 2 seconds) is not yet completed
+			// giving it a chance to be stored next time
+			break;
+		}
+
+		if(query->privacylevel >= PRIVACY_MAXIMUM)
+		{
+			// Skip, we never store nor count queries recorded
+			// while have been in maximum privacy mode in the database
+			continue;
+		}
+
+		// TIMESTAMP
+		sqlite3_bind_int(stmt, 1, query->timestamp);
+
+		// TYPE
+		sqlite3_bind_int(stmt, 2, query->type);
+
+		// STATUS
+		sqlite3_bind_int(stmt, 3, query->status);
+
+		// DOMAIN
+		const char *domain = getDomainString(queryID);
+		sqlite3_bind_text(stmt, 4, domain, -1, SQLITE_TRANSIENT);
+
+		// CLIENT
+		const char *client = getClientIPString(queryID);
+		sqlite3_bind_text(stmt, 5, client, -1, SQLITE_TRANSIENT);
+
+		// FORWARD
+		if(query->status == QUERY_FORWARDED && query->forwardID > -1)
+		{
+			// Get forward pointer
+			const forwardedData* forward = getForward(query->forwardID, true);
+			sqlite3_bind_text(stmt, 6, getstr(forward->ippos), -1, SQLITE_TRANSIENT);
+		}
+		else
+		{
+			sqlite3_bind_null(stmt, 6);
+		}
+
+		// Step and check if successful
+		rc = sqlite3_step(stmt);
+		sqlite3_clear_bindings(stmt);
+		sqlite3_reset(stmt);
+
+		if( rc != SQLITE_DONE ){
+			logg("save_to_DB() - SQL error (%i): %s", rc, sqlite3_errmsg(FTL_db));
+			saved_error++;
+			if(saved_error < 3)
+			{
+				continue;
+			}
+			else
+			{
+				logg("save_to_DB() - exiting due to too many errors");
+				break;
+			}
+			// Check this error message
+			check_database(rc);
+		}
+
+		saved++;
+		// Mark this query as saved in the database by setting the corresponding ID
+		query->db = ++lastID;
+
+		// Total counter information (delta computation)
+		total++;
+		if(query->status == QUERY_GRAVITY ||
+		   query->status == QUERY_BLACKLIST ||
+		   query->status == QUERY_WILDCARD ||
+		   query->status == QUERY_EXTERNAL_BLOCKED_IP ||
+		   query->status == QUERY_EXTERNAL_BLOCKED_NULL ||
+		   query->status == QUERY_EXTERNAL_BLOCKED_NXRA)
+			blocked++;
+
+		// Update lasttimestamp variable with timestamp of the latest stored query
+		if(query->timestamp > newlasttimestamp)
+			newlasttimestamp = query->timestamp;
+	}
+
+	// Finish prepared statement
+	ret = dbquery("END TRANSACTION");
+	int ret2 = sqlite3_finalize(stmt);
+	if(!ret || ret2 != SQLITE_OK){ dbclose(); return; }
+
+	// Store index for next loop interation round and update last time stamp
+	// in the database only if all queries have been saved successfully
+	if(saved > 0 && saved_error == 0)
+	{
+		lastdbindex = queryID;
+		db_set_FTL_property(DB_LASTTIMESTAMP, newlasttimestamp);
+	}
+
+	// Update total counters in FTL_db
+	if(saved > 0 && !db_update_counters(total, blocked))
+	{
+		dbclose();
+		return;
+	}
+
+	// Close database
+	dbclose();
+
+	if(config.debug & DEBUG_DATABASE)
+	{
+		logg("Notice: Queries stored in FTL_db: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID);
+		if(saved_error > 0)
+			logg("        There are queries that have not been saved");
+	}
+}
+
+void delete_old_queries_in_DB(void)
+{
+	// Open database
+	if(!dbopen())
+	{
+		logg("Failed to open FTL_db in delete_old_queries_in_DB()");
+		return;
+	}
+
+	int timestamp = time(NULL) - config.maxDBdays * 86400;
+
+	if(!dbquery("DELETE FROM queries WHERE timestamp <= %i", timestamp))
+	{
+		dbclose();
+		logg("delete_old_queries_in_DB(): Deleting queries due to age of entries failed!");
+		database = true;
+		return;
+	}
+
+	// Get how many rows have been affected (deleted)
+	const int affected = sqlite3_changes(FTL_db);
+
+	// Print final message only if there is a difference
+	if((config.debug & DEBUG_DATABASE) || affected)
+		logg("Notice: Database size is %.2f MB, deleted %i rows", get_FTL_db_filesize(), affected);
+
+	// Close database
+	dbclose();
+
+	// Re-enable database actions
+	database = true;
+}
+
+// Get most recent 24 hours data from long-term database
+void read_data_from_DB(void)
+{
+	// Don't try to load anything to the database if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
+	// Open database file
+	if(!dbopen())
+	{
+		logg("read_data_from_DB() - Failed to open FTL_db");
+		return;
+	}
+
+	// Prepare request
+	char *rstr = NULL;
+	// Get time stamp 24 hours in the past
+	const time_t now = time(NULL);
+	const time_t mintime = now - config.maxlogage;
+	int rc = asprintf(&rstr, "SELECT * FROM queries WHERE timestamp >= %li", mintime);
+	if(rc < 1)
+	{
+		logg("read_data_from_DB() - Allocation error (%i): %s", rc, sqlite3_errmsg(FTL_db));
+		return;
+	}
+	// Log FTL_db query string in debug mode
+	if(config.debug & DEBUG_DATABASE) logg("%s", rstr);
+
+	// Prepare SQLite3 statement
+	sqlite3_stmt* stmt = NULL;
+	rc = sqlite3_prepare_v2(FTL_db, rstr, -1, &stmt, NULL);
+	if( rc ){
+		logg("read_data_from_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(FTL_db));
+		dbclose();
+		check_database(rc);
+		return;
+	}
+
+	// Loop through returned database rows
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		const sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0);
+		const time_t queryTimeStamp = sqlite3_column_int(stmt, 1);
+		// 1483228800 = 01/01/2017 @ 12:00am (UTC)
+		if(queryTimeStamp < 1483228800)
+		{
+			logg("FTL_db warn: TIMESTAMP should be larger than 01/01/2017 but is %li", queryTimeStamp);
+			continue;
+		}
+		if(queryTimeStamp > now)
+		{
+			if(config.debug & DEBUG_DATABASE) logg("FTL_db warn: Skipping query logged in the future (%li)", queryTimeStamp);
+			continue;
+		}
+
+		const int type = sqlite3_column_int(stmt, 2);
+		if(type < TYPE_A || type >= TYPE_MAX)
+		{
+			logg("FTL_db warn: TYPE should not be %i", type);
+			continue;
+		}
+		// Don't import AAAA queries from database if the user set
+		// AAAA_QUERY_ANALYSIS=no in pihole-FTL.conf
+		if(type == TYPE_AAAA && !config.analyze_AAAA)
+		{
+			continue;
+		}
+
+		const int status = sqlite3_column_int(stmt, 3);
+		if(status < QUERY_UNKNOWN || status > QUERY_EXTERNAL_BLOCKED_NXRA)
+		{
+			logg("FTL_db warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_EXTERNAL_BLOCKED_NXRA, status);
+			continue;
+		}
+
+		const char * domainname = (const char *)sqlite3_column_text(stmt, 4);
+		if(domainname == NULL)
+		{
+			logg("FTL_db warn: DOMAIN should never be NULL, %li", queryTimeStamp);
+			continue;
+		}
+
+		const char * clientIP = (const char *)sqlite3_column_text(stmt, 5);
+		if(clientIP == NULL)
+		{
+			logg("FTL_db warn: CLIENT should never be NULL, %li", queryTimeStamp);
+			continue;
+		}
+
+		// Check if user wants to skip queries coming from localhost
+		if(config.ignore_localhost &&
+		   (strcmp(clientIP, "127.0.0.1") == 0 || strcmp(clientIP, "::1") == 0))
+		{
+			continue;
+		}
+
+		const char *forwarddest = (const char *)sqlite3_column_text(stmt, 6);
+		int forwardID = 0;
+		// Determine forwardID only when status == 2 (forwarded) as the
+		// field need not to be filled for other query status types
+		if(status == QUERY_FORWARDED)
+		{
+			if(forwarddest == NULL)
+			{
+				logg("FTL_db warn: FORWARD should not be NULL with status QUERY_FORWARDED, %li", queryTimeStamp);
+				continue;
+			}
+			forwardID = findForwardID(forwarddest, true);
+		}
+
+		// Obtain IDs only after filtering which queries we want to keep
+		const int timeidx = getOverTimeID(queryTimeStamp);
+		const int domainID = findDomainID(domainname);
+		const int clientID = findClientID(clientIP, true);
+
+		// Ensure we have enough space in the queries struct
+		memory_check(QUERIES);
+
+		// Set index for this query
+		const int queryIndex = counters->queries;
+
+		// Store this query in memory
+		queriesData* query = getQuery(queryIndex, false);
+		query->magic = MAGICBYTE;
+		query->timestamp = queryTimeStamp;
+		query->type = type;
+		query->status = status;
+		query->domainID = domainID;
+		query->clientID = clientID;
+		query->forwardID = forwardID;
+		query->timeidx = timeidx;
+		query->db = dbid;
+		query->id = 0;
+		query->complete = true; // Mark as all information is available
+		query->response = 0;
+		query->dnssec = DNSSEC_UNKNOWN;
+		query->reply = REPLY_UNKNOWN;
+
+		// Set lastQuery timer and add one query for network table
+		clientsData* client = getClient(clientID, true);
+		client->lastQuery = queryTimeStamp;
+		client->numQueriesARP++;
+
+		// Handle type counters
+		if(type >= TYPE_A && type < TYPE_MAX)
+		{
+			counters->querytype[type-1]++;
+			overTime[timeidx].querytypedata[type-1]++;
+		}
+
+		// Update overTime data
+		overTime[timeidx].total++;
+		// Update overTime data structure with the new client
+		client->overTime[timeidx]++;
+
+		// Increase DNS queries counter
+		counters->queries++;
+
+		// Increment status counters
+		switch(status)
+		{
+			case QUERY_UNKNOWN: // Unknown
+				counters->unknown++;
+				break;
+
+			case QUERY_GRAVITY: // Blocked by gravity.list
+			case QUERY_WILDCARD: // Blocked by regex filter
+			case QUERY_BLACKLIST: // Blocked by black.list
+			case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider
+			case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider
+			case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider
+				counters->blocked++;
+				// Get domain pointer
+				domainsData* domain = getDomain(domainID, true);
+				domain->blockedcount++;
+				client->blockedcount++;
+				// Update overTime data structure
+				overTime[timeidx].blocked++;
+				break;
+
+			case QUERY_FORWARDED: // Forwarded
+				counters->forwardedqueries++;
+				// Update overTime data structure
+				overTime[timeidx].forwarded++;
+				break;
+
+			case QUERY_CACHE: // Cached or local config
+				counters->cached++;
+				// Update overTime data structure
+				overTime[timeidx].cached++;
+				break;
+
+			default:
+				logg("Error: Found unknown status %i in long term database!", status);
+				logg("       Timestamp: %li", queryTimeStamp);
+				logg("       Continuing anyway...");
+				break;
+		}
+	}
+	logg("Imported %i queries from the long-term database", counters->queries);
+
+	// Update lastdbindex so that the next call to save_to_DB()
+	// skips the queries that we just imported from the database
+	lastdbindex = counters->queries;
+
+	if( rc != SQLITE_DONE ){
+		logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(FTL_db));
+		dbclose();
+		check_database(rc);
+		return;
+	}
+
+	// Finalize SQLite3 statement
+	sqlite3_finalize(stmt);
+	dbclose();
+	free(rstr);
+}

database/query-table.h

@@ -0,0 +1,18 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Query table database prototypes
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef DATABASE_QUERY_TABLE_H
+#define DATABASE_QUERY_TABLE_H
+
+int get_number_of_queries_in_DB(void);
+void delete_old_queries_in_DB(void);
+void save_to_DB(void);
+void read_data_from_DB(void);
+
+#endif //DATABASE_QUERY_TABLE_H

dnsmasq_interface.c

@@ -1163,9 +1163,11 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw)
 	if(ent_pw != NULL && getuid() == 0)
 	{
 		if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1)
-			logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno);
-		if(database && chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1)
-			logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, strerror(errno), errno);
+			logg("Setting ownership (%i:%i) of %s failed: %s (%i)",
+			     ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno);
+		if(database && chown(FTLfiles.FTL_db, ent_pw->pw_uid, ent_pw->pw_gid) == -1)
+			logg("Setting ownership (%i:%i) of %s failed: %s (%i)",
+			     ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.FTL_db, strerror(errno), errno);
 	}
 }
 

files.c

@@ -159,3 +159,19 @@ bool chmod_file(const char *filename, const mode_t mode)
 	return true;
 }
 
+bool file_exists(const char *filename)
+{
+	struct stat st;
+	return stat(filename, &st) == 0;
+}
+
+double get_FTL_db_filesize(void)
+{
+	struct stat st;
+	if(stat(FTLfiles.FTL_db, &st) != 0)
+	{
+		// stat() failed (maybe the DB file does not exist?)
+		return 0;
+	}
+	return 1e-6*st.st_size;
+}

files.h

@@ -14,6 +14,8 @@ int countlines(const char* fname);
 int countlineswith(const char* str, const char* fname);
 void check_blocking_status(void);
 bool chmod_file(const char *filename, const mode_t mode);
+bool file_exists(const char *filename);
+double get_FTL_db_filesize(void);
 
 extern unsigned char blockingstatus;
 

main.c

@@ -16,6 +16,7 @@
 #include "args.h"
 #include "config.h"
 #include "database/common.h"
+#include "database/query-table.h"
 #include "main.h"
 #include "signals.h"
 #include "regex_r.h"

request.c

@@ -16,7 +16,7 @@
 #include "socket.h"
 #include "resolve.h"
 #include "regex_r.h"
-#include "database/networktable.h"
+#include "database/network-table.h"
 #include "log.h"
 
 bool __attribute__((pure)) command(const char *client_message, const char* cmd) {