FTL/src/database/query-table.c

/* Pi-hole: A black hole for Internet advertisements
*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
*  Network-wide ad blocking via your own hardware.
*
*  FTL Engine
*  Query table database routines
*
*  This file is copyright under the latest version of the EUPL.
*  Please see LICENSE file for your rights under this license. */

#include "../FTL.h"
#include "query-table.h"
#include "common.h"
// get[Domain,ClientIP,Forward]String(), etc.
#include "../datastructure.h"
// getOverTimeID()
#include "../overTime.h"
// get_FTL_db_filesize()
#include "../files.h"
// timer_elapsed_msec()
#include "../timers.h"
// logg()
#include "../log.h"
// struct config
#include "../config.h"
// getstr()
#include "../shmem.h"

static bool saving_failed_before = false;

int get_number_of_queries_in_DB(sqlite3 *db)
{
	// Return early if database is known to be broken
	if(FTLDBerror())
		return DB_FAILED;

	// Open pihole-FTL.db database file if needed
	bool db_opened = false;
	if(db == NULL)
	{
		if((db = dbopen(false)) == NULL)
		{
			logg("get_number_of_queries_in_DB() - Failed to open DB");
			return DB_FAILED;
		}

		// Successful
		db_opened = true;
	}

	// Count number of rows using the index timestamp is faster than select(*)
	int result = db_query_int(db, "SELECT COUNT(timestamp) FROM query_storage");

	if(db_opened) dbclose(&db);

	return result;
}

int DB_save_queries(sqlite3 *db)
{
	// Return early if database is known to be broken
	if(FTLDBerror())
		return DB_FAILED;

	// Start database timer
	if(config.debug & DEBUG_DATABASE)
		timer_start(DATABASE_WRITE_TIMER);

	// Open pihole-FTL.db database file if needed
	bool db_opened = false;
	if(db == NULL)
	{
		if((db = dbopen(false)) == NULL)
		{
			logg("DB_save_queries() - Failed to open DB");
			return DB_FAILED;
		}

		// Successful
		db_opened = true;
	}

	int saved = 0;
	bool error = false;
	sqlite3_stmt *query_stmt = NULL;
	sqlite3_stmt *domain_stmt = NULL;
	sqlite3_stmt *client_stmt = NULL;
	sqlite3_stmt *forward_stmt = NULL;
	sqlite3_stmt *addinfo_stmt = NULL;

	int rc = dbquery(db, "BEGIN TRANSACTION IMMEDIATE");
	if( rc != SQLITE_OK )
	{
		const char *text;
		if( rc == SQLITE_BUSY )
			text = "WARNING";
		else
			text = "ERROR";

		logg("%s: Storing queries in long-term database failed: %s", text, sqlite3_errstr(rc));
		checkFTLDBrc(rc);

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	// Prepare statements
	rc  = sqlite3_prepare_v3(db, "INSERT INTO query_storage "
	                                 "(timestamp,type,status,domain,client,forward,additional_info,reply_type,reply_time,dnssec) "
	                                 "VALUES "
	                                 "(?1,?2,?3,"
	                                 "(SELECT id FROM domain_by_id WHERE domain = ?4),"
	                                 "(SELECT id FROM client_by_id WHERE ip = ?5 AND name = ?6),"
	                                 "(SELECT id FROM forward_by_id WHERE forward = ?7),"
	                                 "(SELECT id FROM addinfo_by_id WHERE type = ?8 AND content = ?9),"
	                                 "?10,?11,?12)",
	                         -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL);
	if( rc != SQLITE_OK )
	{
		const char *text, *spaces;
		if( rc == SQLITE_BUSY )
		{
			text   = "WARNING";
			spaces = "       ";
		}
		else
		{
			text   = "ERROR";
			spaces = "     ";
		}

		logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc));
		if(!checkFTLDBrc(rc))
			logg("%s  Keeping queries in memory for later new attempt", spaces);
		saving_failed_before = true;

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	rc = sqlite3_prepare_v3(db, "INSERT OR IGNORE INTO domain_by_id (domain) VALUES (?)",
	                        -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL);
	if( rc != SQLITE_OK )
	{
		const char *text, *spaces;
		if( rc == SQLITE_BUSY )
		{
			text   = "WARNING";
			spaces = "       ";
		}
		else
		{
			text   = "ERROR";
			spaces = "     ";
		}

		logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc));
		if(!checkFTLDBrc(rc))
			logg("%s  Keeping queries in memory for later new attempt", spaces);
		saving_failed_before = true;

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	rc = sqlite3_prepare_v3(db, "INSERT OR IGNORE INTO client_by_id (ip,name) VALUES (?,?)",
	                        -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL);
	if( rc != SQLITE_OK )
	{
		const char *text, *spaces;
		if( rc == SQLITE_BUSY )
		{
			text   = "WARNING";
			spaces = "       ";
		}
		else
		{
			text   = "ERROR";
			spaces = "     ";
		}

		logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc));
		if(!checkFTLDBrc(rc))
			logg("%s  Keeping queries in memory for later new attempt", spaces);
		saving_failed_before = true;

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	rc = sqlite3_prepare_v3(db, "INSERT OR IGNORE INTO forward_by_id (forward) VALUES (?)",
	                        -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL);
	if( rc != SQLITE_OK )
	{
		const char *text, *spaces;
		if( rc == SQLITE_BUSY )
		{
			text   = "WARNING";
			spaces = "       ";
		}
		else
		{
			text   = "ERROR";
			spaces = "     ";
		}

		logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc));
		if(!checkFTLDBrc(rc))
			logg("%s  Keeping queries in memory for later new attempt", spaces);
		saving_failed_before = true;

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	rc = sqlite3_prepare_v3(db, "INSERT OR IGNORE INTO addinfo_by_id (type,content) VALUES (?,?)",
	                        -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL);
	if( rc != SQLITE_OK )
	{
		const char *text, *spaces;
		if( rc == SQLITE_BUSY )
		{
			text   = "WARNING";
			spaces = "       ";
		}
		else
		{
			text   = "ERROR";
			spaces = "     ";
		}

		logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc));
		if(!checkFTLDBrc(rc))
			logg("%s  Keeping queries in memory for later new attempt", spaces);
		saving_failed_before = true;

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	// Get last ID stored in the database
	long int lastID = get_max_query_ID(db);

	int total = 0, blocked = 0;
	time_t currenttimestamp = time(NULL);
	time_t newlasttimestamp = 0;
	long int queryID;
	for(queryID = MAX(0, lastdbindex); queryID < counters->queries; queryID++)
	{
		queriesData* query = getQuery(queryID, true);
		if(!query)
		{
			// Memory error
			continue;
		}

		if(query->flags.database)
		{
			// Skip, already saved in database
			continue;
		}

		if(!query->flags.complete && query->timestamp > currenttimestamp-2)
		{
			// Break if a brand new query (age < 2 seconds) is not yet completed
			// giving it a chance to be stored next time
			break;
		}

		if(query->privacylevel >= PRIVACY_MAXIMUM)
		{
			// Skip, we never store nor count queries recorded
			// while have been in maximum privacy mode in the database
			continue;
		}

		// TIMESTAMP
		sqlite3_bind_int(query_stmt, 1, query->timestamp);

		// TYPE
		if(query->type != TYPE_OTHER)
		{
			// Store mapped type if query->type is not OTHER
			sqlite3_bind_int(query_stmt, 2, query->type);
		}
		else
		{
			// Store query type + offset if query-> type is OTHER
			sqlite3_bind_int(query_stmt, 2, query->qtype + 100);
		}

		// STATUS
		sqlite3_bind_int(query_stmt, 3, query->status);

		// DOMAIN
		const char *domain = getDomainString(query);
		sqlite3_bind_text(domain_stmt, 1, domain, -1, SQLITE_STATIC);
		sqlite3_bind_text(query_stmt, 4, domain, -1, SQLITE_STATIC);

		// Execute prepare client statement and check if successful
		if(sqlite3_step(domain_stmt) != SQLITE_DONE)
		{
			logg("Encountered error while trying to store client in long-term database");
			error = true;
			break;
		}
		sqlite3_clear_bindings(domain_stmt);
		sqlite3_reset(domain_stmt);

		// CLIENT
		const char *clientIP = getClientIPString(query);
		sqlite3_bind_text(query_stmt, 5, clientIP, -1, SQLITE_STATIC);
		sqlite3_bind_text(client_stmt, 1, clientIP, -1, SQLITE_STATIC);
		const char *clientName = getClientNameString(query);
		sqlite3_bind_text(query_stmt, 6, clientName, -1, SQLITE_STATIC);
		sqlite3_bind_text(client_stmt, 2, clientName, -1, SQLITE_STATIC);

		// Execute prepare client statement and check if successful
		if(sqlite3_step(client_stmt) != SQLITE_DONE)
		{
			logg("Encountered error while trying to store client in long-term database");
			error = true;
			break;
		}
		sqlite3_clear_bindings(client_stmt);
		sqlite3_reset(client_stmt);

		// FORWARD
		if(query->upstreamID > -1)
		{
			// Get forward pointer
			const upstreamsData* upstream = getUpstream(query->upstreamID, true);
			const char *forwardIP = getstr(upstream->ippos);
			if(upstream && forwardIP)
			{
				char *buffer = NULL;
				int len = 0; // The length of the string WITHOUT the NUL byte. This is what sqlite3_bind_text() expects.
				if((len = asprintf(&buffer, "%s#%u", forwardIP, upstream->port)) > 0)
				{
					// Use transient here as we step only after the buffer is freed below
					sqlite3_bind_text(query_stmt, 7, buffer, len, SQLITE_TRANSIENT);
					// Use static here as we insert right away
					sqlite3_bind_text(forward_stmt, 1, buffer, len, SQLITE_STATIC);

					// Execute prepared forward statement and check if successful
					if(sqlite3_step(forward_stmt) != SQLITE_DONE)
					{
						logg("Encountered error while trying to store forward destination in long-term database");
						error = true;
						break;
					}
					sqlite3_clear_bindings(forward_stmt);
					sqlite3_reset(forward_stmt);
				}
				else
				{
					// Memory error: Do not store the forward destination
					sqlite3_bind_null(query_stmt, 7);
				}

				if(buffer) free(buffer);
			}
		}
		else
		{
			// No forward destination
			sqlite3_bind_null(query_stmt, 7);
		}

		const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
		DNSCacheData *cache = getDNSCache(cacheID, true);

		// ADDITIONAL_INFO
		if(query->status == QUERY_GRAVITY_CNAME ||
		   query->status == QUERY_REGEX_CNAME ||
		   query->status == QUERY_BLACKLIST_CNAME)
		{
			// Save domain blocked during deep CNAME inspection
			const char *cname = getCNAMEDomainString(query);
			const int len = strlen(cname);
			sqlite3_bind_int(query_stmt, 8, ADDINFO_CNAME_DOMAIN);
			sqlite3_bind_text(query_stmt, 9, cname, len, SQLITE_STATIC);

			// Execute prepared addinfo statement and check if successful
			sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_CNAME_DOMAIN);
			sqlite3_bind_text(addinfo_stmt, 2, cname, len, SQLITE_STATIC);
			if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
			{
				logg("Encountered error while trying to store addinfo in long-term database (CNAME)");
				error = true;
				break;
			}
			sqlite3_clear_bindings(addinfo_stmt);
			sqlite3_reset(addinfo_stmt);
		}
		else if(cache != NULL && cache->domainlist_id > -1)
		{
			sqlite3_bind_int(query_stmt, 8, ADDINFO_REGEX_ID);
			sqlite3_bind_int(query_stmt, 9, cache->domainlist_id);

			// Execute prepared addinfo statement and check if successful
			sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
			sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id);
			if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
			{
				logg("Encountered error while trying to store addinfo in long-term database (domainlist_id)");
				error = true;
				break;
			}
			sqlite3_clear_bindings(addinfo_stmt);
			sqlite3_reset(addinfo_stmt);
		}
		else
		{
			// Nothing to add here
			sqlite3_bind_null(query_stmt, 8);
			sqlite3_bind_null(query_stmt, 9);
		}

		// REPLY_TYPE
		sqlite3_bind_int(query_stmt, 10, query->reply);

		// REPLY_TIME (stored in units of seconds) if available, NULL otherwise
		if(query->flags.response_calculated)
			sqlite3_bind_double(query_stmt, 11, 1e-4*query->response);
		else
			sqlite3_bind_null(query_stmt, 11);

		// DNSSEC
		sqlite3_bind_int(query_stmt, 12, query->dnssec);

		// Step and check if successful
		if(sqlite3_step(query_stmt) != SQLITE_DONE)
		{
			logg("Encountered error while trying to store queries in long-term database");
			error = true;
			break;
		}
		sqlite3_clear_bindings(query_stmt);
		sqlite3_reset(query_stmt);

		// Increment counters
		saved++;
		lastID++;

		// Mark this query as saved in the database
		query->flags.database = true;

		// Total counter information (delta computation)
		total++;
		if(query->flags.blocked)
			blocked++;

		// Update lasttimestamp variable with timestamp of the latest stored query
		if(query->timestamp > newlasttimestamp)
			newlasttimestamp = query->timestamp;
	}

	if(sqlite3_finalize(query_stmt) != SQLITE_OK ||
	   sqlite3_finalize(domain_stmt) != SQLITE_OK ||
	   sqlite3_finalize(client_stmt) != SQLITE_OK ||
	   sqlite3_finalize(forward_stmt) != SQLITE_OK ||
	   sqlite3_finalize(addinfo_stmt) != SQLITE_OK)
	{
		logg("Statement finalization failed when trying to store queries to long-term database");

		if(!checkFTLDBrc(rc) && rc == SQLITE_BUSY)
		{
			logg("Keeping queries in memory for later new attempt");
			saving_failed_before = true;
		}

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	// Store index for next loop iteration round and update last time stamp
	// in the database only if all queries have been saved successfully
	if(saved > 0 && !error)
	{
		lastdbindex = queryID;
		db_set_FTL_property(db, DB_LASTTIMESTAMP, newlasttimestamp);
		db_update_counters(db, total, blocked);
	}

	// Finish prepared statement
	if((rc = dbquery(db,"END TRANSACTION")) != SQLITE_OK)
	{
		// No need to log the error string here, dbquery() did that already above
		logg("END TRANSACTION failed when trying to store queries to long-term database");

		if(!checkFTLDBrc(rc) && rc == SQLITE_BUSY)
		{
			logg("Keeping queries in memory for later new attempt");
			saving_failed_before = true;
		}

		if(db_opened) dbclose(&db);

		return DB_FAILED;
	}

	if(config.debug & DEBUG_DATABASE || saving_failed_before)
	{
		logg("Notice: Queries stored in long-term database: %u (took %.1f ms, last SQLite ID %li)",
		     saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID);
		if(saving_failed_before)
		{
			logg("        Queries from earlier attempt(s) stored successfully");
			saving_failed_before = false;
		}
	}

	if(db_opened) dbclose(&db);

	return saved;
}

void delete_old_queries_in_DB(sqlite3 *db)
{
	// Return early if database is known to be broken
	if(FTLDBerror())
		return;

	int timestamp = time(NULL) - config.maxDBdays * 86400;

	if(dbquery(db, "DELETE FROM query_storage WHERE timestamp <= %i", timestamp) != SQLITE_OK)
	{
		logg("delete_old_queries_in_DB(): Deleting queries due to age of entries failed!");
		return;
	}

	// Get how many rows have been affected (deleted)
	const int affected = sqlite3_changes(db);

	// Print final message only if there is a difference
	if((config.debug & DEBUG_DATABASE) || affected)
		logg("Notice: Database size is %.2f MB, deleted %i rows", 1e-6*get_FTL_db_filesize(), affected);
}

bool add_additional_info_column(sqlite3 *db)
{
	// Add column additinal_info to queries table
	SQL_bool(db, "ALTER TABLE queries ADD COLUMN additional_info TEXT;");

	// Update the database version to 7
	SQL_bool(db, "INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %i );", DB_VERSION, 7);

	return true;
}

bool add_query_storage_columns(sqlite3 *db)
{
	// Start transaction of database update
	SQL_bool(db, "BEGIN TRANSACTION");

	// Add additional columns to the query_storage table
	SQL_bool(db, "ALTER TABLE query_storage ADD COLUMN reply_type INTEGER");
	SQL_bool(db, "ALTER TABLE query_storage ADD COLUMN reply_time REAL");
	SQL_bool(db, "ALTER TABLE query_storage ADD COLUMN dnssec INTEGER");

	// Update VIEW queries
	SQL_bool(db, "DROP VIEW queries");
	SQL_bool(db, "CREATE VIEW queries AS "
	                     "SELECT id, timestamp, type, status, "
	                       "CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,"
	                       "CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,"
	                       "CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,"
	                       "CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, "
	                       "reply_type, reply_time, dnssec "
	                       "FROM query_storage q");

	// Update database version to 12
	if(!db_set_FTL_property(db, DB_VERSION, 12))
	{
		logg("add_query_storage_columns(): Failed to update database version!");
		return false;
	}

	// Finish transaction
	SQL_bool(db, "COMMIT");

	return true;
}

bool optimize_queries_table(sqlite3 *db)
{
	// Start transaction of database update
	SQL_bool(db, "BEGIN TRANSACTION;");

	// Create link tables for domain, client, and forward strings
	SQL_bool(db, "CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);");
	SQL_bool(db, "CREATE TABLE client_by_id (id INTEGER PRIMARY KEY, ip TEXT NOT NULL, name TEXT);");
	SQL_bool(db, "CREATE TABLE forward_by_id (id INTEGER PRIMARY KEY, forward TEXT NOT NULL);");

	// Create UNIQUE index for the new tables
	SQL_bool(db, "CREATE UNIQUE INDEX domain_by_id_domain_idx ON domain_by_id(domain);");
	SQL_bool(db, "CREATE UNIQUE INDEX client_by_id_client_idx ON client_by_id(ip,name);");
	SQL_bool(db, "CREATE UNIQUE INDEX forward_by_id_forward_idx ON forward_by_id(forward);");

	// Rename current queries table
	SQL_bool(db, "ALTER TABLE queries RENAME TO query_storage;");

	// Change column definitions of the queries_storage table to allow
	// integer IDs. If we would leave the column definitions as TEXT, we
	// could not tell apart integer IDs easily as everything INSERTed would
	// be converted to TEXT form (this is very inefficient)
	// We have to turn off defensive mode to do this.
	SQL_bool(db, "PRAGMA writable_schema = ON;");
	SQL_bool(db, "UPDATE sqlite_master SET sql = 'CREATE TABLE \"query_storage\" (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain INTEGER NOT NULL, client INTEGER NOT NULL, forward INTEGER , additional_info TEXT)' WHERE type = 'table' AND name = 'query_storage';");
	SQL_bool(db, "PRAGMA writable_schema = OFF;");

	// Create VIEW queries so user scripts continue to work despite our
	// optimization here. The VIEW will pull the strings from the linked
	// tables when needed to always server the strings.
	SQL_bool(db, "CREATE VIEW queries AS "
	                     "SELECT id, timestamp, type, status, "
	                       "CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,"
	                       "CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,"
	                       "CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,"
	                       "additional_info FROM query_storage q;");

	// Update database version to 10
	if(!db_set_FTL_property(db, DB_VERSION, 10))
	{
		logg("optimize_queries_table(): Failed to update database version!");
		return false;
	}

	// Finish transaction
	SQL_bool(db, "COMMIT");

	return true;
}

bool create_addinfo_table(sqlite3 *db)
{
	// Start transaction of database update
	SQL_bool(db, "BEGIN TRANSACTION;");

	// Create link table for additional_info column
	SQL_bool(db, "CREATE TABLE addinfo_by_id (id INTEGER PRIMARY KEY, type INTEGER NOT NULL, content NOT NULL);");

	// Create UNIQUE index for the new tables
	SQL_bool(db, "CREATE UNIQUE INDEX addinfo_by_id_idx ON addinfo_by_id(type,content);");

	// Change column definitions of the queries_storage table to allow
	// integer IDs. If we would leave the column definitions as TEXT, we
	// could not tell apart integer IDs easily as everything INSERTed would
	// be converted to TEXT form (this is very inefficient)
	// We have to turn off defensive mode to do this.
	SQL_bool(db, "PRAGMA writable_schema = ON;");
	SQL_bool(db, "UPDATE sqlite_master SET sql = 'CREATE TABLE \"query_storage\" (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain INTEGER NOT NULL, client INTEGER NOT NULL, forward INTEGER, additional_info INTEGER)' WHERE type = 'table' AND name = 'query_storage';");
	SQL_bool(db, "PRAGMA writable_schema = OFF;");

	// Create VIEW queries so user scripts continue to work despite our
	// optimization here. The VIEW will pull the strings from the linked
	// tables when needed to always server the strings.
	SQL_bool(db, "DROP VIEW queries");
	SQL_bool(db, "CREATE VIEW queries AS "
	                     "SELECT id, timestamp, type, status, "
	                       "CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,"
	                       "CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,"
	                       "CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,"
	                       "CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info "
	                       "FROM query_storage q;");

	// Update database version to 11
	if(!db_set_FTL_property(db, DB_VERSION, 11))
	{
		logg("create_addinfo_table(): Failed to update database version!");
		return false;
	}

	// Finish transaction
	SQL_bool(db, "COMMIT");

	return true;
}

// Get most recent 24 hours data from long-term database
void DB_read_queries(void)
{
	// Return early if database is known to be broken
	if(FTLDBerror())
		return;

	// Open database
	sqlite3 *db;
	if((db = dbopen(false)) == NULL)
	{
		logg("DB_read_queries() - Failed to open DB");
		return;
	}

	// Prepare request
	// Get time stamp 24 hours in the past
	const time_t now = time(NULL);
	const time_t mintime = now - config.maxlogage;
	const char *querystr = "SELECT id,timestamp,type,status,domain,client,forward,additional_info,reply_type,reply_time,dnssec FROM queries WHERE timestamp >= ?";
	// Log FTL_db query string in debug mode
	if(config.debug & DEBUG_DATABASE)
		logg("DB_read_queries(): \"%s\" with ? = %lli", querystr, (long long)mintime);

	// Prepare SQLite3 statement
	sqlite3_stmt* stmt = NULL;
	int rc = sqlite3_prepare_v3(db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
	if( rc != SQLITE_OK ){
		logg("DB_read_queries() - SQL error prepare: %s", sqlite3_errstr(rc));
		checkFTLDBrc(rc);
		goto end_of_DB_read_queries;
	}

	// Bind limit
	if((rc = sqlite3_bind_int(stmt, 1, mintime)) != SQLITE_OK)
	{
		logg("DB_read_queries() - Failed to bind type mintime: %s", sqlite3_errstr(rc));
		checkFTLDBrc(rc);
		goto end_of_DB_read_queries;
	}

	// Lock shared memory
	lock_shm();

	// Loop through returned database rows
	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
	{
		const time_t queryTimeStamp = sqlite3_column_int(stmt, 1);
		// 1483228800 = 01/01/2017 @ 12:00am (UTC)
		if(queryTimeStamp < 1483228800)
		{
			logg("DB warn: TIMESTAMP should be larger than 01/01/2017 but is %lli", (long long)queryTimeStamp);
			continue;
		}
		if(queryTimeStamp > now)
		{
			if(config.debug & DEBUG_DATABASE) logg("DB warn: Skipping query logged in the future (%lli)", (long long)queryTimeStamp);
			continue;
		}

		const int type = sqlite3_column_int(stmt, 2);
		const bool mapped_type = type >= TYPE_A && type < TYPE_MAX;
		const bool offset_type = type > 100 && type < (100 + UINT16_MAX);
		if(!mapped_type && !offset_type)
		{
			logg("DB warn: TYPE should not be %i", type);
			continue;
		}
		// Don't import AAAA queries from database if the user set
		// AAAA_QUERY_ANALYSIS=no in pihole-FTL.conf
		if(type == TYPE_AAAA && !config.analyze_AAAA)
		{
			continue;
		}

		const int status_int = sqlite3_column_int(stmt, 3);
		if(status_int < QUERY_UNKNOWN || status_int >= QUERY_STATUS_MAX)
		{
			logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_STATUS_MAX-1, status_int);
			continue;
		}
		const enum query_status status = status_int;

		const char * domainname = (const char *)sqlite3_column_text(stmt, 4);
		if(domainname == NULL)
		{
			logg("DB warn: DOMAIN should never be NULL, %lli", (long long)queryTimeStamp);
			continue;
		}

		const char * clientIP = (const char *)sqlite3_column_text(stmt, 5);
		if(clientIP == NULL)
		{
			logg("DB warn: CLIENT should never be NULL, %lli", (long long)queryTimeStamp);
			continue;
		}

		// Check if user wants to skip queries coming from localhost
		if(config.ignore_localhost &&
		   (strcmp(clientIP, "127.0.0.1") == 0 || strcmp(clientIP, "::1") == 0))
		{
			continue;
		}

		// Ensure we have enough shared memory available for new data
		shm_ensure_size();

		const char *buffer = NULL;
		int upstreamID = -1; // Default if not forwarded
		// Try to extract the upstream from the "forward" column if non-empty
		if(sqlite3_column_bytes(stmt, 6) > 0 &&
		   (buffer = (const char *)sqlite3_column_text(stmt, 6)) != NULL)
		{
			// Get IP address and port of upstream destination
			char serv_addr[INET6_ADDRSTRLEN] = { 0 };
			unsigned int serv_port = 53;
			// We limit the number of bytes written into the serv_addr buffer
			// to prevent buffer overflows. If there is no port available in
			// the database, we skip extracting them and use the default port
			sscanf(buffer, "%"xstr(INET6_ADDRSTRLEN)"[^#]#%u", serv_addr, &serv_port);
			serv_addr[INET6_ADDRSTRLEN-1] = '\0';
			upstreamID = findUpstreamID(serv_addr, (in_port_t)serv_port);
		}

		int reply_type = REPLY_UNKNOWN;
		if(sqlite3_column_type(stmt, 8) == SQLITE_INTEGER)
		{
			// The field has been added for database version 12
			reply_type = sqlite3_column_int(stmt, 8);
			if(reply_type < REPLY_UNKNOWN || reply_type >= QUERY_REPLY_MAX)
			{
				logg("DB warn: REPLY value %i is invalid, %lli", reply_type, (long long)queryTimeStamp);
				continue;
			}
		}

		double reply_time = 0.0;
		bool reply_time_avail = false;
		if(sqlite3_column_type(stmt, 9) == SQLITE_FLOAT)
		{
			// The field has been added for database version 12
			reply_time = sqlite3_column_double(stmt, 9);
			reply_time_avail = true;
			if(reply_time < 0.0)
			{
				logg("DB warn: REPLY_TIME value %f is invalid, %lli", reply_time, (long long)queryTimeStamp);
				continue;
			}
		}

		int dnssec = DNSSEC_UNSPECIFIED;
		if(sqlite3_column_type(stmt, 10) == SQLITE_INTEGER)
		{
			// The field has been added for database version 12
			dnssec = sqlite3_column_int(stmt, 10);
			if(dnssec < DNSSEC_UNSPECIFIED || dnssec > DNSSEC_ABANDONED)
			{
				logg("DB warn: DNSSEC value %i is invalid, %lli", dnssec, (long long)queryTimeStamp);
				continue;
			}
		}

		// Obtain IDs only after filtering which queries we want to keep
		const int timeidx = getOverTimeID(queryTimeStamp);
		const int domainID = findDomainID(domainname, true);
		const int clientID = findClientID(clientIP, true, false);

		// Set index for this query
		const int queryIndex = counters->queries;

		// Store this query in memory
		queriesData* query = getQuery(queryIndex, false);
		query->magic = MAGICBYTE;
		query->timestamp = queryTimeStamp;
		if(type < 100)
		{
			// Mapped query type
			if(type >= TYPE_A && type < TYPE_MAX)
				query->type = type;
			else
			{
				// Invalid query type
				logg("DB warn: Query type %d is invalid.", type);
				continue;
			}
		}
		else
		{
			// Offset query type
			query->type = TYPE_OTHER;
			query->qtype = type - 100;
		}

		// Status is set below
		query->domainID = domainID;
		query->clientID = clientID;
		query->upstreamID = upstreamID;
		query->id = 0;
		query->response = 0;
		query->flags.response_calculated = reply_time_avail;
		query->dnssec = dnssec;
		query->reply = reply_type;
		counters->reply[query->reply]++;
		query->response = reply_time * 1e4; // convert to tenth-millisecond unit
		query->CNAME_domainID = -1;
		// Initialize flags
		query->flags.complete = true; // Mark as all information is available
		query->flags.blocked = false;
		query->flags.whitelisted = false;
		query->flags.database = true;
		query->ede = -1; // EDE_UNSET == -1

		// Set lastQuery timer for network table
		clientsData* client = getClient(clientID, true);
		client->lastQuery = queryTimeStamp;

		// Handle type counters
		counters->querytype[query->type-1]++;

		// Update overTime data
		overTime[timeidx].total++;
		// Update overTime data structure with the new client
		change_clientcount(client, 0, 0, timeidx, 1);

		// Increase DNS queries counter
		counters->queries++;

		// Get additional information from the additional_info column if applicable
		if(status == QUERY_GRAVITY_CNAME ||
		   status == QUERY_REGEX_CNAME ||
		   status == QUERY_BLACKLIST_CNAME)
		{
			// QUERY_*_CNAME: Get domain causing the blocking
			const char *CNAMEdomain = (const char *)sqlite3_column_text(stmt, 7);
			if(CNAMEdomain != NULL && strlen(CNAMEdomain) > 0)
			{
				// Add domain to FTL's memory but do not count it. Seeing a
				// domain in the middle of a CNAME trajectory does not mean
				// it was queried intentionally.
				const int CNAMEdomainID = findDomainID(CNAMEdomain, false);
				query->CNAME_domainID = CNAMEdomainID;
			}
		}
		else if(sqlite3_column_bytes(stmt, 7) != 0)
		{
			// Set ID of the domainlist entry that was the reason for permitting/blocking this query
			// We assume the value in this field is said ID when it is not a CNAME-related domain
			// (checked above) and the value of additional_info is not NULL (0 bytes storage size)
			const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true);
			DNSCacheData *cache = getDNSCache(cacheID, true);
			// Only load if
			//  a) we have a cache entry
			if(cache != NULL)
				cache->domainlist_id = sqlite3_column_int(stmt, 7);
		}

		// Increment status counters, we first have to add one to the count of
		// unknown queries because query_set_status() will subtract from there
		// when setting a different status
		counters->status[QUERY_UNKNOWN]++;
		query_set_status(query, status);

		// Do further processing based on the query status we read from the database
		switch(status)
		{
			case QUERY_UNKNOWN: // Unknown
				break;

			case QUERY_GRAVITY: // Blocked by gravity
			case QUERY_REGEX: // Blocked by regex blacklist
			case QUERY_BLACKLIST: // Blocked by exact blacklist
			case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider
			case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider
			case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider
			case QUERY_GRAVITY_CNAME: // Blocked by gravity (inside CNAME path)
			case QUERY_REGEX_CNAME: // Blocked by regex blacklist (inside CNAME path)
			case QUERY_BLACKLIST_CNAME: // Blocked by exact blacklist (inside CNAME path)
			case QUERY_DBBUSY: // Blocked because gravity database was busy
			case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling
				query->flags.blocked = true;
				// Get domain pointer
				domainsData* domain = getDomain(domainID, true);
				domain->blockedcount++;
				change_clientcount(client, 0, 1, -1, 0);
				break;

			case QUERY_FORWARDED: // Forwarded
			case QUERY_RETRIED: // (fall through)
			case QUERY_RETRIED_DNSSEC: // (fall through)
				// Only update upstream if there is one (there
				// won't be one for retried DNSSEC queries)
				if(upstreamID > -1)
				{
					upstreamsData *upstream = getUpstream(upstreamID, true);
					if(upstream != NULL)
					{
						upstream->overTime[timeidx]++;
						upstream->lastQuery = queryTimeStamp;
					}
				}
				break;

			case QUERY_CACHE: // Cached or local config
			case QUERY_CACHE_STALE:
				// Nothing to be done here
				break;

			case QUERY_IN_PROGRESS:
				// Nothing to be done here
				break;

			case QUERY_STATUS_MAX:
			default:
				logg("Warning: Found unknown status %i in long term database!", status);
				break;
		}
	}

	unlock_shm();
	logg("Imported %i queries from the long-term database", counters->queries);

	// Update lastdbindex so that the next call to DB_save_queries()
	// skips the queries that we just imported from the database
	lastdbindex = counters->queries;

	if( rc != SQLITE_DONE ){
		logg("DB_read_queries() - SQL error step: %s", sqlite3_errstr(rc));
		checkFTLDBrc(rc);
		goto end_of_DB_read_queries;
	}

	// Finalize SQLite3 statement
	sqlite3_finalize(stmt);

end_of_DB_read_queries:	// Close database here, we have to reopen it later (after forking)
	dbclose(&db);
}