pi-hole/FTL
1
0
Fork 0
mirror of https://github.com/pi-hole/FTL.git synced 2024-10-26 16:52:18 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rename query_storage.regex_id to query_storage.list_id as it is already now used to also store exact matching domainlist entries by their ID. This commit further extends this to also store the (first) matching anti-/gravity list (if available)

Browse Source 
Signed-off-by: DL6ER <dl6er@dl6er.de>
This commit is contained in:
DL6ER
2023-12-25 04:54:44 +01:00
parent 07c403a358
commit 1bc8e7fc06
12 changed files with 111 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/docs/content/specs/queries.yaml
+4 -4
View File
@@ -216,9 +216,9 @@ components:
time:
type: number
description: Time until the response was received (ms, negative if N/A)
regex_id:
list_id:
type: integer
description: ID of regex (`NULL` if N/A)
description: ID of corresponding database table (adlist for anti-/gravity, else domainlist) (`NULL` if N/A)
nullable: true
upstream:
type: string
@@ -237,7 +237,7 @@ components:
reply:
type: "IP"
time: 19
regex_id: NULL
list_id: NULL
upstream: "localhost#5353"
dbid: 112421354
- time: 1581907871.583821
@@ -252,7 +252,7 @@ components:
reply:
type: "IP"
time: 12.3
regex_id: NULL
list_id: NULL
upstream: "localhost#5353"
dbid: 112421355
cursor:
src/api/queries.c
+6 -6
View File
@@ -174,7 +174,7 @@ int api_queries_suggestions(struct ftl_conn *api)
JSON_SEND_OBJECT(json);
}
#define QUERYSTR "SELECT q.id,timestamp,q.type,status,d.domain,f.forward,additional_info,reply_type,reply_time,dnssec,c.ip,c.name,a.content,regex_id"
#define QUERYSTR "SELECT q.id,timestamp,q.type,status,d.domain,f.forward,additional_info,reply_type,reply_time,dnssec,c.ip,c.name,a.content,list_id"
// JOIN: Only return rows where there is a match in BOTH tables
// LEFT JOIN: Return all rows from the left table, and the matched rows from the right table
#define JOINSTR "JOIN client_by_id c ON q.client = c.id JOIN domain_by_id d ON q.domain = d.id LEFT JOIN forward_by_id f ON q.forward = f.id LEFT JOIN addinfo_by_id a ON a.id = q.additional_info"
@@ -215,8 +215,8 @@ static void querystr_finish(char *querystr, const char *sort_col, const char *so
sort_col_sql = "q.reply_time";
else if(strcasecmp(sort_col, "dnssec") == 0)
sort_col_sql = "q.dnssec";
else if(strcasecmp(sort_col, "regex_id") == 0)
sort_col_sql = "regex_id";
else if(strcasecmp(sort_col, "list_id") == 0)
sort_col_sql = "list_id";
// ... and the sort direction
if(strcasecmp(sort_dir, "asc") == 0 || strcasecmp(sort_dir, "ascending") == 0)
@@ -828,11 +828,11 @@ int api_queries(struct ftl_conn *api)
JSON_ADD_NULL_TO_OBJECT(client, "name");
JSON_ADD_ITEM_TO_OBJECT(item, "client", client);
// Add regex_id if it exists
// Add list_id if it exists
if(sqlite3_column_type(read_stmt, 13) == SQLITE_INTEGER)
JSON_ADD_NUMBER_TO_OBJECT(item, "regex_id", sqlite3_column_int(read_stmt, 13)); // regex_id
JSON_ADD_NUMBER_TO_OBJECT(item, "list_id", sqlite3_column_int(read_stmt, 13)); // list_id
else
JSON_ADD_NULL_TO_OBJECT(item, "regex_id");
JSON_ADD_NULL_TO_OBJECT(item, "list_id");
const unsigned char *cname = NULL;
switch(query.status)
src/database/common.c
+20
View File
@@ -543,6 +543,26 @@ void db_init(void)
dbversion = db_get_int(db, DB_VERSION);
}
// Update to version 17 if lower
if(dbversion < 17)
{
// Update to version 17: Rename regex_id column to regex_id_old
log_info("Updating long-term database to version 17");
if(!rename_query_storage_column_regex_id(db))
{
log_info("regex_id cannot be renamed to list_id, database not available");
dbclose(&db);
return;
}
// Get updated version
dbversion = db_get_int(db, DB_VERSION);
}
// Last check after all migrations, if this happens, it will cause the
// CI to fail the tests
if(dbversion != MEMDB_VERSION)
log_err("Database version %i does not match MEMDB_VERSION %i", dbversion, MEMDB_VERSION);
lock_shm();
import_aliasclients(db);
unlock_shm();
src/database/gravity-db.c
+2 -2
View File
@@ -1262,7 +1262,7 @@ enum db_result in_allowlist(const char *domain, DNSCacheData *dns_cache, clients
// We have to check both the exact whitelist (using a prepared database statement)
// as well the compiled regex whitelist filters to check if the current domain is
// whitelisted.
return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id);
return domain_in_list(domain, stmt, "whitelist", &dns_cache->list_id);
}
cJSON *gen_abp_patterns(const char *domain, const bool antigravity)
@@ -1466,7 +1466,7 @@ enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsD
if(stmt == NULL)
stmt = blacklist_stmt->get(blacklist_stmt, client->id);
return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id);
return domain_in_list(domain, stmt, "blacklist", &dns_cache->list_id);
}
bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex,
src/database/query-table.c
+32 -9
View File
@@ -784,6 +784,29 @@ bool add_ftl_table_description(sqlite3 *db)
return true;
}
bool rename_query_storage_column_regex_id(sqlite3 *db)
{
// Start transaction of database update
SQL_bool(db, "BEGIN TRANSACTION");
// Rename column regex_id to list_id
SQL_bool(db, "ALTER TABLE query_storage RENAME COLUMN regex_id TO list_id;");
// The VIEW queries is automatically updated by SQLite3
// Update database version to 17
if(!db_set_FTL_property(db, DB_VERSION, 17))
{
log_err("rename_query_storage_column_regex_id(): Failed to update database version!");
return false;
}
// Finish transaction
SQL_bool(db, "COMMIT");
return true;
}
bool optimize_queries_table(sqlite3 *db)
{
// Start transaction of database update
@@ -1131,7 +1154,7 @@ void DB_read_queries(void)
// a) we have a cache entry
// b) the value of additional_info is not NULL (0 bytes storage size)
if(cache != NULL && sqlite3_column_bytes(stmt, 7) != 0)
cache->domainlist_id = sqlite3_column_int(stmt, 7);
cache->list_id = sqlite3_column_int(stmt, 7);
}
// Increment status counters
@@ -1485,15 +1508,15 @@ bool queries_to_database(void)
break;
}
}
else if(cache != NULL && query->status == QUERY_REGEX)
else if(cache != NULL && cache->list_id > -1)
{
// Restore regex ID if applicable
sqlite3_bind_int(query_stmt, 9, ADDINFO_REGEX_ID);
sqlite3_bind_int(query_stmt, 10, cache->domainlist_id);
sqlite3_bind_int(query_stmt, 9, ADDINFO_LIST_ID);
sqlite3_bind_int(query_stmt, 10, cache->list_id);
// Execute prepared addinfo statement and check if successful
sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id);
sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_LIST_ID);
sqlite3_bind_int(addinfo_stmt, 2, cache->list_id);
rc = sqlite3_step(addinfo_stmt);
sqlite3_clear_bindings(addinfo_stmt);
sqlite3_reset(addinfo_stmt);
@@ -1524,9 +1547,9 @@ bool queries_to_database(void)
// DNSSEC
sqlite3_bind_int(query_stmt, 13, query->dnssec);
// REGEX_ID
if(cache != NULL && cache->domainlist_id > -1)
sqlite3_bind_int(query_stmt, 14, cache->domainlist_id);
// LIST_ID
if(cache != NULL && cache->list_id > -1)
sqlite3_bind_int(query_stmt, 14, cache->list_id);
else
// Not applicable, setting NULL
sqlite3_bind_null(query_stmt, 14);
src/database/query-table.h
+20 -20
View File
@@ -23,20 +23,21 @@
"client TEXT NOT NULL, " \
"forward TEXT );"
#define CREATE_QUERY_STORAGE_TABLE_V13 "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \
"timestamp INTEGER NOT NULL, " \
"type INTEGER NOT NULL, " \
"status INTEGER NOT NULL, " \
"domain INTEGER NOT NULL, " \
"client INTEGER NOT NULL, " \
"forward INTEGER, " \
"additional_info INTEGER, " \
"reply_type INTEGER, " \
"reply_time REAL, " \
"dnssec INTEGER, " \
"regex_id INTEGER );"
#define MEMDB_VERSION 17
#define CREATE_QUERY_STORAGE_TABLE "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \
"timestamp INTEGER NOT NULL, " \
"type INTEGER NOT NULL, " \
"status INTEGER NOT NULL, " \
"domain INTEGER NOT NULL, " \
"client INTEGER NOT NULL, " \
"forward INTEGER, " \
"additional_info INTEGER, " \
"reply_type INTEGER, " \
"reply_time REAL, " \
"dnssec INTEGER, " \
"list_id INTEGER );"
#define CREATE_QUERIES_VIEW_V13 "CREATE VIEW queries AS " \
#define CREATE_QUERIES_VIEW "CREATE VIEW queries AS " \
"SELECT id, timestamp, type, status, " \
"CASE typeof(domain) " \
"WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain," \
@@ -46,7 +47,7 @@
"WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward," \
"CASE typeof(additional_info) "\
"WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, " \
"reply_type, reply_time, dnssec, regex_id FROM query_storage q"
"reply_type, reply_time, dnssec, list_id FROM query_storage q"
// Version 1
#define CREATE_QUERIES_TIMESTAMP_INDEX "CREATE INDEX idx_queries_timestamp ON queries (timestamp);"
@@ -62,8 +63,7 @@
#define CREATE_QUERY_STORAGE_REPLY_TYPE_INDEX "CREATE INDEX idx_query_storage_reply_type ON query_storage (reply_type);"
#define CREATE_QUERY_STORAGE_REPLY_TIME_INDEX "CREATE INDEX idx_query_storage_reply_time ON query_storage (reply_time);"
#define CREATE_QUERY_STORAGE_DNSSEC_INDEX "CREATE INDEX idx_query_storage_dnssec ON query_storage (dnssec);"
//#define CREATE_QUERY_STORAGE_TTL_INDEX "CREATE INDEX idx_query_storage_ttl ON query_storage (ttl);"
//#define CREATE_QUERY_STORAGE_REGEX_ID_INDEX "CREATE INDEX idx_query_storage_regex_id ON query_storage (regex_id);"
#define CREATE_QUERY_STORAGE_LIST_ID_INDEX "CREATE INDEX idx_query_storage_list_id ON query_storage (list_id);"
#define CREATE_DOMAINS_BY_ID "CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"
#define CREATE_CLIENTS_BY_ID "CREATE TABLE client_by_id (id INTEGER PRIMARY KEY, ip TEXT NOT NULL, name TEXT);"
@@ -77,12 +77,12 @@
#ifdef QUERY_TABLE_PRIVATE
const char *table_creation[] = {
CREATE_QUERY_STORAGE_TABLE_V13,
CREATE_QUERY_STORAGE_TABLE,
CREATE_DOMAINS_BY_ID,
CREATE_CLIENTS_BY_ID,
CREATE_FORWARD_BY_ID,
CREATE_ADDINFO_BY_ID,
CREATE_QUERIES_VIEW_V13,
CREATE_QUERIES_VIEW,
};
const char *index_creation[] = {
CREATE_QUERY_STORAGE_ID_INDEX,
@@ -96,8 +96,7 @@ const char *index_creation[] = {
CREATE_QUERY_STORAGE_REPLY_TYPE_INDEX,
CREATE_QUERY_STORAGE_REPLY_TIME_INDEX,
CREATE_QUERY_STORAGE_DNSSEC_INDEX,
// CREATE_QUERY_STORAGE_TTL_INDEX,
// CREATE_QUERY_STORAGE_REGEX_ID_INDEX
CREATE_QUERY_STORAGE_LIST_ID_INDEX
CREATE_DOMAIN_BY_ID_DOMAIN_INDEX,
CREATE_CLIENTS_BY_ID_IPNAME_INDEX,
CREATE_FORWARD_BY_ID_FORWARD_INDEX,
@@ -128,5 +127,6 @@ bool create_addinfo_table(sqlite3 *db);
bool add_query_storage_columns(sqlite3 *db);
bool add_query_storage_column_regex_id(sqlite3 *db);
bool add_ftl_table_description(sqlite3 *db);
bool rename_query_storage_column_regex_id(sqlite3 *db);
#endif //QUERY_TABLE_PRIVATE_H
src/datastructure.c
+1 -1
View File
@@ -433,7 +433,7 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q
dns_cache->clientID = clientID;
dns_cache->query_type = query_type;
dns_cache->force_reply = 0u;
dns_cache->domainlist_id = -1; // -1 = not set
dns_cache->list_id = -1; // -1 = not set
// Increase counter by one
counters->dns_cache_size++;
src/datastructure.h
+1 -1
View File
@@ -112,7 +112,7 @@ typedef struct {
enum query_type query_type;
int domainID;
int clientID;
int domainlist_id;
int list_id;
char *cname_target;
} DNSCacheData;
src/dnsmasq_interface.c
+20 -11
View File
@@ -1139,16 +1139,20 @@ static bool check_domain_blocked(const char *domain, const int clientID,
}
// Check domain against antigravity
int domain_id = -1;
const enum db_result antigravity = in_gravity(domain, client, true, &domain_id);
int list_id = -1;
const enum db_result antigravity = in_gravity(domain, client, true, &list_id);
if(antigravity == FOUND)
{
log_debug(DEBUG_QUERIES, "Allowing query due to antigravity match (ID %i)", domain_id);
log_debug(DEBUG_QUERIES, "Allowing query due to antigravity match (list ID %i)", list_id);
// Store ID of the matching antigravity list
dns_cache->list_id = list_id;
return false;
}
// Check domains against gravity domains
const enum db_result gravity = in_gravity(domain, client, false, &domain_id);
const enum db_result gravity = in_gravity(domain, client, false, &list_id);
if(gravity == FOUND)
{
// Set new status
@@ -1158,6 +1162,11 @@ static bool check_domain_blocked(const char *domain, const int clientID,
// Mark domain as gravity blocked for this client
set_dnscache_blockingstatus(dns_cache, client, GRAVITY_BLOCKED, domain);
log_debug(DEBUG_QUERIES, "Blocking query due to gravity match (list ID %i)", list_id);
// Store ID of the matching gravity list
dns_cache->list_id = list_id;
// We block this domain
return true;
}
@@ -1218,7 +1227,7 @@ static bool check_domain_blocked(const char *domain, const int clientID,
cname_target = dns_cache->cname_target;
// Store ID of this regex (fork-private)
last_regex_idx = dns_cache->domainlist_id;
last_regex_idx = dns_cache->list_id;
// We block this domain
return true;
@@ -1353,14 +1362,14 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
// the lengthy tests below
blockingreason = "regex denied";
log_debug(DEBUG_QUERIES, "%s is known as %s (cache regex ID: %i)",
domainstr, blockingreason, dns_cache->domainlist_id);
domainstr, blockingreason, dns_cache->list_id);
// Do not block if the entire query is to be permitted as something
// along the CNAME path hit the whitelist
if(!query->flags.allowed)
{
force_next_DNS_reply = dns_cache->force_reply;
last_regex_idx = dns_cache->domainlist_id;
last_regex_idx = dns_cache->list_id;
query_blocked(query, domain, client, QUERY_REGEX);
return true;
}
@@ -1472,7 +1481,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
if(config.debug.queries.v.b)
{
log_debug(DEBUG_QUERIES, "Blocking %s as %s is %s (domainlist ID: %i)",
domainstr, blockedDomain, blockingreason, dns_cache->domainlist_id);
domainstr, blockedDomain, blockingreason, dns_cache->list_id);
if(force_next_DNS_reply != 0)
log_debug(DEBUG_QUERIES, "Forcing next reply to %s", get_query_reply_str(force_next_DNS_reply));
}
@@ -1487,7 +1496,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
// Debug output
// client is guaranteed to be non-NULL above
log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s (domainlist ID: %i)", getstr(client->ippos),
domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->domainlist_id);
domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->list_id);
}
free(domainstr);
@@ -1597,8 +1606,8 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file
// Propagate ID of responsible regex up from the child to the parent
// domain (but only if set)
if(parent_cache != NULL && child_cache != NULL && child_cache->domainlist_id != -1)
parent_cache->domainlist_id = child_cache->domainlist_id;
if(parent_cache != NULL && child_cache != NULL && child_cache->list_id != -1)
parent_cache->list_id = child_cache->list_id;
// Set status
query_set_status(query, QUERY_REGEX_CNAME);
src/enums.h
+1 -1
View File
@@ -283,7 +283,7 @@ enum ptr_type {
enum addinfo_type {
ADDINFO_CNAME_DOMAIN = 1,
ADDINFO_REGEX_ID
ADDINFO_LIST_ID
} __attribute__ ((packed));
enum listening_mode {
src/regex.c
+1 -1
View File
@@ -548,7 +548,7 @@ bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, c
if(regex_id != -1)
{
// We found a match
dns_cache->domainlist_id = regex_id;
dns_cache->list_id = regex_id;
return true;
}
test/test_suite.bats
+3 -3
View File
@@ -454,16 +454,16 @@
@test "pihole-FTL.db schema is as expected" {
run bash -c './pihole-FTL sqlite3 /etc/pihole/pihole-FTL.db .dump'
printf "%s\n" "${lines[@]}"
[[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"query_storage\" (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain INTEGER NOT NULL, client INTEGER NOT NULL, forward INTEGER, additional_info INTEGER, reply_type INTEGER, reply_time REAL, dnssec INTEGER, regex_id INTEGER);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"query_storage\" (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain INTEGER NOT NULL, client INTEGER NOT NULL, forward INTEGER, additional_info INTEGER, reply_type INTEGER, reply_time REAL, dnssec INTEGER, list_id INTEGER);"* ]]
[[ "${lines[@]}" == *"CREATE INDEX idx_queries_timestamps ON \"query_storage\" (timestamp);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE ftl (id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL, description TEXT);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE counters (id INTEGER PRIMARY KEY NOT NULL, value INTEGER NOT NULL);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network\" (id INTEGER PRIMARY KEY NOT NULL, hwaddr TEXT UNIQUE NOT NULL, interface TEXT NOT NULL, firstSeen INTEGER NOT NULL, lastQuery INTEGER NOT NULL, numQueries INTEGER NOT NULL, macVendor TEXT, aliasclient_id INTEGER);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network_addresses\" (network_id INTEGER NOT NULL, ip TEXT UNIQUE NOT NULL, lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), name TEXT, nameUpdated INTEGER, FOREIGN KEY(network_id) REFERENCES network(id));"* ]]
[[ "${lines[@]}" == *"CREATE TABLE aliasclient (id INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, comment TEXT);"* ]]
[[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,16,'Database version');"* ]]
[[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,17,'Database version');"* ]]
# vvv This has been added in version 10 vvv
[[ "${lines[@]}" == *"CREATE VIEW queries AS SELECT id, timestamp, type, status, CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, reply_type, reply_time, dnssec, regex_id FROM query_storage q;"* ]]
[[ "${lines[@]}" == *"CREATE VIEW queries AS SELECT id, timestamp, type, status, CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, reply_type, reply_time, dnssec, list_id FROM query_storage q;"* ]]
[[ "${lines[@]}" == *"CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE client_by_id (id INTEGER PRIMARY KEY, ip TEXT NOT NULL, name TEXT);"* ]]
[[ "${lines[@]}" == *"CREATE TABLE forward_by_id (id INTEGER PRIMARY KEY, forward TEXT NOT NULL);"* ]]