From 4c74a29f87e2fc86b73b9b8debf28fa64cc4b062 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 30 Dec 2019 17:13:57 +0000 Subject: [PATCH] Add shared per-client regex array. This ensures TCP and UDP workers know the same details about client/regex combinations. This commit also fixes an issue with regex group associations for configured clients that have no assigned group. Signed-off-by: DL6ER --- src/FTL.h | 1 + src/database/gravity-db.c | 82 +++++++++++++++++++++++++++++++++------ src/database/gravity-db.h | 4 +- src/datastructure.c | 6 +-- src/datastructure.h | 1 - src/dnsmasq_interface.c | 4 +- src/memory.h | 1 + src/regex.c | 82 ++++++++++++++++----------------------- src/regex_r.h | 6 +-- src/shmem.c | 63 ++++++++++++++++++++++++++++++ src/shmem.h | 6 +++ 11 files changed, 184 insertions(+), 72 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index cc8fafe9..db56bc6f 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -112,6 +112,7 @@ enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER }; enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS }; enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; +enum { REGEX_BLACKLIST, REGEX_WHITELIST }; // Use out own memory handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crashs diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index a13eeb6e..31eec6b4 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -109,6 +109,7 @@ static bool get_client_groupids(const clientsData* client, char **groups) // Get associated groups for this client (if defined) char *querystr = NULL; const char *ip = getstr(client->ippos); + *groups = NULL; // Do not proceed when database is not available if(!gravity_database_avail) @@ -117,21 +118,75 @@ static bool get_client_groupids(const clientsData* client, char **groups) return false; } + if(config.debug & DEBUG_DATABASE) + logg("Querying gravity database for client %s", ip); + + // Check if client is configured through the client table + if(asprintf(&querystr, "SELECT COUNT(*) FROM client WHERE ip = \'%s\';", ip) < 1) + { + logg("get_client_groupids() - asprintf() error 1"); + return false; + } + + // Prepare query + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + if(rc != SQLITE_OK){ + logg("get_client_groupids(%s) - SQL error prepare (%i): %s", + querystr, rc, sqlite3_errmsg(gravity_db)); + sqlite3_finalize(table_stmt); + gravityDB_close(); + free(querystr); + return false; + } + + // Perform query + rc = sqlite3_step(table_stmt); + if(rc == SQLITE_ROW) + { + // There is a record for this client in the database + const int result = sqlite3_column_int(table_stmt, 0); + + // Found no record for this client in the database + // This makes this client qualify for the special "all" group + if(result == 0) + *groups = strdup("0"); + } + else if(rc == SQLITE_DONE) + { + // Found no record for this client in the database + // This makes this client qualify for the special "all" group + *groups = strdup("0"); + } + else + { + logg("get_client_groupids(%s) - SQL error step (%i): %s", + querystr, rc, sqlite3_errmsg(gravity_db)); + sqlite3_finalize(table_stmt); + gravityDB_close(); + free(querystr); + return false; + } + // Finalize statement + gravityDB_finalizeTable(); + + if(*groups != NULL) + { + // The client is not configured through the client table, return early + return true; + } + // Build query string to get possible group associations for this particular client // The SQL GROUP_CONCAT() function returns a string which is the concatenation of all // non-NULL values of group_id separated by ','. The order of the concatenated elements // is arbitrary, however, is of no relevance for your use case. if(asprintf(&querystr, "SELECT GROUP_CONCAT(group_id) FROM client_by_group WHERE client_id = (SELECT id FROM client WHERE ip = \'%s\');", ip) < 1) { - logg("get_client_groupids() - asprintf() error"); + logg("get_client_groupids() - asprintf() error 2"); return false; } - if(config.debug & DEBUG_DATABASE) - logg("Querying gravity database for client %s", ip); - // Prepare query - int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); if(rc != SQLITE_OK){ logg("get_client_groupids(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db)); @@ -150,13 +205,13 @@ static bool get_client_groupids(const clientsData* client, char **groups) if(result != NULL) *groups = strdup(result); else - *groups = strdup("0"); + *groups = strdup(""); } else if(rc == SQLITE_DONE) { // Found no record for this client in the database - // This makes this client qualify for the special "all" group - *groups = strdup("0"); + // -> No associated groups + *groups = strdup(""); } else { @@ -482,7 +537,7 @@ static bool domain_in_list(const char *domain, sqlite3_stmt* stmt, const char* l return (result == 1); } -inline bool in_whitelist(const char *domain, clientsData* client) +inline bool in_whitelist(const char *domain, clientsData* client, const int clientID) { if(client->whitelist_stmt == NULL) gravityDB_prepare_client_statements(client); @@ -493,7 +548,7 @@ inline bool in_whitelist(const char *domain, clientsData* client) // optimization as the database lookup will most likely hit (a) more domains and (b) // will be faster (given a sufficiently large number of regex whitelisting filters). return domain_in_list(domain, client->whitelist_stmt, "whitelist") || - match_regex(domain, client, REGEX_WHITELIST); + match_regex(domain, clientID, REGEX_WHITELIST); } inline bool in_gravity(const char *domain, clientsData* client) @@ -517,7 +572,7 @@ bool in_auditlist(const char *domain) } bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid, - const unsigned char type, const char* table) + const unsigned char type, const char* table, const int clientID) { char *querystr = NULL; char *groups = NULL; @@ -553,7 +608,10 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, { if(regexid[i] == result) { - client->regex_enabled[type][i] = true; + unsigned int regexID = i; + if(type == REGEX_WHITELIST) + regexID += counters->num_regex[REGEX_BLACKLIST]; + set_per_client_regex(clientID, regexID, true); break; } } diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 40f5f748..c4bab199 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -25,11 +25,11 @@ int gravityDB_count(unsigned char list); bool in_auditlist(const char *domain); bool in_gravity(const char *domain, clientsData* client); -bool in_whitelist(const char *domain, clientsData* client); +bool in_whitelist(const char *domain, clientsData* client, const int clientID); bool in_blacklist(const char *domain, clientsData* client); bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid, - const unsigned char type, const char* table); + const unsigned char type, const char* table, const int clientID); #endif //GRAVITY_H diff --git a/src/datastructure.c b/src/datastructure.c index 05457c39..bffce83e 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -200,12 +200,12 @@ int findClientID(const char *clientIP, const bool count) for(int i = 0; i < OVERTIME_SLOTS; i++) client->overTime[i] = 0; - // Allocate regex substructure - allocate_regex_client_enabled(client); - // Increase counter by one counters->clients++; + // Allocate regex substructure + allocate_regex_client_enabled(client, clientID); + return clientID; } diff --git a/src/datastructure.h b/src/datastructure.h index 09c9fcaa..98bbbdc8 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -68,7 +68,6 @@ typedef struct { sqlite3_stmt* whitelist_stmt; sqlite3_stmt* gravity_stmt; sqlite3_stmt* blacklist_stmt; - bool *regex_enabled[2]; } clientsData; typedef struct { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 8cd88a42..861f1281 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -180,7 +180,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Check whitelist (exact + regex) for match const char *domainString = getstr(domain->domainpos); - query->whitelisted = in_whitelist(domainString, client); + query->whitelisted = in_whitelist(domainString, client, clientID); // Check domains against blacklist and gravity (blacklist is checked first) // Skipped when the domain is whitelisted @@ -209,7 +209,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Check domain against regex filters // Skipped when the domain is whitelisted or blocked by blacklist or gravity if(!query->whitelisted && !blockDomain && - match_regex(domainString, client, REGEX_BLACKLIST)) + match_regex(domainString, clientID, REGEX_BLACKLIST)) { // Mark domain as regex matched for this one client dns_cache->blocking_status = REGEX_BLOCKED; diff --git a/src/memory.h b/src/memory.h index 5abef537..c604f429 100644 --- a/src/memory.h +++ b/src/memory.h @@ -39,6 +39,7 @@ typedef struct { int reply_domain; int dns_cache_size; int dns_cache_MAX; + int num_regex[2]; } countersStruct; extern countersStruct *counters; diff --git a/src/regex.c b/src/regex.c index 569e8d39..ee985794 100644 --- a/src/regex.c +++ b/src/regex.c @@ -20,8 +20,9 @@ #include "database/gravity-db.h" // bool startup #include "main.h" +// add_per_client_regex_client() +#include "shmem.h" -static int num_regex[2] = { 0 }; static regex_t *regex[2] = { NULL }; static bool *regex_available[2] = { NULL }; static int *regex_id[2] = { NULL }; @@ -63,19 +64,13 @@ static bool compile_regex(const char *regexin, const int index, const unsigned c return true; } -bool match_regex(const char *input, const clientsData *client, const unsigned char regexid) +bool match_regex(const char *input, const int clientID, const unsigned char regexid) { bool matched = false; - if(client->regex_enabled[regexid] == NULL) - { - logg("Regex list %d for client not configured!", regexid); - return false; - } - // Start matching timer timer_start(REGEX_TIMER); - for(int index = 0; index < num_regex[regexid]; index++) + for(int index = 0; index < counters->num_regex[regexid]; index++) { // Only check regex which have been successfully compiled ... if(!regex_available[regexid][index]) @@ -86,7 +81,11 @@ bool match_regex(const char *input, const clientsData *client, const unsigned ch continue; } // ... and are enabled for this client - if(!client->regex_enabled[regexid][index]) + int regexID = index; + if(regexid == REGEX_WHITELIST) + regexID += counters->num_regex[REGEX_BLACKLIST]; + + if(!get_per_client_regex(clientID, regexID)) { if(config.debug & DEBUG_REGEX) logg("Regex %s ID %d not enabled for this client", regextype[regexid], index); @@ -131,30 +130,15 @@ static void free_regex(void) return; // Reset client configuration - for(int i = 0; i < counters->clients; i++) + for(int clientID = 0; clientID < counters->clients; clientID++) { - // Get client pointer - clientsData *client = getClient(i, true); - if(client == NULL) - continue; - - if(client->regex_enabled[REGEX_WHITELIST] != NULL) - { - free(client->regex_enabled[REGEX_WHITELIST]); - client->regex_enabled[REGEX_WHITELIST] = NULL; - } - - if(client->regex_enabled[REGEX_BLACKLIST] != NULL) - { - free(client->regex_enabled[REGEX_BLACKLIST]); - client->regex_enabled[REGEX_BLACKLIST] = NULL; - } + reset_per_client_regex(clientID); } // Free regex datastructure for(int regexid = 0; regexid < 2; regexid++) { - for(int index = 0; index < num_regex[regexid]; index++) + for(int index = 0; index < counters->num_regex[regexid]; index++) { if(!regex_available[regexid][index]) continue; @@ -177,24 +161,23 @@ static void free_regex(void) } // Reset counter for number of regex - num_regex[regexid] = 0; + counters->num_regex[regexid] = 0; } } -void allocate_regex_client_enabled(clientsData *client) +void allocate_regex_client_enabled(clientsData *client, const int clientID) { - client->regex_enabled[REGEX_BLACKLIST] = calloc(num_regex[REGEX_BLACKLIST], sizeof(bool)); - client->regex_enabled[REGEX_WHITELIST] = calloc(num_regex[REGEX_WHITELIST], sizeof(bool)); + add_per_client_regex(clientID); // Only initialize regex associations when dnsmasq is ready (otherwise, we're still in history reading mode) if(!startup) { - gravityDB_get_regex_client_groups(client, num_regex[REGEX_BLACKLIST], + gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_BLACKLIST], regex_id[REGEX_BLACKLIST], REGEX_BLACKLIST, - "vw_regex_blacklist"); - gravityDB_get_regex_client_groups(client, num_regex[REGEX_WHITELIST], + "vw_regex_blacklist", clientID); + gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_WHITELIST], regex_id[REGEX_WHITELIST], REGEX_WHITELIST, - "vw_regex_whitelist"); + "vw_regex_whitelist", clientID); } } @@ -204,28 +187,28 @@ static void read_regex_table(const unsigned char regexid) unsigned char databaseID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE; // Get number of lines in the regex table - num_regex[regexid] = gravityDB_count(databaseID); + counters->num_regex[regexid] = gravityDB_count(databaseID); - if(num_regex[regexid] == 0) + if(counters->num_regex[regexid] == 0) { logg("INFO: No regex %s entries found", regextype[regexid]); return; } - else if(num_regex[regexid] == DB_FAILED) + else if(counters->num_regex[regexid] == DB_FAILED) { logg("WARN: Database query failed, assuming there are no regex %s entries", regextype[regexid]); - num_regex[regexid] = 0; + counters->num_regex[regexid] = 0; return; } // Allocate memory for regex - regex[regexid] = calloc(num_regex[regexid], sizeof(regex_t)); - regex_id[regexid] = calloc(num_regex[regexid], sizeof(int)); - regex_available[regexid] = calloc(num_regex[regexid], sizeof(bool)); + regex[regexid] = calloc(counters->num_regex[regexid], sizeof(regex_t)); + regex_id[regexid] = calloc(counters->num_regex[regexid], sizeof(int)); + regex_available[regexid] = calloc(counters->num_regex[regexid], sizeof(bool)); // Buffer strings if in regex debug mode if(config.debug & DEBUG_REGEX) - regexbuffer[regexid] = calloc(num_regex[regexid], sizeof(char*)); + regexbuffer[regexid] = calloc(counters->num_regex[regexid], sizeof(char*)); // Connect to regex table if(!gravityDB_getTable(databaseID)) @@ -241,7 +224,7 @@ static void read_regex_table(const unsigned char regexid) { // Avoid buffer overflow if database table changed // since we counted its entries - if(i >= num_regex[regexid]) + if(i >= counters->num_regex[regexid]) break; // Skip this entry if empty: an empty regex filter would match @@ -282,17 +265,18 @@ void read_regex_from_database(void) read_regex_table(REGEX_WHITELIST); - for(int i = 0; i < counters->clients; i++) + for(int clientID = 0; clientID < counters->clients; clientID++) { // Get client pointer - clientsData *client = getClient(i, true); + clientsData *client = getClient(clientID, true); if(client == NULL) continue; - allocate_regex_client_enabled(client); + allocate_regex_client_enabled(client, clientID); } // Print message to FTL's log after reloading regex filters logg("Compiled %i whitelist and %i blacklist regex filters in %.1f msec", - num_regex[REGEX_WHITELIST], num_regex[REGEX_BLACKLIST], timer_elapsed_msec(REGEX_TIMER)); + counters->num_regex[REGEX_WHITELIST], counters->num_regex[REGEX_BLACKLIST], + timer_elapsed_msec(REGEX_TIMER)); } diff --git a/src/regex_r.h b/src/regex_r.h index ae0525ba..2581b570 100644 --- a/src/regex_r.h +++ b/src/regex_r.h @@ -13,11 +13,11 @@ // clientsData type #include "datastructure.h" -bool match_regex(const char *input, const clientsData *client, const unsigned char regexid); -void allocate_regex_client_enabled(clientsData *client); + +bool match_regex(const char *input, const int clientID, const unsigned char regexid); +void allocate_regex_client_enabled(clientsData *client, const int clientID); void read_regex_from_database(void); -enum { REGEX_BLACKLIST, REGEX_WHITELIST }; // Blocking status constants used by the domain->clientstatus vector // We explicitly force UNKNOWN_BLOCKED to zero on all platforms as this is the // default value set initially with calloc diff --git a/src/shmem.c b/src/shmem.c index d85ddee6..65ae2e4c 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -31,6 +31,7 @@ #define SHARED_OVERTIME_NAME "/FTL-overTime" #define SHARED_SETTINGS_NAME "/FTL-settings" #define SHARED_DNS_CACHE "/FTL-dns-cache" +#define SHARED_PER_CLIENT_REGEX "/FTL-per-client-regex" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_lock = { 0 }; @@ -43,6 +44,7 @@ static SharedMemory shm_forwarded = { 0 }; static SharedMemory shm_overTime = { 0 }; static SharedMemory shm_settings = { 0 }; static SharedMemory shm_dns_cache = { 0 }; +static SharedMemory shm_per_client_regex = { 0 }; // Variable size array structs static queriesData *queries = NULL; @@ -100,6 +102,7 @@ void chown_all_shmem(struct passwd *ent_pw) chown_shmem(&shm_overTime, ent_pw); chown_shmem(&shm_settings, ent_pw); chown_shmem(&shm_dns_cache, ent_pw); + chown_shmem(&shm_per_client_regex, ent_pw); } size_t addstr(const char *str) @@ -328,6 +331,11 @@ bool init_shmem(void) dns_cache = (DNSCacheData*)shm_dns_cache.ptr; counters->dns_cache_MAX = size; + /****************************** shared per-client regex buffer ******************************/ + size = get_optimal_object_size(1, 2); + // Try to create shared memory object + shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size); + return true; } @@ -346,6 +354,7 @@ void destroy_shmem(void) delete_shm(&shm_overTime); delete_shm(&shm_settings); delete_shm(&shm_dns_cache); + delete_shm(&shm_per_client_regex); } SharedMemory create_shm(const char *name, const size_t size) @@ -668,6 +677,60 @@ void memory_check(int which) } } +void reset_per_client_regex(const int clientID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + for(unsigned int i = 0u; i < num_regex_tot; i++) + { + // Zero-initialize/reset (= false) all regex (white + black) + set_per_client_regex(clientID, i, false); + } +} + +void add_per_client_regex(unsigned int clientID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const size_t size = counters->clients * num_regex_tot; + if(size > shm_per_client_regex.size && + realloc_shm(&shm_per_client_regex, size, true)) + { + reset_per_client_regex(clientID); + } +} + +bool get_per_client_regex(const int clientID, const int regexID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const unsigned int id = clientID * num_regex_tot + regexID; + const unsigned int maxval = counters->clients * num_regex_tot; + if(id > maxval) + { + logg("ERROR: get_per_client_regex(%d,%d): Out of bounds (%d > %d * %d == %d)!", + clientID, regexID, id, counters->clients-1, num_regex_tot, maxval); + return false; + } + return ((bool*) shm_per_client_regex.ptr)[id]; +} + +void set_per_client_regex(const int clientID, const int regexID, const bool value) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const unsigned int id = clientID * num_regex_tot + regexID; + const unsigned int maxval = counters->clients * num_regex_tot; + if(id > maxval) + { + logg("ERROR: set_per_client_regex(%d,%d,%s): Out of bounds (%d > %d * %d == %d)!", + clientID, regexID, value ? "true" : "false", + id, counters->clients-1, num_regex_tot, maxval); + return; + } + ((bool*) shm_per_client_regex.ptr)[id] = value; +} + static inline bool check_range(int ID, int MAXID, const char* type, int line, const char * function, const char * file) { if(ID < 0 || ID > MAXID) diff --git a/src/shmem.h b/src/shmem.h index 95efb60f..13f77bc9 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -77,4 +77,10 @@ void addOverTimeClientSlot(void); // Change ownership of shared memory objects void chown_all_shmem(struct passwd *ent_pw); +// Per-client regex buffer storing whether or not a specific regex is enabled for a particular client +void add_per_client_regex(unsigned int clientID); +void reset_per_client_regex(const int clientID); +bool get_per_client_regex(const int clientID, const int regexID); +void set_per_client_regex(const int clientID, const int regexID, const bool value); + #endif //SHARED_MEMORY_SERVER_H