Add shared per-client regex array. This ensures TCP and UDP workers know the same details about client/regex combinations. This commit also fixes an issue with regex group associations for configured clients that have no assigned group.

Signed-off-by: DL6ER <dl6er@dl6er.de>

src/FTL.h

@@ -112,6 +112,7 @@ enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT,
 enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER };
 enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };
 enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
+enum { REGEX_BLACKLIST, REGEX_WHITELIST };
 
 // Use out own memory handling functions that will detect possible errors
 // and report accordingly in the log. This will make debugging FTL crashs

src/database/gravity-db.c

@@ -109,6 +109,7 @@ static bool get_client_groupids(const clientsData* client, char **groups)
 	// Get associated groups for this client (if defined)
 	char *querystr = NULL;
 	const char *ip = getstr(client->ippos);
+	*groups = NULL;
 
 	// Do not proceed when database is not available
 	if(!gravity_database_avail)
@@ -117,21 +118,75 @@ static bool get_client_groupids(const clientsData* client, char **groups)
 		return false;
 	}
 
+	if(config.debug & DEBUG_DATABASE)
+		logg("Querying gravity database for client %s", ip);
+
+	// Check if client is configured through the client table
+	if(asprintf(&querystr, "SELECT COUNT(*) FROM client WHERE ip = \'%s\';", ip) < 1)
+	{
+		logg("get_client_groupids() - asprintf() error 1");
+		return false;
+	}
+
+	// Prepare query
+	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
+	if(rc != SQLITE_OK){
+		logg("get_client_groupids(%s) - SQL error prepare (%i): %s",
+		     querystr, rc, sqlite3_errmsg(gravity_db));
+		sqlite3_finalize(table_stmt);
+		gravityDB_close();
+		free(querystr);
+		return false;
+	}
+
+	// Perform query
+	rc = sqlite3_step(table_stmt);
+	if(rc == SQLITE_ROW)
+	{
+		// There is a record for this client in the database
+		const int result = sqlite3_column_int(table_stmt, 0);
+
+		// Found no record for this client in the database
+		// This makes this client qualify for the special "all" group
+		if(result == 0)
+			*groups = strdup("0");
+	}
+	else if(rc == SQLITE_DONE)
+	{
+		// Found no record for this client in the database
+		// This makes this client qualify for the special "all" group
+		*groups = strdup("0");
+	}
+	else
+	{
+		logg("get_client_groupids(%s) - SQL error step (%i): %s",
+		     querystr, rc, sqlite3_errmsg(gravity_db));
+		sqlite3_finalize(table_stmt);
+		gravityDB_close();
+		free(querystr);
+		return false;
+	}
+	// Finalize statement
+	gravityDB_finalizeTable();
+
+	if(*groups != NULL)
+	{
+		// The client is not configured through the client table, return early
+		return true;
+	}
+
 	// Build query string to get possible group associations for this particular client
 	// The SQL GROUP_CONCAT() function returns a string which is the concatenation of all
 	// non-NULL values of group_id separated by ','. The order of the concatenated elements
 	// is arbitrary, however, is of no relevance for your use case.
 	if(asprintf(&querystr, "SELECT GROUP_CONCAT(group_id) FROM client_by_group WHERE client_id = (SELECT id FROM client WHERE ip = \'%s\');", ip) < 1)
 	{
-		logg("get_client_groupids() - asprintf() error");
+		logg("get_client_groupids() - asprintf() error 2");
 		return false;
 	}
 
-	if(config.debug & DEBUG_DATABASE)
-		logg("Querying gravity database for client %s", ip);
-
 	// Prepare query
-	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
+	rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
 	if(rc != SQLITE_OK){
 		logg("get_client_groupids(%s) - SQL error prepare (%i): %s",
 		     querystr, rc, sqlite3_errmsg(gravity_db));
@@ -150,13 +205,13 @@ static bool get_client_groupids(const clientsData* client, char **groups)
 		if(result != NULL)
 			*groups = strdup(result);
 		else
-			*groups = strdup("0");
+			*groups = strdup("");
 	}
 	else if(rc == SQLITE_DONE)
 	{
 		// Found no record for this client in the database
-		// This makes this client qualify for the special "all" group
-		*groups = strdup("0");
+		// -> No associated groups
+		*groups = strdup("");
 	}
 	else
 	{
@@ -482,7 +537,7 @@ static bool domain_in_list(const char *domain, sqlite3_stmt* stmt, const char* l
 	return (result == 1);
 }
 
-inline bool in_whitelist(const char *domain, clientsData* client)
+inline bool in_whitelist(const char *domain, clientsData* client, const int clientID)
 {
 	if(client->whitelist_stmt == NULL)
 		gravityDB_prepare_client_statements(client);
@@ -493,7 +548,7 @@ inline bool in_whitelist(const char *domain, clientsData* client)
 	// optimization as the database lookup will most likely hit (a) more domains and (b)
 	// will be faster (given a sufficiently large number of regex whitelisting filters).
 	return domain_in_list(domain, client->whitelist_stmt, "whitelist") ||
-	       match_regex(domain, client, REGEX_WHITELIST);
+	       match_regex(domain, clientID, REGEX_WHITELIST);
 }
 
 inline bool in_gravity(const char *domain, clientsData* client)
@@ -517,7 +572,7 @@ bool in_auditlist(const char *domain)
 }
 
 bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid,
-                                       const unsigned char type, const char* table)
+                                       const unsigned char type, const char* table, const int clientID)
 {
 	char *querystr = NULL;
 	char *groups = NULL;
@@ -553,7 +608,10 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex,
 		{
 			if(regexid[i] == result)
 			{
-				client->regex_enabled[type][i] = true;
+				unsigned int regexID = i;
+				if(type == REGEX_WHITELIST)
+					regexID += counters->num_regex[REGEX_BLACKLIST];
+				set_per_client_regex(clientID, regexID, true);
 				break;
 			}
 		}

src/database/gravity-db.h

@@ -25,11 +25,11 @@ int gravityDB_count(unsigned char list);
 bool in_auditlist(const char *domain);
 
 bool in_gravity(const char *domain, clientsData* client);
-bool in_whitelist(const char *domain, clientsData* client);
+bool in_whitelist(const char *domain, clientsData* client, const int clientID);
 bool in_blacklist(const char *domain, clientsData* client);
 
 bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid,
-                                       const unsigned char type, const char* table);
+                                       const unsigned char type, const char* table, const int clientID);
 
 
 #endif //GRAVITY_H

src/datastructure.c

@@ -200,12 +200,12 @@ int findClientID(const char *clientIP, const bool count)
 	for(int i = 0; i < OVERTIME_SLOTS; i++)
 		client->overTime[i] = 0;
 
-	// Allocate regex substructure
-	allocate_regex_client_enabled(client);
-
 	// Increase counter by one
 	counters->clients++;
 
+	// Allocate regex substructure
+	allocate_regex_client_enabled(client, clientID);
+
 	return clientID;
 }
 

src/datastructure.h

@@ -68,7 +68,6 @@ typedef struct {
 	sqlite3_stmt* whitelist_stmt;
 	sqlite3_stmt* gravity_stmt;
 	sqlite3_stmt* blacklist_stmt;
-	bool *regex_enabled[2];
 } clientsData;
 
 typedef struct {

src/dnsmasq_interface.c

@@ -180,7 +180,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 
 	// Check whitelist (exact + regex) for match
 	const char *domainString = getstr(domain->domainpos);
-	query->whitelisted = in_whitelist(domainString, client);
+	query->whitelisted = in_whitelist(domainString, client, clientID);
 
 	// Check domains against blacklist and gravity (blacklist is checked first)
 	// Skipped when the domain is whitelisted
@@ -209,7 +209,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 	// Check domain against regex filters
 	// Skipped when the domain is whitelisted or blocked by blacklist or gravity
 	if(!query->whitelisted && !blockDomain &&
-	   match_regex(domainString, client, REGEX_BLACKLIST))
+	   match_regex(domainString, clientID, REGEX_BLACKLIST))
 	{
 		// Mark domain as regex matched for this one client
 		dns_cache->blocking_status = REGEX_BLOCKED;

src/memory.h

@@ -39,6 +39,7 @@ typedef struct {
 	int reply_domain;
 	int dns_cache_size;
 	int dns_cache_MAX;
+	int num_regex[2];
 } countersStruct;
 
 extern countersStruct *counters;

src/regex.c

@@ -20,8 +20,9 @@
 #include "database/gravity-db.h"
 // bool startup
 #include "main.h"
+// add_per_client_regex_client()
+#include "shmem.h"
 
-static int num_regex[2] = { 0 };
 static regex_t *regex[2] = { NULL };
 static bool *regex_available[2] = { NULL };
 static int *regex_id[2] = { NULL };
@@ -63,19 +64,13 @@ static bool compile_regex(const char *regexin, const int index, const unsigned c
 	return true;
 }
 
-bool match_regex(const char *input, const clientsData *client, const unsigned char regexid)
+bool match_regex(const char *input, const int clientID, const unsigned char regexid)
 {
 	bool matched = false;
 
-	if(client->regex_enabled[regexid] == NULL)
-	{
-		logg("Regex list %d for client not configured!", regexid);
-		return false;
-	}
-
 	// Start matching timer
 	timer_start(REGEX_TIMER);
-	for(int index = 0; index < num_regex[regexid]; index++)
+	for(int index = 0; index < counters->num_regex[regexid]; index++)
 	{
 		// Only check regex which have been successfully compiled ...
 		if(!regex_available[regexid][index])
@@ -86,7 +81,11 @@ bool match_regex(const char *input, const clientsData *client, const unsigned ch
 			continue;
 		}
 		// ... and are enabled for this client
-		if(!client->regex_enabled[regexid][index])
+		int regexID = index;
+		if(regexid == REGEX_WHITELIST)
+			regexID += counters->num_regex[REGEX_BLACKLIST];
+
+		if(!get_per_client_regex(clientID, regexID))
 		{
 			if(config.debug & DEBUG_REGEX)
 				logg("Regex %s ID %d not enabled for this client", regextype[regexid], index);
@@ -131,30 +130,15 @@ static void free_regex(void)
 		return;
 
 	// Reset client configuration
-	for(int i = 0; i < counters->clients; i++)
+	for(int clientID = 0; clientID < counters->clients; clientID++)
 	{
-		// Get client pointer
-		clientsData *client = getClient(i, true);
-		if(client == NULL)
-			continue;
-
-		if(client->regex_enabled[REGEX_WHITELIST] != NULL)
-		{
-			free(client->regex_enabled[REGEX_WHITELIST]);
-			client->regex_enabled[REGEX_WHITELIST] = NULL;
-		}
-
-		if(client->regex_enabled[REGEX_BLACKLIST] != NULL)
-		{
-			free(client->regex_enabled[REGEX_BLACKLIST]);
-			client->regex_enabled[REGEX_BLACKLIST] = NULL;
-		}
+		reset_per_client_regex(clientID);
 	}
 
 	// Free regex datastructure
 	for(int regexid = 0; regexid < 2; regexid++)
 	{
-		for(int index = 0; index < num_regex[regexid]; index++)
+		for(int index = 0; index < counters->num_regex[regexid]; index++)
 		{
 			if(!regex_available[regexid][index])
 				continue;
@@ -177,24 +161,23 @@ static void free_regex(void)
 		}
 
 		// Reset counter for number of regex
-		num_regex[regexid] = 0;
+		counters->num_regex[regexid] = 0;
 	}
 }
 
-void allocate_regex_client_enabled(clientsData *client)
+void allocate_regex_client_enabled(clientsData *client, const int clientID)
 {
-	client->regex_enabled[REGEX_BLACKLIST] = calloc(num_regex[REGEX_BLACKLIST], sizeof(bool));
-	client->regex_enabled[REGEX_WHITELIST] = calloc(num_regex[REGEX_WHITELIST], sizeof(bool));
+	add_per_client_regex(clientID);
 
 	// Only initialize regex associations when dnsmasq is ready (otherwise, we're still in history reading mode)
 	if(!startup)
 	{
-		gravityDB_get_regex_client_groups(client, num_regex[REGEX_BLACKLIST],
+		gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_BLACKLIST],
 						regex_id[REGEX_BLACKLIST], REGEX_BLACKLIST,
-						"vw_regex_blacklist");
-		gravityDB_get_regex_client_groups(client, num_regex[REGEX_WHITELIST],
+						"vw_regex_blacklist", clientID);
+		gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_WHITELIST],
 						regex_id[REGEX_WHITELIST], REGEX_WHITELIST,
-						"vw_regex_whitelist");
+						"vw_regex_whitelist", clientID);
 	}
 }
 
@@ -204,28 +187,28 @@ static void read_regex_table(const unsigned char regexid)
 	unsigned char databaseID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE;
 
 	// Get number of lines in the regex table
-	num_regex[regexid] = gravityDB_count(databaseID);
+	counters->num_regex[regexid] = gravityDB_count(databaseID);
 
-	if(num_regex[regexid] == 0)
+	if(counters->num_regex[regexid] == 0)
 	{
 		logg("INFO: No regex %s entries found", regextype[regexid]);
 		return;
 	}
-	else if(num_regex[regexid] == DB_FAILED)
+	else if(counters->num_regex[regexid] == DB_FAILED)
 	{
 		logg("WARN: Database query failed, assuming there are no regex %s entries", regextype[regexid]);
-		num_regex[regexid] = 0;
+		counters->num_regex[regexid] = 0;
 		return;
 	}
 
 	// Allocate memory for regex
-	regex[regexid] = calloc(num_regex[regexid], sizeof(regex_t));
-	regex_id[regexid] = calloc(num_regex[regexid], sizeof(int));
-	regex_available[regexid] = calloc(num_regex[regexid], sizeof(bool));
+	regex[regexid] = calloc(counters->num_regex[regexid], sizeof(regex_t));
+	regex_id[regexid] = calloc(counters->num_regex[regexid], sizeof(int));
+	regex_available[regexid] = calloc(counters->num_regex[regexid], sizeof(bool));
 
 	// Buffer strings if in regex debug mode
 	if(config.debug & DEBUG_REGEX)
-		regexbuffer[regexid] = calloc(num_regex[regexid], sizeof(char*));
+		regexbuffer[regexid] = calloc(counters->num_regex[regexid], sizeof(char*));
 
 	// Connect to regex table
 	if(!gravityDB_getTable(databaseID))
@@ -241,7 +224,7 @@ static void read_regex_table(const unsigned char regexid)
 	{
 		// Avoid buffer overflow if database table changed
 		// since we counted its entries
-		if(i >= num_regex[regexid])
+		if(i >= counters->num_regex[regexid])
 			break;
 
 		// Skip this entry if empty: an empty regex filter would match
@@ -282,17 +265,18 @@ void read_regex_from_database(void)
 	read_regex_table(REGEX_WHITELIST);
 
 
-	for(int i = 0; i < counters->clients; i++)
+	for(int clientID = 0; clientID < counters->clients; clientID++)
 	{
 		// Get client pointer
-		clientsData *client = getClient(i, true);
+		clientsData *client = getClient(clientID, true);
 		if(client == NULL)
 			continue;
 
-		allocate_regex_client_enabled(client);
+		allocate_regex_client_enabled(client, clientID);
 	}
 
 	// Print message to FTL's log after reloading regex filters
 	logg("Compiled %i whitelist and %i blacklist regex filters in %.1f msec",
-	     num_regex[REGEX_WHITELIST], num_regex[REGEX_BLACKLIST], timer_elapsed_msec(REGEX_TIMER));
+	     counters->num_regex[REGEX_WHITELIST], counters->num_regex[REGEX_BLACKLIST],
+	     timer_elapsed_msec(REGEX_TIMER));
 }

src/regex_r.h

@@ -13,11 +13,11 @@
 // clientsData type
 #include "datastructure.h"
 
-bool match_regex(const char *input, const clientsData *client, const unsigned char regexid);
-void allocate_regex_client_enabled(clientsData *client);
+
+bool match_regex(const char *input, const int clientID, const unsigned char regexid);
+void allocate_regex_client_enabled(clientsData *client, const int clientID);
 void read_regex_from_database(void);
 
-enum { REGEX_BLACKLIST, REGEX_WHITELIST };
 // Blocking status constants used by the domain->clientstatus vector
 // We explicitly force UNKNOWN_BLOCKED to zero on all platforms as this is the
 // default value set initially with calloc

src/shmem.c

@@ -31,6 +31,7 @@
 #define SHARED_OVERTIME_NAME "/FTL-overTime"
 #define SHARED_SETTINGS_NAME "/FTL-settings"
 #define SHARED_DNS_CACHE "/FTL-dns-cache"
+#define SHARED_PER_CLIENT_REGEX "/FTL-per-client-regex"
 
 /// The pointer in shared memory to the shared string buffer
 static SharedMemory shm_lock = { 0 };
@@ -43,6 +44,7 @@ static SharedMemory shm_forwarded = { 0 };
 static SharedMemory shm_overTime = { 0 };
 static SharedMemory shm_settings = { 0 };
 static SharedMemory shm_dns_cache = { 0 };
+static SharedMemory shm_per_client_regex = { 0 };
 
 // Variable size array structs
 static queriesData *queries = NULL;
@@ -100,6 +102,7 @@ void chown_all_shmem(struct passwd *ent_pw)
 	chown_shmem(&shm_overTime, ent_pw);
 	chown_shmem(&shm_settings, ent_pw);
 	chown_shmem(&shm_dns_cache, ent_pw);
+	chown_shmem(&shm_per_client_regex, ent_pw);
 }
 
 size_t addstr(const char *str)
@@ -328,6 +331,11 @@ bool init_shmem(void)
 	dns_cache = (DNSCacheData*)shm_dns_cache.ptr;
 	counters->dns_cache_MAX = size;
 
+	/****************************** shared per-client regex buffer ******************************/
+	size = get_optimal_object_size(1, 2);
+	// Try to create shared memory object
+	shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size);
+
 	return true;
 }
 
@@ -346,6 +354,7 @@ void destroy_shmem(void)
 	delete_shm(&shm_overTime);
 	delete_shm(&shm_settings);
 	delete_shm(&shm_dns_cache);
+	delete_shm(&shm_per_client_regex);
 }
 
 SharedMemory create_shm(const char *name, const size_t size)
@@ -668,6 +677,60 @@ void memory_check(int which)
 	}
 }
 
+void reset_per_client_regex(const int clientID)
+{
+	const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] +
+	                                   counters->num_regex[REGEX_WHITELIST];
+	for(unsigned int i = 0u; i < num_regex_tot; i++)
+	{
+		// Zero-initialize/reset (= false) all regex (white + black)
+		set_per_client_regex(clientID, i, false);
+	}
+}
+
+void add_per_client_regex(unsigned int clientID)
+{
+	const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] +
+	                                   counters->num_regex[REGEX_WHITELIST];
+	const size_t size = counters->clients * num_regex_tot;
+	if(size > shm_per_client_regex.size &&
+	   realloc_shm(&shm_per_client_regex, size, true))
+	{
+		reset_per_client_regex(clientID);
+	}
+}
+
+bool get_per_client_regex(const int clientID, const int regexID)
+{
+	const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] +
+	                                   counters->num_regex[REGEX_WHITELIST];
+	const unsigned int id = clientID * num_regex_tot + regexID;
+	const unsigned int maxval = counters->clients * num_regex_tot;
+	if(id > maxval)
+	{
+		logg("ERROR: get_per_client_regex(%d,%d): Out of bounds (%d > %d * %d == %d)!",
+		     clientID, regexID, id, counters->clients-1, num_regex_tot, maxval);
+		return false;
+	}
+	return ((bool*) shm_per_client_regex.ptr)[id];
+}
+
+void set_per_client_regex(const int clientID, const int regexID, const bool value)
+{
+	const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] +
+	                                   counters->num_regex[REGEX_WHITELIST];
+	const unsigned int id = clientID * num_regex_tot + regexID;
+	const unsigned int maxval = counters->clients * num_regex_tot;
+	if(id > maxval)
+	{
+		logg("ERROR: set_per_client_regex(%d,%d,%s): Out of bounds (%d > %d * %d == %d)!",
+		     clientID, regexID, value ? "true" : "false",
+		     id, counters->clients-1, num_regex_tot, maxval);
+		return;
+	}
+	((bool*) shm_per_client_regex.ptr)[id] = value;
+}
+
 static inline bool check_range(int ID, int MAXID, const char* type, int line, const char * function, const char * file)
 {
 	if(ID < 0 || ID > MAXID)

src/shmem.h

@@ -77,4 +77,10 @@ void addOverTimeClientSlot(void);
 // Change ownership of shared memory objects
 void chown_all_shmem(struct passwd *ent_pw);
 
+// Per-client regex buffer storing whether or not a specific regex is enabled for a particular client
+void add_per_client_regex(unsigned int clientID);
+void reset_per_client_regex(const int clientID);
+bool get_per_client_regex(const int clientID, const int regexID);
+void set_per_client_regex(const int clientID, const int regexID, const bool value);
+
 #endif //SHARED_MEMORY_SERVER_H