Merge branch 'development' into new/mac_clients

Signed-off-by: DL6ER <dl6er@dl6er.de>

build.sh

@@ -9,9 +9,27 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-rm -rf cmake/ && \
-mkdir cmake && \
-cd cmake && \
-cmake .. && \
-cmake --build . -- -j $(nproc) && \
-cp pihole-FTL ../
+# Abort script if one command returns a non-zero value
+set -e
+
+# Prepare build environment
+if [[ "${1}" == "clean" ]]; then
+    rm -rf cmake/
+    exit 0
+fi
+
+# Configure build
+mkdir -p cmake
+cd cmake
+cmake ..
+
+# Build the sources
+cmake --build . -- -j $(nproc)
+
+# If we are asked to install, we do this here
+# Otherwise, we simply copy the binary one level up
+if [[ "${1}" == "install" ]]; then
+    sudo make install
+else
+    cp pihole-FTL ../
+fi

src/capabilities.c

@@ -100,6 +100,14 @@ bool check_capabilities(void)
 		logg("*************************************************************************");
 		capabilities_okay = false;
 	}
+	if (!(data->permitted & (1 << CAP_SYS_NICE)))
+	{
+		// Necessary for dynamic port binding
+		logg("*************************************************************************");
+		logg("* WARNING: Required Linux capability CAP_SYS_NICE not available         *");
+		logg("*************************************************************************");
+		capabilities_okay = false;
+	}
 
 	// Free allocated memory
 	free(hdr);

src/config.c

@@ -13,6 +13,8 @@
 #include "memory.h"
 #include "setupVars.h"
 #include "log.h"
+// nice()
+#include <unistd.h>
 
 ConfigStruct config;
 FTLFileNamesStruct FTLfiles = {
@@ -39,6 +41,8 @@ static size_t size = 0;
 static char *parse_FTLconf(FILE *fp, const char * key);
 static void release_config_memory(void);
 static void getpath(FILE* fp, const char *option, const char *defaultloc, char **pointer);
+static void set_nice(const char *buffer, int fallback);
+static bool read_bool(const char *option, const bool fallback);
 
 void getLogFilePath(void)
 {
@@ -113,11 +117,8 @@ void read_FTLconf(void)
 
 	// AAAA_QUERY_ANALYSIS
 	// defaults to: Yes
-	config.analyze_AAAA = true;
 	buffer = parse_FTLconf(fp, "AAAA_QUERY_ANALYSIS");
-
-	if(buffer != NULL && strcasecmp(buffer, "no") == 0)
-		config.analyze_AAAA = false;
+	config.analyze_AAAA = read_bool(buffer, true);
 
 	if(config.analyze_AAAA)
 		logg("   AAAA_QUERY_ANALYSIS: Show AAAA queries");
@@ -141,11 +142,8 @@ void read_FTLconf(void)
 
 	// RESOLVE_IPV6
 	// defaults to: Yes
-	config.resolveIPv6 = true;
 	buffer = parse_FTLconf(fp, "RESOLVE_IPV6");
-
-	if(buffer != NULL && strcasecmp(buffer, "no") == 0)
-		config.resolveIPv6 = false;
+	config.resolveIPv6 = read_bool(buffer, true);
 
 	if(config.resolveIPv6)
 		logg("   RESOLVE_IPV6: Resolve IPv6 addresses");
@@ -154,10 +152,9 @@ void read_FTLconf(void)
 
 	// RESOLVE_IPV4
 	// defaults to: Yes
-	config.resolveIPv4 = true;
 	buffer = parse_FTLconf(fp, "RESOLVE_IPV4");
-	if(buffer != NULL && strcasecmp(buffer, "no") == 0)
-		config.resolveIPv4 = false;
+	config.resolveIPv4 = read_bool(buffer, true);
+
 	if(config.resolveIPv4)
 		logg("   RESOLVE_IPV4: Resolve IPv4 addresses");
 	else
@@ -245,8 +242,8 @@ void read_FTLconf(void)
 
 	// IGNORE_LOCALHOST
 	// defaults to: false
-	config.ignore_localhost = false;
 	buffer = parse_FTLconf(fp, "IGNORE_LOCALHOST");
+	config.ignore_localhost = read_bool(buffer, false);
 
 	if(buffer != NULL && strcasecmp(buffer, "yes") == 0)
 		config.ignore_localhost = true;
@@ -280,8 +277,8 @@ void read_FTLconf(void)
 
 	// ANALYZE_ONLY_A_AND_AAAA
 	// defaults to: false
-	config.analyze_only_A_AAAA = false;
 	buffer = parse_FTLconf(fp, "ANALYZE_ONLY_A_AND_AAAA");
+	config.analyze_only_A_AAAA = read_bool(buffer, false);
 
 	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
 		config.analyze_only_A_AAAA = true;
@@ -293,10 +290,9 @@ void read_FTLconf(void)
 
 	// DBIMPORT
 	// defaults to: Yes
-	config.DBimport = true;
 	buffer = parse_FTLconf(fp, "DBIMPORT");
-	if(buffer != NULL && strcasecmp(buffer, "no") == 0)
-		config.DBimport = false;
+	config.DBimport = read_bool(buffer, true);
+
 	if(config.DBimport)
 		logg("   DBIMPORT: Importing history from database");
 	else
@@ -322,11 +318,8 @@ void read_FTLconf(void)
 
 	// PARSE_ARP_CACHE
 	// defaults to: true
-	config.parse_arp_cache = true;
 	buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE");
-
-	if(buffer != NULL && strcasecmp(buffer, "false") == 0)
-		config.parse_arp_cache = false;
+	config.parse_arp_cache = read_bool(buffer, true);
 
 	if(config.parse_arp_cache)
 		logg("   PARSE_ARP_CACHE: Active");
@@ -335,11 +328,8 @@ void read_FTLconf(void)
 
 	// CNAME_DEEP_INSPECT
 	// defaults to: true
-	config.cname_inspection = true;
 	buffer = parse_FTLconf(fp, "CNAME_DEEP_INSPECT");
-
-	if(buffer != NULL && strcasecmp(buffer, "false") == 0)
-		config.cname_inspection = false;
+	config.cname_inspection = read_bool(buffer, true);
 
 	if(config.cname_inspection)
 		logg("   CNAME_DEEP_INSPECT: Active");
@@ -353,25 +343,34 @@ void read_FTLconf(void)
 	config.delay_startup = 0;
 	if(buffer != NULL && sscanf(buffer, "%u", &config.delay_startup) &&
 	   (config.delay_startup > 0 && config.delay_startup <= 300))
-	{
 		logg("   DELAY_STARTUP: Requested to wait %u seconds during startup.", config.delay_startup);
-	}
 	else
 		logg("   DELAY_STARTUP: No delay requested.");
 
 	// BLOCK_ESNI
 	// defaults to: true
-	config.block_esni = true;
 	buffer = parse_FTLconf(fp, "BLOCK_ESNI");
-
-	if(buffer != NULL && strcasecmp(buffer, "false") == 0)
-		config.block_esni = false;
+	config.block_esni = read_bool(buffer, true);
 
 	if(config.block_esni)
 		logg("   BLOCK_ESNI: Enabled, blocking _esni.{blocked domain}");
 	else
 		logg("   BLOCK_ESNI: Disabled");
 
+	// NICE
+	// Shall we change the nice of the current process?
+	// defaults to: -10 (can be disabled by setting value to -999)
+	//
+	// The nice value is an attribute that can be used to influence the CPU
+	// scheduler to favor or disfavor a process in scheduling decisions.
+	//
+	// The range of the nice value varies across UNIX systems. On modern Linux,
+	// the range is -20 (high priority) to +19 (low priority). On some other
+	// systems, the range is -20..20. Very early Linux kernels (Before Linux
+	// 2.0) had the range -infinity..15.
+	buffer = parse_FTLconf(fp, "NICE");
+	set_nice(buffer, -10);
+
 	// MAXNETAGE
 	// IP addresses (and associated host names) older than the specified number
 	// of hours are removed to avoid dead entries in the network overview table
@@ -384,12 +383,30 @@ void read_FTLconf(void)
 	    sscanf(buffer, "%f", &fvalue) &&
 	   fvalue >= 0.0f && fvalue <= 8760.0f) // 8760 = 24 * 365
 			config.network_expire = (unsigned int)(fvalue * 3600);
+
 	if(config.network_expire > 0u)
 		logg("   MAXNETAGE: Removing IP addresses and host names from network table after %.1f hours",
 		     (float)config.network_expire/3600.0f);
 	else
 		logg("   MAXNETAGE: No automated removal of IP addresses and host names from the network table");
 
+	// NAMES_FROM_NETDB
+	// Should we use the fallback option to try to obtain client names from
+	// checking the network table? Assume this is an IPv6 client without a
+	// host names itself but the network table tells us that this is the same
+	// device where we have a host names for its IPv4 address. In this case,
+	// we use the host name associated to the other address as this is the same
+	// device. This behavior can be disabled using NAMES_FROM_NETDB=false
+	// defaults to: true
+	config.names_from_netdb = true;
+	buffer = parse_FTLconf(fp, "NAMES_FROM_NETDB");
+	config.names_from_netdb = read_bool(buffer, true);
+
+	if(config.names_from_netdb)
+		logg("   NAMES_FROM_NETDB: Enabled, trying to get names from network database");
+	else
+		logg("   NAMES_FROM_NETDB: Disabled");
+
 	// Read DEBUG_... setting from pihole-FTL.conf
 	read_debuging_settings(fp);
 
@@ -580,7 +597,7 @@ static void setDebugOption(FILE* fp, const char* option, enum debug_flags bitmas
 		return;
 
 	// Set bit if value equals "true", clear bit otherwise
-	if(strcasecmp(buffer, "true") == 0)
+	if(read_bool(buffer, false))
 		config.debug |= bitmask;
 	else
 		config.debug &= ~bitmask;
@@ -712,3 +729,62 @@ void read_debuging_settings(FILE *fp)
 		release_config_memory();
 	}
 }
+
+
+static void set_nice(const char *buffer, const int fallback)
+{
+	int value, nice_set, nice_target = fallback;
+
+	// Try to read niceness value
+	// Attempts to set a nice value outside the range are clamped to the range.
+	if(buffer != NULL && sscanf(buffer, "%i", &value) == 1)
+		nice_target = value;
+
+	// Skip setting niceness if set to -999
+	if(nice_target == -999)
+	{
+		logg("   NICE: Not changing nice value");
+		return;
+	}
+
+	// Adjust if != -999
+	errno = 0;
+	if((nice_set = nice(nice_target)) == -1 &&
+	   errno == EPERM)
+	{
+		// ERROR EPERM: The calling process attempted to increase its priority
+		// by supplying a negative value but has insufficient privileges.
+		// On Linux, the RLIMIT_NICE resource limit can be used to define a limit to
+		// which an unprivileged process's nice value can be raised. We are not
+		// affected by this limit when pihole-FTL is running with CAP_SYS_NICE
+		logg("   NICE: Cannot change niceness to %d (permission denied)",
+		     nice_target);
+		return;
+	}
+	if(nice_set == nice_target)
+	{
+		logg("   NICE: Set process niceness to %d%s",
+		     nice_set, (nice_set == fallback) ? " (default)" : "");
+	}
+	else
+	{
+		logg("   NICE: Set process niceness to %d (asked for %d)",
+		     nice_set, nice_target);
+	}
+}
+
+static bool read_bool(const char *option, const bool fallback)
+{
+	if(option == NULL)
+		return fallback;
+
+	else if(strcasecmp(option, "false") == 0 ||
+	        strcasecmp(option, "no") == 0)
+		return false;
+
+	else if(strcasecmp(option, "true") == 0 ||
+	        strcasecmp(option, "yes") == 0)
+		return true;
+
+	return fallback;
+}

src/config.h

@@ -62,6 +62,7 @@ typedef struct {
 	bool parse_arp_cache;
 	bool cname_inspection;
 	bool block_esni;
+	bool names_from_netdb;
 } ConfigStruct;
 
 typedef struct {

src/database/common.c

@@ -24,9 +24,15 @@ sqlite3 *FTL_db = NULL;
 bool database = true;
 bool DBdeleteoldqueries = false;
 long int lastdbindex = 0;
+static bool db_avail = false;
 
 static pthread_mutex_t dblock;
 
+__attribute__ ((pure)) bool FTL_DB_avail(void)
+{
+	return db_avail;
+}
+
 void dbclose(void)
 {
 	// Only try to close an existing database connection
@@ -37,6 +43,8 @@ void dbclose(void)
 		FTL_db = NULL;
 	}
 
+	db_avail = false;
+
 	// Report any error
 	if( rc != SQLITE_OK )
 	{
@@ -86,6 +94,8 @@ bool dbopen(void)
 		return false;
 	}
 
+	db_avail = true;
+
 	return true;
 }
 

src/database/common.h

@@ -19,6 +19,7 @@ bool db_set_FTL_property(const unsigned int ID, const int value);
 /// Execute a formatted SQL query and get the return code
 int dbquery(const char *format, ...);
 
+bool FTL_DB_avail(void) __attribute__ ((pure));
 bool dbopen(void);
 void dbclose(void);
 int db_query_int(const char*);

src/database/gravity-db.c

@@ -226,7 +226,8 @@ static inline const char *show_client_string(const char *hwaddr, const char *hos
 static bool get_client_groupids(clientsData* client)
 {
 	const char *ip = getstr(client->ippos);
-	client->groups = NULL;
+	client->found_group = false;
+	client->groupspos = 0u;
 
 	// Do not proceed when database is not available
 	if(!gravityDB_opened && !gravityDB_open())
@@ -290,6 +291,11 @@ static bool get_client_groupids(clientsData* client)
 	{
 		if(config.debug & DEBUG_CLIENTS)
 			logg("--> No record for %s in the client table", ip);
+
+		// Found no record for this client in the database
+		// This makes this client qualify for the special "all" group
+		client->groupspos = addstr("0");
+		client->found_group = true;
 	}
 	else
 	{
@@ -300,9 +306,16 @@ static bool get_client_groupids(clientsData* client)
 		return false;
 	}
 
-	// Finalize statement nad free allocated memory
+	// Finalize statement
 	gravityDB_finalizeTable();
 
+	if(client->found_group)
+	{
+		// The client is not configured through the client table, we
+		// substituted the default group. Return early here.
+		return true;
+	}
+
 	if(matching_count > 1)
 	{
 		// There is more than one configured subnet that matches to current device
@@ -590,7 +603,8 @@ static bool get_client_groupids(clientsData* client)
 			logg("Gravity database: Client %s not found. Using default group.\n",
 			     show_client_string(hwaddr, hostname, ip));
 
-		client->groups = strdup("0");
+		client->groupspos = addstr("0");
+		client->found_group = true;
 
 		if(hwaddr != NULL)
 		{
@@ -651,15 +665,17 @@ static bool get_client_groupids(clientsData* client)
 		// There is a record for this client in the database
 		const char* result = (const char*)sqlite3_column_text(table_stmt, 0);
 		if(result != NULL)
-			client->groups = strdup(result);
-		else
-			client->groups = strdup("");
+		{
+			client->groupspos = addstr(result);
+			client->found_group = true;
+		}
 	}
 	else if(rc == SQLITE_DONE)
 	{
 		// Found no record for this client in the database
 		// -> No associated groups
-		client->groups = strdup("");
+		client->groupspos = addstr("");
+		client->found_group = true;
 	}
 	else
 	{
@@ -673,7 +689,7 @@ static bool get_client_groupids(clientsData* client)
 
 	if(config.debug & DEBUG_CLIENTS)
 		logg("Gravity database: Client %s found. Using groups [%s].\n",
-		     show_client_string(hwaddr, hostname, ip), client->groups);
+		     show_client_string(hwaddr, hostname, ip), getstr(client->groupspos));
 
 	// Free possibly allocated memory
 	if(hwaddr != NULL)
@@ -764,7 +780,7 @@ bool gravityDB_prepare_client_statements(const int clientID, clientsData *client
 
 	// Get associated groups for this client (if defined)
 	char *querystr = NULL;
-	if(client->groups == NULL && !get_client_groupids(client))
+	if(!client->found_group && !get_client_groupids(client))
 		return false;
 
 	// Prepare whitelist statement
@@ -775,7 +791,7 @@ bool gravityDB_prepare_client_statements(const int clientID, clientsData *client
 	// of EXISTS().
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip);
-	querystr = get_client_querystr("vw_whitelist", client->groups);
+	querystr = get_client_querystr("vw_whitelist", getstr(client->groupspos));
 	sqlite3_stmt* stmt = NULL;
 	int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
 	if( rc != SQLITE_OK )
@@ -790,7 +806,7 @@ bool gravityDB_prepare_client_statements(const int clientID, clientsData *client
 	// Prepare gravity statement
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_gravity statement for client %s", clientip);
-	querystr = get_client_querystr("vw_gravity", client->groups);
+	querystr = get_client_querystr("vw_gravity", getstr(client->groupspos));
 	rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
@@ -804,7 +820,7 @@ bool gravityDB_prepare_client_statements(const int clientID, clientsData *client
 	// Prepare blacklist statement
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip);
-	querystr = get_client_querystr("vw_blacklist", client->groups);
+	querystr = get_client_querystr("vw_blacklist", getstr(client->groupspos));
 	rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
@@ -840,12 +856,12 @@ static inline void gravityDB_finalize_client_statements(const int clientID)
 		gravity_stmt->set(gravity_stmt, clientID, NULL);
 	}
 
-	// Free group memory
+	// Unset group found property to trigger a check next time the
+	// client sends a query
 	clientsData* client = getClient(clientID, true);
-	if(client != NULL && client->groups != NULL)
+	if(client != NULL)
 	{
-		free(client->groups);
-		client->groups = NULL;
+		client->found_group = false;
 	}
 }
 
@@ -1231,13 +1247,14 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex,
 	gravityDB_check_fork();
 
 	char *querystr = NULL;
-	if(client->groups == NULL && !get_client_groupids(client))
+	if(!client->found_group && !get_client_groupids(client))
 		return false;
 
 	// Group filtering
-	if(asprintf(&querystr, "SELECT id from %s WHERE group_id IN (%s);", table, client->groups) < 1)
+	const char *groups = getstr(client->groupspos);
+	if(asprintf(&querystr, "SELECT id from %s WHERE group_id IN (%s);", table, groups) < 1)
 	{
-		logg("gravityDB_get_regex_client_groups(%s, %s) - asprintf() error", table, client->groups);
+		logg("gravityDB_get_regex_client_groups(%s, %s) - asprintf() error", table, groups);
 		return false;
 	}
 

src/database/message-table.c

@@ -16,7 +16,7 @@
 #include "database/gravity-db.h"
 
 static const char *message_types[MAX_MESSAGE] =
-	{ "REGEX", "SUBNET" };
+	{ "REGEX", "SUBNET", "HOSTNAME" };
 
 static unsigned char message_blob_types[MAX_MESSAGE][5] =
 	{
@@ -33,6 +33,13 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] =
 			SQLITE_TEXT, // comma-separated list of matching subnets (database IDs)
 			SQLITE_TEXT, // chosen subnet (text representation)
 			SQLITE_INTEGER // chosen subnet (database ID)
+		},
+		{	// HOSTNAME_MESSAGE: The message column contains the IP address of the device
+			SQLITE_TEXT, // Obtained host name
+			SQLITE_INTEGER, // Position of error in string
+			SQLITE_NULL, // not used
+			SQLITE_NULL, // not used
+			SQLITE_NULL // not used
 		}
 	};
 // Create message table in the database
@@ -77,17 +84,78 @@ bool flush_message_table(void)
 static bool add_message(enum message_type type, const char *message,
                         const int count,...)
 {
-	// Open database connection
-	dbopen();
+	bool opened_database = false;
+	// Open database connection (if not already open)
+	if(!FTL_DB_avail())
+	{
+		if(!dbopen())
+			return false;
+		opened_database = true;
+	}
+
+	// Ensure there are no duplicates when adding host name messages
+	if(type == HOSTNAME_MESSAGE)
+	{
+		sqlite3_stmt* stmt = NULL;
+		const char *querystr = "DELETE FROM message WHERE type = ?1 AND message = ?2";
+		int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL);
+		if( rc != SQLITE_OK ){
+			logg("add_message(type=%u, message=%s) - SQL error prepare DELETE: %s",
+				type, message, sqlite3_errstr(rc));
+			if(opened_database)
+				dbclose();
+			return false;
+		}
+
+		// Bind type to prepared statement
+		if((rc = sqlite3_bind_text(stmt, 1, message_types[type], -1, SQLITE_STATIC)) != SQLITE_OK)
+		{
+			logg("add_message(type=%u, message=%s) - Failed to bind type DELETE: %s",
+				type, message, sqlite3_errstr(rc));
+			sqlite3_reset(stmt);
+			sqlite3_finalize(stmt);
+			if(opened_database)
+				dbclose();
+			return false;
+		}
+
+		// Bind message to prepared statement
+		if((rc = sqlite3_bind_text(stmt, 2, message, -1, SQLITE_STATIC)) != SQLITE_OK)
+		{
+			logg("add_message(type=%u, message=%s) - Failed to bind message DELETE: %s",
+				type, message, sqlite3_errstr(rc));
+			sqlite3_reset(stmt);
+			sqlite3_finalize(stmt);
+			if(opened_database)
+				dbclose();
+			return false;
+		}
+
+		// Execute and finalize
+		if((rc = sqlite3_step(stmt)) != SQLITE_OK && rc != SQLITE_DONE)
+		{
+			logg("add_message(type=%u, message=%s) - SQL error step DELETE: %s",
+			type, message, sqlite3_errstr(rc));
+			if(opened_database)
+				dbclose();
+			return false;
+		}
+		sqlite3_clear_bindings(stmt);
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+	}
 
 	// Prepare SQLite statement
 	sqlite3_stmt* stmt = NULL;
 	const char *querystr = "INSERT INTO message (timestamp,type,message,blob1,blob2,blob3,blob4,blob5) "
 	                       "VALUES ((cast(strftime('%s', 'now') as int)),?,?,?,?,?,?,?);";
 	int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL);
-	if( rc != SQLITE_OK ){
+	if( rc != SQLITE_OK )
+	{
 		logg("add_message(type=%u, message=%s) - SQL error prepare: %s",
 		     type, message, sqlite3_errstr(rc));
+		if(opened_database)
+			dbclose();
 		return false;
 	}
 
@@ -98,6 +166,8 @@ static bool add_message(enum message_type type, const char *message,
 		     type, message, sqlite3_errstr(rc));
 		sqlite3_reset(stmt);
 		sqlite3_finalize(stmt);
+		if(opened_database)
+			dbclose();
 		return false;
 	}
 
@@ -108,6 +178,8 @@ static bool add_message(enum message_type type, const char *message,
 		     type, message, sqlite3_errstr(rc));
 		sqlite3_reset(stmt);
 		sqlite3_finalize(stmt);
+		if(opened_database)
+			dbclose();
 		return false;
 	}
 
@@ -139,6 +211,8 @@ static bool add_message(enum message_type type, const char *message,
 			     type, message, 3 + j, datatype, sqlite3_errstr(rc));
 			sqlite3_reset(stmt);
 			sqlite3_finalize(stmt);
+			if(opened_database)
+				dbclose();
 			return false;
 		}
 	}
@@ -156,8 +230,9 @@ static bool add_message(enum message_type type, const char *message,
 		return false;
 	}
 
-	// Close database connection
-	dbclose();
+	// Close database connection (if we opened it)
+	if(opened_database)
+		dbclose();
 
 	return true;
 }
@@ -186,4 +261,14 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m
 	char *names = get_client_names_from_ids(matching_ids);
 	add_message(SUBNET_MESSAGE, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id);
 	free(names);
-}
+}
+
+void logg_hostname_warning(const char *ip, const char *name, const unsigned int pos)
+{
+	// Log to pihole-FTL.log
+	logg("HOSTNAME WARNING: Host name of client \"%s\" => \"%s\" contains (at least) one invalid character at position %d",
+	     ip, name, pos);
+
+	// Log to database
+	add_message(HOSTNAME_MESSAGE, ip, 2, name, (const int)pos);
+}

src/database/message-table.h

@@ -16,7 +16,8 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex
 void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids,
                          const int matching_bits, const char *chosen_match_text,
                          const int chosen_match_id);
+void logg_hostname_warning(const char *ip, const char *name, const unsigned int pos);
 
-enum message_type { REGEX_MESSAGE, SUBNET_MESSAGE, MAX_MESSAGE };
+enum message_type { REGEX_MESSAGE, SUBNET_MESSAGE, HOSTNAME_MESSAGE, MAX_MESSAGE };
 
 #endif //MESSAGETABLE_H

src/database/network-table.c

@@ -1319,6 +1319,25 @@ void updateMACVendorRecords(void)
 
 char* __attribute__((malloc)) getDatabaseHostname(const char* ipaddr)
 {
+	// Test if this is an IPv6 address
+	bool IPv6 = false;
+	if(ipaddr != NULL && strstr(ipaddr,":") != NULL)
+	{
+		IPv6 = true;
+	}
+
+	// Do we want to resolve IPv4/IPv6 names at all?
+	if( (IPv6 && !config.resolveIPv6) ||
+	   (!IPv6 && !config.resolveIPv4))
+	{
+		if(config.debug & DEBUG_RESOLVER)
+		{
+			logg(" ---> \"\" (configured to not resolve %s host names)",
+			     IPv6 ? "IPv6" : "IPv4");
+		}
+		return strdup("");
+	}
+
 	// Open pihole-FTL.db database file
 	if(!dbopen())
 	{
@@ -1546,3 +1565,136 @@ char* __attribute__((malloc)) getIfaceFromIP(const char* ipaddr)
 
 	return iface;
 }
+
+// Resolve unknown names of recently seen IP addresses in network table
+void resolveNetworkTableNames(void)
+{
+	// Open database file
+	if(!dbopen())
+	{
+		logg("resolveNetworkTableNames() - Failed to open DB");
+		return;
+	}
+
+	const char sql[] = "BEGIN TRANSACTION IMMEDIATE";
+	int rc = dbquery(sql);
+	if( rc != SQLITE_OK )
+	{
+		const char *text;
+		if( rc == SQLITE_BUSY )
+		{
+			text = "WARNING";
+		}
+		else
+		{
+			text = "ERROR";
+		}
+
+		// dbquery() above already logs the reson for why the query failed
+		logg("%s: Trying to resolve unknown network table host names (\"%s\") failed", text, sql);
+		dbclose();
+		return;
+	}
+
+	// Get IP addresses seen within the last 24 hours with empty or NULL host names
+	const char querystr[] = "SELECT ip FROM network_addresses "
+	                               "WHERE lastSeen > cast(strftime('%%s', 'now') as int)-86400;";
+
+	// Prepare query
+	sqlite3_stmt *table_stmt = NULL;
+	rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &table_stmt, NULL);
+	if(rc != SQLITE_OK)
+	{
+		logg("resolveNetworkTableNames() - SQL error prepare: %s",
+		     sqlite3_errstr(rc));
+		sqlite3_finalize(table_stmt);
+		dbclose();
+		return;
+	}
+
+
+	// Get data
+	while((rc = sqlite3_step(table_stmt)) == SQLITE_ROW)
+	{
+		// Get IP address from database
+		const char* ip = (const char*)sqlite3_column_text(table_stmt, 0);
+
+		if(config.debug & DEBUG_DATABASE)
+			logg("Resolving database IP %s", ip);
+
+		// Try to obtain host name
+		char* newname = resolveHostname(ip);
+
+		if(config.debug & DEBUG_DATABASE)
+			logg("---> \"%s\"", newname);
+
+		// Store new host name in database if not empty
+		if(newname != NULL && strlen(newname) > 0)
+		{
+			const char updatestr[] = "UPDATE network_addresses "
+			                                "SET name = ?1,"
+			                                    "nameUpdated = cast(strftime('%s', 'now') as int) "
+			                                "WHERE ip = ?2;";
+			sqlite3_stmt *update_stmt = NULL;
+			int rc2 = sqlite3_prepare_v2(FTL_db, updatestr, -1, &update_stmt, NULL);
+			if(rc2 != SQLITE_OK){
+				logg("resolveNetworkTableNames(%s -> \"%s\") - SQL error prepare: %s",
+					ip, newname, sqlite3_errstr(rc2));
+				sqlite3_finalize(update_stmt);
+				break;
+			}
+
+			// Bind newname to prepared statement
+			if((rc2 = sqlite3_bind_text(update_stmt, 1, newname, -1, SQLITE_STATIC)) != SQLITE_OK)
+			{
+				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to bind newname: %s",
+					ip, newname, sqlite3_errstr(rc2));
+				sqlite3_finalize(update_stmt);
+				break;
+			}
+
+			// Bind ip to prepared statement
+			if((rc2 = sqlite3_bind_text(update_stmt, 2, ip, -1, SQLITE_STATIC)) != SQLITE_OK)
+			{
+				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to bind ip: %s",
+					ip, newname, sqlite3_errstr(rc2));
+				sqlite3_finalize(update_stmt);
+				break;
+			}
+
+			if(config.debug & DEBUG_DATABASE)
+				logg("dbquery: \"%s\" with ?1 = \"%s\" and ?2 = \"%s\"", updatestr, newname, ip);
+
+			rc2 = sqlite3_step(update_stmt);
+			if(rc2 != SQLITE_BUSY && rc2 != SQLITE_DONE)
+			{
+				// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
+				// is a real error we should log
+				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to perform step: %s",
+				     ip, newname, sqlite3_errstr(rc2));
+				sqlite3_finalize(update_stmt);
+				break;
+			}
+
+			// Finalize host name update statement
+			sqlite3_finalize(update_stmt);
+		}
+		free(newname);
+	}
+
+	// Possible error handling and reporting
+	if(rc != SQLITE_DONE)
+	{
+		logg("resolveNetworkTableNames() - SQL error step: %s",
+		     sqlite3_errstr(rc));
+		sqlite3_finalize(table_stmt);
+		dbclose();
+		return;
+	}
+
+	// Close and unlock database connection
+	sqlite3_finalize(table_stmt);
+
+	dbquery("COMMIT;");
+	dbclose();
+}

src/database/network-table.h

@@ -20,5 +20,6 @@ char* getDatabaseHostname(const char* ipaddr) __attribute__((malloc));
 char* __attribute__((malloc)) getMACfromIP(const char* ipaddr);
 char* __attribute__((malloc)) getNameFromIP(const char* ipaddr);
 char* __attribute__((malloc)) getIfaceFromIP(const char* ipaddr);
+void resolveNetworkTableNames(void);
 
 #endif //NETWORKTABLE_H

src/datastructure.c

@@ -198,7 +198,8 @@ int findClientID(const char *clientIP, const bool count)
 	client->lastQuery = 0;
 	client->numQueriesARP = client->count;
 	// Configured groups are yet unknown
-	client->groups = NULL;
+	client->found_group = false;
+	client->groupspos = 0u;
 	// Interface is not yet known
 	client->ifacepos = 0;
 

src/datastructure.h

@@ -56,11 +56,12 @@ typedef struct {
 typedef struct {
 	unsigned char magic;
 	bool new;
+	bool found_group;
 	int count;
 	int blockedcount;
 	int overTime[OVERTIME_SLOTS];
 	unsigned int numQueriesARP;
-	char *groups;
+	size_t groupspos;
 	size_t ippos;
 	size_t namepos;
 	size_t ifacepos;

src/dnsmasq/rfc1035.c

@@ -1941,12 +1941,16 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 			    if (crecp->flags & F_NXDOMAIN)
 			      nxdomain = 1;
 			    if (!dryrun)
+			    {
 			      log_query(crecp->flags, name, NULL, NULL);
+			      FTL_cache(crecp->flags, name, NULL, NULL, daemon->log_display_id);
+			    }
 			  }
 			else if (!dryrun)
 			  {
 			    char *target = blockdata_retrieve(crecp->addr.srv.target, crecp->addr.srv.targetlen, NULL);
 			    log_query(crecp->flags, name, NULL, 0);
+			    FTL_cache(crecp->flags, name, NULL, NULL, daemon->log_display_id);
 			    
 			    if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, 
 						    crec_ttl(crecp, now), NULL, T_SRV, C_IN, "sssd",

src/resolve.c

@@ -22,10 +22,12 @@
 #include "database/network-table.h"
 // struct _res
 #include <resolv.h>
-// FTL_db
-#include "database/common.h"
+// resolveNetworkTableNames()
+#include "database/network-table.h"
 // resolver_ready
 #include "daemon.h"
+// logg_hostname_warning()
+#include "database/message-table.h"
 
 static bool res_initialized = false;
 
@@ -49,8 +51,10 @@ static bool valid_hostname(char* name, const char* clientip)
 
 	// Iterate over characters in hostname
 	// to check for legal char: A-Z a-z 0-9 - _ .
-	for (char c; (c = *name); name++)
+	unsigned int len = strlen(name);
+	for (unsigned int i = 0; i < len; i++)
 	{
+		const char c = name[i];
 		if ((c >= 'A' && c <= 'Z') ||
 		    (c >= 'a' && c <= 'z') ||
 		    (c >= '0' && c <= '9') ||
@@ -60,8 +64,7 @@ static bool valid_hostname(char* name, const char* clientip)
 			continue;
 
 		// Invalid character found, log and return hostname being invalid
-		logg("WARN: Hostname of client %s contains invalid character: %c (char code %d)",
-		     clientip, (unsigned char)c, (unsigned char)c);
+		logg_hostname_warning(clientip, name, i);
 		return false;
 	}
 
@@ -111,8 +114,8 @@ char *resolveHostname(const char *addr)
 		{
 			logg(" ---> \"\" (configured to not resolve %s host names)",
 			     IPv6 ? "IPv6" : "IPv4");
-			return strdup("");
 		}
+		return strdup("");
 	}
 
 	// Initialize resolver subroutines if trying to resolve for the first time
@@ -260,7 +263,8 @@ static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos)
 	char* newname = resolveHostname(ipaddr);
 
 	// If no hostname was found, try to obtain hostname from the network table
-	if(strlen(newname) == 0)
+	// This may be disabled due to a user setting
+	if(strlen(newname) == 0 && config.names_from_netdb)
 	{
 		free(newname);
 		newname = getDatabaseHostname(ipaddr);
@@ -306,17 +310,17 @@ void resolveClients(const bool onlynew)
 	int skipped = 0;
 	for(int clientID = 0; clientID < clientscount; clientID++)
 	{
-		// Get client pointer
+		// Memory access needs to get locked
+		lock_shm();
+		// Get client pointer for the first time (reading data)
 		clientsData* client = getClient(clientID, true);
 		if(client == NULL)
 		{
-			logg("ERROR: Unable to get client pointer with ID %i, skipping...", clientID);
+			logg("ERROR: Unable to get client pointer (1) with ID %i, skipping...", clientID);
 			skipped++;
 			continue;
 		}
 
-		// Memory access needs to get locked
-		lock_shm();
 		bool newflag = client->new;
 		size_t ippos = client->ippos;
 		size_t oldnamepos = client->namepos;
@@ -334,6 +338,18 @@ void resolveClients(const bool onlynew)
 		size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos);
 
 		lock_shm();
+		// Get client pointer for the second time (writing data)
+		// We cannot use the same pointer again as we released
+		// the lock in between so we cannot know if something
+		// happened to the shared memory object (resize event)
+		client = getClient(clientID, true);
+		if(client == NULL)
+		{
+			logg("ERROR: Unable to get client pointer (2) with ID %i, skipping...", clientID);
+			skipped++;
+			continue;
+		}
+
 		// Store obtained host name (may be unchanged)
 		client->namepos = newnamepos;
 		// Mark entry as not new
@@ -359,7 +375,9 @@ void resolveForwardDestinations(const bool onlynew)
 	int skipped = 0;
 	for(int upstreamID = 0; upstreamID < upstreams; upstreamID++)
 	{
-		// Get upstream pointer
+		// Memory access needs to get locked
+		lock_shm();
+		// Get upstream pointer for the first time (reading data)
 		upstreamsData* upstream = getUpstream(upstreamID, true);
 		if(upstream == NULL)
 		{
@@ -368,8 +386,6 @@ void resolveForwardDestinations(const bool onlynew)
 			continue;
 		}
 
-		// Memory access needs to get locked
-		lock_shm();
 		bool newflag = upstream->new;
 		size_t ippos = upstream->ippos;
 		size_t oldnamepos = upstream->namepos;
@@ -387,6 +403,18 @@ void resolveForwardDestinations(const bool onlynew)
 		size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos);
 
 		lock_shm();
+		// Get upstream pointer for the second time (writing data)
+		// We cannot use the same pointer again as we released
+		// the lock in between so we cannot know if something
+		// happened to the shared memory object (resize event)
+		upstream = getUpstream(upstreamID, true);
+		if(upstream == NULL)
+		{
+			logg("ERROR: Unable to get upstream pointer with ID %i, skipping...", upstreamID);
+			skipped++;
+			continue;
+		}
+
 		// Store obtained host name (may be unchanged)
 		upstream->namepos = newnamepos;
 		// Mark entry as not new
@@ -401,139 +429,6 @@ void resolveForwardDestinations(const bool onlynew)
 	}
 }
 
-// Resolve unknown names of recently seen IP addresses in network table
-static void resolveNetworkTableNames(void)
-{
-	// Open database file
-	if(!dbopen())
-	{
-		logg("resolveNetworkTableNames() - Failed to open DB");
-		return;
-	}
-
-	const char sql[] = "BEGIN TRANSACTION IMMEDIATE";
-	int rc = dbquery(sql);
-	if( rc != SQLITE_OK )
-	{
-		const char *text;
-		if( rc == SQLITE_BUSY )
-		{
-			text = "WARNING";
-		}
-		else
-		{
-			text = "ERROR";
-		}
-
-		// dbquery() above already logs the reson for why the query failed
-		logg("%s: Trying to resolve unknown network table host names (\"%s\") failed", text, sql);
-		dbclose();
-		return;
-	}
-
-	// Get IP addresses seen within the last 24 hours with empty or NULL host names
-	const char querystr[] = "SELECT ip FROM network_addresses "
-	                               "WHERE lastSeen > cast(strftime('%%s', 'now') as int)-86400;";
-
-	// Prepare query
-	sqlite3_stmt *table_stmt = NULL;
-	rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &table_stmt, NULL);
-	if(rc != SQLITE_OK)
-	{
-		logg("resolveNetworkTableNames() - SQL error prepare: %s",
-		     sqlite3_errstr(rc));
-		sqlite3_finalize(table_stmt);
-		dbclose();
-		return;
-	}
-
-
-	// Get data
-	while((rc = sqlite3_step(table_stmt)) == SQLITE_ROW)
-	{
-		// Get IP address from database
-		const char* ip = (const char*)sqlite3_column_text(table_stmt, 0);
-
-		if(config.debug & DEBUG_DATABASE)
-			logg("Resolving database IP %s", ip);
-
-		// Try to obtain host name
-		char* newname = resolveHostname(ip);
-
-		if(config.debug & DEBUG_DATABASE)
-			logg("---> \"%s\"", newname);
-
-		// Store new host name in database if not empty
-		if(newname != NULL && strlen(newname) > 0)
-		{
-			const char updatestr[] = "UPDATE network_addresses "
-			                                "SET name = ?1,"
-			                                    "nameUpdated = cast(strftime('%s', 'now') as int) "
-			                                "WHERE ip = ?2;";
-			sqlite3_stmt *update_stmt = NULL;
-			int rc2 = sqlite3_prepare_v2(FTL_db, updatestr, -1, &update_stmt, NULL);
-			if(rc2 != SQLITE_OK){
-				logg("resolveNetworkTableNames(%s -> \"%s\") - SQL error prepare: %s",
-					ip, newname, sqlite3_errstr(rc2));
-				sqlite3_finalize(update_stmt);
-				break;
-			}
-
-			// Bind newname to prepared statement
-			if((rc2 = sqlite3_bind_text(update_stmt, 1, newname, -1, SQLITE_STATIC)) != SQLITE_OK)
-			{
-				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to bind newname: %s",
-					ip, newname, sqlite3_errstr(rc2));
-				sqlite3_finalize(update_stmt);
-				break;
-			}
-
-			// Bind ip to prepared statement
-			if((rc2 = sqlite3_bind_text(update_stmt, 2, ip, -1, SQLITE_STATIC)) != SQLITE_OK)
-			{
-				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to bind ip: %s",
-					ip, newname, sqlite3_errstr(rc2));
-				sqlite3_finalize(update_stmt);
-				break;
-			}
-
-			if(config.debug & DEBUG_DATABASE)
-				logg("dbquery: \"%s\" with ?1 = \"%s\" and ?2 = \"%s\"", updatestr, newname, ip);
-
-			rc2 = sqlite3_step(update_stmt);
-			if(rc2 != SQLITE_BUSY && rc2 != SQLITE_DONE)
-			{
-				// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
-				// is a real error we should log
-				logg("resolveNetworkTableNames(%s -> \"%s\"): Failed to perform step: %s",
-				     ip, newname, sqlite3_errstr(rc2));
-				sqlite3_finalize(update_stmt);
-				break;
-			}
-
-			// Finalize host name update statement
-			sqlite3_finalize(update_stmt);
-		}
-		free(newname);
-	}
-
-	// Possible error handling and reporting
-	if(rc != SQLITE_DONE)
-	{
-		logg("resolveNetworkTableNames() - SQL error step: %s",
-		     sqlite3_errstr(rc));
-		sqlite3_finalize(table_stmt);
-		dbclose();
-		return;
-	}
-
-	// Close and unlock database connection
-	sqlite3_finalize(table_stmt);
-
-	dbquery("COMMIT;");
-	dbclose();
-}
-
 void *DNSclient_thread(void *val)
 {
 	// Set thread name

test/test_suite.bats

@@ -360,7 +360,7 @@
 }
 
 @test "No WARNING messages in pihole-FTL.log (besides known capability issues)" {
-  run bash -c 'grep "WARNING:" /var/log/pihole-FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW"'
+  run bash -c 'grep "WARNING:" /var/log/pihole-FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE"'
   printf "%s\n" "${lines[@]}"
   [[ ${lines[0]} == "0" ]]
 }