Merge branch 'development' into fix/externally_blocked_use_RA

Signed-off-by: DL6ER <dl6er@dl6er.de>

.circleci/config.yml

@@ -14,7 +14,7 @@ version: 2
         command: |
           BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master")
 
-          make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}"
+          make GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}"
           file pihole-FTL
     - run:
         name: "Upload"
@@ -59,7 +59,6 @@ jobs:
     <<: *job_template
     environment:
       BIN_NAME: "pihole-FTL-linux-x86_32"
-      CFLAGS: "-m32"
 
 workflows:
   version: 2

.gitignore

@@ -19,3 +19,7 @@ version*
 /pihole-FTL.conf
 /pihole-FTL.db
 /pihole-FTL.log
+
+# aux files
+aux/manuf.data
+aux/macvendor.db

FTL.h

@@ -36,8 +36,6 @@
 #include <pwd.h>
 // syslog
 #include <syslog.h>
-// SQLite
-#include "sqlite3.h"
 // tolower()
 #include <ctype.h>
 // Unix socket
@@ -69,7 +67,7 @@
 #define MAXITER 1000
 
 // FTLDNS enums
-enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER };
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER };
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED };
@@ -79,6 +77,23 @@ enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS,
 enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
 enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED };
 enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN };
+enum {
+  DEBUG_DATABASE   = (1 << 0), /* 00000000 00000001 */
+  DEBUG_NETWORKING = (1 << 1), /* 00000000 00000010 */
+  DEBUG_LOCKS      = (1 << 2), /* 00000000 00000100 */
+  DEBUG_QUERIES    = (1 << 3), /* 00000000 00001000 */
+  DEBUG_FLAGS      = (1 << 4), /* 00000000 00010000 */
+  DEBUG_SHMEM      = (1 << 5), /* 00000000 00100000 */
+  DEBUG_GC         = (1 << 6), /* 00000000 01000000 */
+  DEBUG_ARP        = (1 << 7), /* 00000000 10000000 */
+  DEBUG_REGEX      = (1 << 8), /* 00000001 00000000 */
+  DEBUG_API        = (1 << 9), /* 00000010 00000000 */
+};
+
+// Database table "ftl"
+enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
+// Database table "counters"
+enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
 
 // Privacy mode constants
 #define HIDDEN_DOMAIN "hidden"
@@ -93,6 +108,7 @@ typedef struct {
 	char* port;
 	char* db;
 	char* socketfile;
+	char* macvendordb;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -141,9 +157,10 @@ typedef struct {
 	unsigned char privacylevel;
 	bool ignore_localhost;
 	unsigned char blockingmode;
-	bool regex_debugmode;
 	bool analyze_only_A_AAAA;
 	bool DBimport;
+	bool parse_arp_cache;
+	int16_t debug;
 } ConfigStruct;
 
 // Dynamic structs
@@ -156,7 +173,7 @@ typedef struct {
 	int domainID;
 	int clientID;
 	int forwardID;
-	sqlite3_int64 db;
+	int64_t db;
 	int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here
 	bool complete;
 	unsigned char privacylevel;
@@ -181,6 +198,8 @@ typedef struct {
 	unsigned long long ippos;
 	unsigned long long namepos;
 	bool new;
+	time_t lastQuery;
+	unsigned int numQueriesARP;
 } clientsDataStruct;
 
 typedef struct {
@@ -206,12 +225,20 @@ typedef struct {
 	char **domains;
 } whitelistStruct;
 
+typedef struct {
+	int version;
+} ShmSettings;
+
 // Prepare timers, used mainly for debugging purposes
-#define NUMTIMERS 5
+#define NUMTIMERS LAST_TIMER
 
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
 
+// Some magic database constants
+#define DB_FAILED -2
+#define DB_NODATA -1
+
 extern logFileNamesStruct files;
 extern FTLFileNamesStruct FTLfiles;
 extern countersStruct *counters;
@@ -234,7 +261,6 @@ extern char ** setupVarsArray;
 extern int setupVarsElements;
 
 extern bool initialscan;
-extern bool debug;
 extern bool threadwritelock;
 extern bool threadreadlock;
 extern unsigned char blockingstatus;
@@ -249,7 +275,6 @@ extern long int lastdbindex;
 extern bool travis;
 extern bool DBdeleteoldqueries;
 extern bool rereadgravity;
-extern long int lastDBimportedtimestamp;
 extern bool ipv4telnet, ipv6telnet;
 extern bool istelnet[MAXCONNS];
 

Makefile

@@ -14,9 +14,9 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h
-FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o
+FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o networktable.o
 
-DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h
+DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h
 DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o
 
 # Get git commit version and date
@@ -51,7 +51,7 @@ CCFLAGS=-std=gnu11 -I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_
 # for dnsmasq we need the nettle crypto library and the gmp maths library
 # We link the two libraries statically. Althougth this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier
 #LIBS=-pthread -lnettle -lgmp -lhogweed
-LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle  -Wl,-Bdynamic -lrt
+LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic -lrt -lcap
 # Flags for compiling with libidn : -lidn
 # Flags for compiling with libidn2: -lidn2
 

api.c

@@ -11,6 +11,8 @@
 #include "FTL.h"
 #include "api.h"
 #include "version.h"
+// needed for sqlite3_libversion()
+#include "sqlite3.h"
 
 #define min(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; })
 
@@ -801,7 +803,7 @@ void getAllQueries(char *client_message, int *sock)
 
 		if(istelnet[*sock])
 		{
-			if(debug)
+			if(config.debug & DEBUG_API)
 				ssend(*sock,"%i %s %s %s %i %i %i %lu %i\n",queries[i].timestamp,qtype,domain,client,queries[i].status,queries[i].dnssec,queries[i].reply,delay,i);
 			else
 				ssend(*sock,"%i %s %s %s %i %i %i %lu\n",queries[i].timestamp,qtype,domain,client,queries[i].status,queries[i].dnssec,queries[i].reply,delay);

args.c

@@ -11,7 +11,7 @@
 #include "FTL.h"
 #include "version.h"
 
-bool debug = false;
+static bool debug = false;
 bool daemonmode = true;
 bool travis = false;
 int argc_dnsmasq = 0;

aux/macvendor.py

@@ -0,0 +1,83 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine - auxiliary files
+# MAC -> Vendor database generator
+#
+# This is a python3 script
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+import os
+import re
+import urllib.request
+import sqlite3
+
+# Download raw data from Wireshark's website
+# We use the official URL recommended in the header of this file
+print("Downloading...")
+urllib.request.urlretrieve("https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf", "manuf.data")
+print("...done")
+
+# Read file into memory and process lines
+manuf = open("manuf.data", "r")
+data = []
+print("Processing...")
+for line in manuf:
+	line = line.strip()
+
+	# Skip comments and empty lines
+	if line[1] == "#" or line == "":
+		continue
+
+	# Remove quotation marks as these might interfere with later INSERT / UPDATE commands
+	line = re.sub("\'|\"","", line)
+	# \s = Unicode whitespace characters, including [ \t\n\r\f\v]
+	cols = re.split("\s\s+|\t", line)
+	# Use try/except chain to catch empty/incomplete lines without failing hard
+	try:
+		# Strip whitespace and quotation marks (some entries are incomplete and cause errors with the CSV parser otherwise)
+		mac = cols[0].strip().strip("\"")
+	except:
+		continue
+	try:
+		desc_short = cols[1].strip().strip("\"")
+	except:
+		desc_short = ""
+	try:
+		desc_long = cols[2].strip().strip("\"")
+	except:
+		desc_long = ""
+
+	# Only add long description where available
+	# There are a few vendors for which only the
+	# short description field is used
+	if(desc_long):
+		data.append([mac, desc_long])
+	else:
+		data.append([mac, desc_short])
+print("...done")
+manuf.close()
+
+# Create database
+database = "macvendor.db"
+
+# Try to delete old database file, pass if no old file exists
+try:
+	os.remove(database)
+except OSError:
+	pass
+
+print("Generating database...")
+con = sqlite3.connect(database)
+cur = con.cursor()
+cur.execute("CREATE TABLE macvendor (mac TEXT NOT NULL, vendor TEXT NOT NULL, PRIMARY KEY (mac))")
+cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data)
+con.commit()
+print("...done.")
+print("Optimizing database...")
+con.execute("VACUUM")
+print("...done")
+print("Lines inserted into database:", cur.rowcount)

capabilities.c

@@ -0,0 +1,43 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Linux capability check routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include <sys/capability.h>
+
+bool check_capabilities()
+{
+	if(!cap_get_bound(CAP_NET_ADMIN))
+	{
+		// Needed for ARP-injection (used when we're the DHCP server)
+		logg("**************************************************************");
+		logg("WARNING: Required linux capability CAP_NET_ADMIN not available");
+		logg("**************************************************************");
+		return false;
+	}
+	if(!cap_get_bound(CAP_NET_RAW))
+	{
+		// Needed for raw socket access (necessary for ICMP)
+		logg("************************************************************");
+		logg("WARNING: Required linux capability CAP_NET_RAW not available");
+		logg("************************************************************");
+		return false;
+	}
+	if(!cap_get_bound(CAP_NET_BIND_SERVICE))
+	{
+		// Necessary for dynamic port binding
+		logg("*********************************************************************");
+		logg("WARNING: Required linux capability CAP_NET_BIND_SERVICE not available");
+		logg("*********************************************************************");
+		return false;
+	}
+
+	// All okay!
+	return true;
+}

config.c

@@ -255,19 +255,6 @@ void read_FTLconf(void)
 			break;
 	}
 
-	// REGEX_DEBUGMODE
-	// defaults to: No
-	config.regex_debugmode = false;
-	buffer = parse_FTLconf(fp, "REGEX_DEBUGMODE");
-
-	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
-		config.regex_debugmode = true;
-
-	if(config.regex_debugmode)
-		logg("   REGEX_DEBUGMODE: Active. May increase log file size!");
-	else
-		logg("   REGEX_DEBUGMODE: Inactive");
-
 	// ANALYZE_ONLY_A_AND_AAAA
 	// defaults to: No
 	config.analyze_only_A_AAAA = false;
@@ -319,6 +306,25 @@ void read_FTLconf(void)
 	// AUDITLISTFILE
 	getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist);
 
+	// MACVENDORDB
+	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb);
+
+	// PARSE_ARP_CACHE
+	// defaults to: true
+	config.parse_arp_cache = true;
+	buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE");
+
+	if(buffer != NULL && strcasecmp(buffer, "false") == 0)
+		config.parse_arp_cache = false;
+
+	if(config.parse_arp_cache)
+		logg("   PARSE_ARP_CACHE: Active");
+	else
+		logg("   PARSE_ARP_CACHE: Inactive");
+
+	// Read DEBUG_... setting from pihole-FTL.conf
+	read_debuging_settings(fp);
+
 	logg("Finished config file parsing");
 
 	// Release memory
@@ -383,9 +389,6 @@ static char *parse_FTLconf(FILE *fp, const char * key)
 	errno = 0;
 	while(getline(&conflinebuffer, &size, fp) != -1)
 	{
-		// Strip (possible) newline
-		conflinebuffer[strcspn(conflinebuffer, "\n")] = '\0';
-
 		// Skip comment lines
 		if(conflinebuffer[0] == '#' || conflinebuffer[0] == ';')
 			continue;
@@ -396,7 +399,13 @@ static char *parse_FTLconf(FILE *fp, const char * key)
 
 		// otherwise: key found
 		free(keystr);
-		return (find_equals(conflinebuffer) + 1);
+		// Note: value is still a pointer into the conflinebuffer
+		//       its memory will get released in release_config_memory()
+		char* value = find_equals(conflinebuffer) + 1;
+		// Trim whitespace at beginning and end, this function
+		// modifies the string inplace
+		trim_whitespace(value);
+		return value;
 	}
 
 	if(errno == ENOMEM)
@@ -493,3 +502,104 @@ void get_blocking_mode(FILE *fp)
 	if(opened)
 		fclose(fp);
 }
+
+void read_debuging_settings(FILE *fp)
+{
+	// Set default (no debug instructions set)
+	config.debug = 0;
+
+	// See if we got a file handle, if not we have to open
+	// the config file ourselves
+	bool opened = false;
+	if(fp == NULL)
+	{
+		if((fp = fopen(FTLfiles.conf, "r")) == NULL)
+			// Return silently if there is no config file available
+			return;
+		opened = true;
+	}
+
+	// DEBUG_DATABASE
+	// defaults to: false
+	char* buffer = parse_FTLconf(fp, "DEBUG_DATABASE");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_DATABASE;
+
+	// DEBUG_NETWORKING
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_NETWORKING");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_NETWORKING;
+
+	// DEBUG_LOCKS
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_LOCKS");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_LOCKS;
+
+	// DEBUG_QUERIES
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_QUERIES");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_QUERIES;
+
+	// DEBUG_FLAGS
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_FLAGS");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_FLAGS;
+
+	// DEBUG_SHMEM
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_SHMEM");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_SHMEM;
+
+	// DEBUG_GC
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_GC");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_GC;
+
+	// DEBUG_ARP
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_ARP");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_ARP;
+
+	// DEBUG_REGEX or REGEX_DEBUGMODE (legacy config option)
+	// defaults to: false
+	buffer = parse_FTLconf(fp, "DEBUG_REGEX");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_REGEX;
+	buffer = parse_FTLconf(fp, "REGEX_DEBUGMODE");
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.debug |= DEBUG_REGEX;
+
+	if(config.debug)
+	{
+		logg("************************");
+		logg("* Debugging enabled    *");
+		logg("* DEBUG_DATABASE   %s *", (config.debug & DEBUG_DATABASE)? "YES":"NO ");
+		logg("* DEBUG_NETWORKING %s *", (config.debug & DEBUG_NETWORKING)? "YES":"NO ");
+		logg("* DEBUG_LOCKS      %s *", (config.debug & DEBUG_LOCKS)? "YES":"NO ");
+		logg("* DEBUG_QUERIES    %s *", (config.debug & DEBUG_QUERIES)? "YES":"NO ");
+		logg("* DEBUG_FLAGS      %s *", (config.debug & DEBUG_FLAGS)? "YES":"NO ");
+		logg("* DEBUG_SHMEM      %s *", (config.debug & DEBUG_SHMEM)? "YES":"NO ");
+		logg("* DEBUG_GC         %s *", (config.debug & DEBUG_GC)? "YES":"NO ");
+		logg("* DEBUG_ARP        %s *", (config.debug & DEBUG_ARP)? "YES":"NO ");
+		logg("* DEBUG_REGEX      %s *", (config.debug & DEBUG_REGEX)? "YES":"NO ");
+		logg("************************");
+	}
+
+	// Have to close the config file if we opened it
+	if(opened)
+	{
+		fclose(fp);
+
+		// Release memory only when we opened the file
+		// Otherwise, it may still be needed outside of
+		// this function (initial config parsing)
+		release_config_memory();
+	}
+}

database.c

@@ -10,22 +10,16 @@
 
 #include "FTL.h"
 #include "shmem.h"
+#include "sqlite3.h"
 
-sqlite3 *db;
+static sqlite3 *db;
 bool database = false;
 bool DBdeleteoldqueries = false;
 long int lastdbindex = 0;
-long int lastDBimportedtimestamp = 0;
 
-pthread_mutex_t dblock;
-
-// TABLE ftl
-enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
-// TABLE counters
-enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
+static pthread_mutex_t dblock;
 
 bool db_set_counter(unsigned int ID, int value);
-bool db_set_FTL_property(unsigned int ID, int value);
 int db_get_FTL_property(unsigned int ID);
 
 void check_database(int rc)
@@ -39,6 +33,7 @@ void check_database(int rc)
 	   rc != SQLITE_ROW &&
 	   rc != SQLITE_BUSY)
 	{
+		logg("check_database(%i): Disabling database connection due to error", rc);
 		database = false;
 	}
 }
@@ -94,6 +89,8 @@ bool dbquery(const char *format, ...)
 		return false;
 	}
 
+	if(config.debug & DEBUG_DATABASE) logg("dbquery: %s", query);
+
 	int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg);
 
 	if( rc != SQLITE_OK ){
@@ -155,8 +152,8 @@ bool db_create(void)
 	ret = dbquery("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );");
 	if(!ret){ dbclose(); return false; }
 
-	// DB version 2
-	ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,2);", DB_VERSION);
+	// Set DB version 1
+	ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION);
 	if(!ret){ dbclose(); return false; }
 
 	// Most recent timestamp initialized to 00:00 1 Jan 1970
@@ -164,9 +161,15 @@ bool db_create(void)
 	if(!ret){ dbclose(); return false; }
 
 	// Create counter table
+	// Will update DB version to 2
 	if(!create_counter_table())
 		return false;
 
+	// Create network table
+	// Will update DB version to 3
+	if(!create_network_table())
+		return false;
+
 	return true;
 }
 
@@ -197,22 +200,40 @@ void db_init(void)
 
 	// Test DB version and see if we need to upgrade the database file
 	int dbversion = db_get_FTL_property(DB_VERSION);
+	logg("Database version is %i", dbversion);
 	if(dbversion < 1)
 	{
 		logg("Database version incorrect, database not available");
 		database = false;
 		return;
 	}
-	else if(dbversion < 2)
+	// Update to version 2 if lower
+	if(dbversion < 2)
 	{
-		// Database is still in version 1
-		// Update to version 2 and create counters table
+		// Update to version 2: Create counters table
+		logg("Updating long-term database to version 2");
 		if (!create_counter_table())
 		{
 			logg("Counter table not initialized, database not available");
 			database = false;
 			return;
 		}
+		// Get updated version
+		dbversion = db_get_FTL_property(DB_VERSION);
+	}
+	// Update to version 3 if lower
+	if(dbversion < 3)
+	{
+		// Update to version 3: Create network table
+		logg("Updating long-term database to version 3");
+		if (!create_network_table())
+		{
+			logg("Network table not initialized, database not available");
+			database = false;
+			return;
+		}
+		// Get updated version
+		dbversion = db_get_FTL_property(DB_VERSION);
 	}
 
 	// Close database to prevent having it opened all time
@@ -232,43 +253,20 @@ void db_init(void)
 
 int db_get_FTL_property(unsigned int ID)
 {
-	int rc, ret = 0;
-	sqlite3_stmt* dbstmt;
-	char *querystring = NULL;
-
 	// Prepare SQL statement
-	ret = asprintf(&querystring, "SELECT VALUE FROM ftl WHERE id = %u;",ID);
+	char* querystr = NULL;
+	int ret = asprintf(&querystr, "SELECT VALUE FROM ftl WHERE id = %u;", ID);
 
-	if(querystring == NULL || ret < 0)
+	if(querystr == NULL || ret < 0)
 	{
-		logg("Memory allocation failed in db_get_FTL_property, not saving query with ID = %u (%i)", ID, ret);
-		return false;
+		logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret);
+		return DB_FAILED;
 	}
 
-	rc = sqlite3_prepare(db, querystring, -1, &dbstmt, NULL);
-	if( rc ){
-		logg("db_get_FTL_property() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
-		logg("Query: \"%s\"", querystring);
-		dbclose();
-		check_database(rc);
-		return -1;
-	}
-	free(querystring);
+	int value = db_query_int(querystr);
+	free(querystr);
 
-	// Evaluate SQL statement
-	rc = sqlite3_step(dbstmt);
-	if( rc != SQLITE_ROW ){
-		logg("db_get_FTL_property() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return -1;
-	}
-
-	int result = sqlite3_column_int(dbstmt, 0);
-
-	sqlite3_finalize(dbstmt);
-
-	return result;
+	return value;
 }
 
 bool db_set_FTL_property(unsigned int ID, int value)
@@ -290,6 +288,42 @@ bool db_update_counters(int total, int blocked)
 	return true;
 }
 
+int db_query_int(const char* querystr)
+{
+	sqlite3_stmt* stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		dbclose();
+		check_database(rc);
+		return DB_FAILED;
+	}
+
+	rc = sqlite3_step(stmt);
+	int result;
+
+	if( rc == SQLITE_ROW )
+	{
+		result = sqlite3_column_int(stmt, 0);
+	}
+	else if( rc == SQLITE_DONE )
+	{
+		// No rows available
+		result = DB_NODATA;
+	}
+	else
+	{
+		logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		dbclose();
+		check_database(rc);
+		return DB_FAILED;
+	}
+
+	sqlite3_finalize(stmt);
+
+	return result;
+}
+
 int number_of_queries_in_DB(void)
 {
 	sqlite3_stmt* stmt;
@@ -300,7 +334,7 @@ int number_of_queries_in_DB(void)
 		logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	rc = sqlite3_step(stmt);
@@ -308,7 +342,7 @@ int number_of_queries_in_DB(void)
 		logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	int result = sqlite3_column_int(stmt, 0);
@@ -327,7 +361,7 @@ static sqlite3_int64 last_ID_in_DB(void)
 		logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	rc = sqlite3_step(stmt);
@@ -335,7 +369,7 @@ static sqlite3_int64 last_ID_in_DB(void)
 		logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	sqlite3_int64 result = sqlite3_column_int64(stmt, 0);
@@ -347,12 +381,12 @@ static sqlite3_int64 last_ID_in_DB(void)
 
 int get_number_of_queries_in_DB(void)
 {
-	int result = -1;
+	int result = DB_NODATA;
 
 	if(!dbopen())
 	{
 		logg("Failed to open DB in get_number_of_queries_in_DB()");
-		return -2;
+		return DB_FAILED;
 	}
 
 	result = number_of_queries_in_DB();
@@ -370,7 +404,7 @@ void save_to_DB(void)
 		return;
 
 	// Start database timer
-	if(debug) timer_start(DATABASE_WRITE_TIMER);
+	if(config.debug & DEBUG_DATABASE) timer_start(DATABASE_WRITE_TIMER);
 
 	// Open database
 	if(!dbopen())
@@ -406,7 +440,7 @@ void save_to_DB(void)
 	int total = 0, blocked = 0;
 	time_t currenttimestamp = time(NULL);
 	time_t newlasttimestamp = 0;
-	for(i = 0; i < counters->queries; i++)
+	for(i = MAX(0, lastdbindex); i < counters->queries; i++)
 	{
 		validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__);
 		if(queries[i].db != 0)
@@ -521,7 +555,7 @@ void save_to_DB(void)
 	// Close database
 	dbclose();
 
-	if(debug)
+	if(config.debug & DEBUG_DATABASE)
 	{
 		logg("Notice: Queries stored in DB: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID);
 		if(saved_error > 0)
@@ -552,7 +586,7 @@ void delete_old_queries_in_DB(void)
 	int affected = sqlite3_changes(db);
 
 	// Print final message only if there is a difference
-	if(debug || affected)
+	if((config.debug & DEBUG_DATABASE) || affected)
 		logg("Notice: Database size is %.2f MB, deleted %i rows", get_db_filesize(), affected);
 
 	// Close database
@@ -596,6 +630,10 @@ void *DB_thread(void *val)
 				delete_old_queries_in_DB();
 				DBdeleteoldqueries = false;
 			}
+
+			// Parse ARP cache (fill network table) if enabled
+			if (config.parse_arp_cache)
+				parse_arp_cache();
 		}
 		sleepms(100);
 	}
@@ -629,7 +667,7 @@ void read_data_from_DB(void)
 		return;
 	}
 	// Log DB query string in debug mode
-	if(debug) logg(rstr);
+	if(config.debug & DEBUG_DATABASE) logg(rstr);
 
 	// Prepare SQLite3 statement
 	sqlite3_stmt* stmt;
@@ -645,7 +683,7 @@ void read_data_from_DB(void)
 	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
 	{
 		sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0);
-		int queryTimeStamp = sqlite3_column_int(stmt, 1);
+		time_t queryTimeStamp = sqlite3_column_int(stmt, 1);
 		// 1483228800 = 01/01/2017 @ 12:00am (UTC)
 		if(queryTimeStamp < 1483228800)
 		{
@@ -654,7 +692,7 @@ void read_data_from_DB(void)
 		}
 		if(queryTimeStamp > now)
 		{
-			if(debug) logg("DB warn: Skipping query logged in the future (%i)", queryTimeStamp);
+			if(config.debug & DEBUG_DATABASE) logg("DB warn: Skipping query logged in the future (%i)", queryTimeStamp);
 			continue;
 		}
 
@@ -717,7 +755,7 @@ void read_data_from_DB(void)
 		int overTimeTimeStamp = queryTimeStamp - (queryTimeStamp % 600) + 300;
 		int timeidx = findOverTimeID(overTimeTimeStamp);
 		int domainID = findDomainID(domain);
-		int clientID = findClientID(client);
+		int clientID = findClientID(client, true);
 
 		// Ensure we have enough space in the queries struct
 		memory_check(QUERIES);
@@ -728,6 +766,7 @@ void read_data_from_DB(void)
 		// Store this query in memory
 		validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
 		validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__);
+		validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
 		queries[queryIndex].magic = MAGICBYTE;
 		queries[queryIndex].timestamp = queryTimeStamp;
 		queries[queryIndex].type = type;
@@ -738,11 +777,14 @@ void read_data_from_DB(void)
 		queries[queryIndex].timeidx = timeidx;
 		queries[queryIndex].db = dbid;
 		queries[queryIndex].id = 0;
-		queries[queryIndex].complete = true; // Mark as all information is avaiable
+		queries[queryIndex].complete = true; // Mark as all information is available
 		queries[queryIndex].response = 0;
 		queries[queryIndex].dnssec = DNSSEC_UNKNOWN;
 		queries[queryIndex].reply = REPLY_UNKNOWN;
-		lastDBimportedtimestamp = queryTimeStamp;
+
+		// Set lastQuery timer and add one query for network table
+		clients[clientID].lastQuery = queryTimeStamp;
+		clients[clientID].numQueriesARP++;
 
 		// Handle type counters
 		if(type >= TYPE_A && type < TYPE_MAX)
@@ -797,6 +839,10 @@ void read_data_from_DB(void)
 	}
 	logg("Imported %i queries from the long-term database", counters->queries);
 
+	// Update lastdbindex so that the next call to save_to_DB()
+	// skips the queries that we just imported from the database
+	lastdbindex = counters->queries;
+
 	if( rc != SQLITE_DONE ){
 		logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();

datastructure.c

@@ -176,7 +176,7 @@ int findDomainID(const char *domain)
 	return domainID;
 }
 
-int findClientID(const char *client)
+int findClientID(const char *client, bool count)
 {
 	int i;
 	// Compare content of client against known client IP addresses
@@ -191,11 +191,17 @@ int findClientID(const char *client)
 		// If so, compare the full IP using strcmp
 		if(strcmp(getstr(clients[i].ippos), client) == 0)
 		{
-			clients[i].count++;
+			// Add one if count == true (do not add one, e.g., during ARP table processing)
+			if(count) clients[i].count++;
 			return i;
 		}
 	}
 
+	// Return -1 (= not found) if count is false ...
+	if(!count)
+		return -1;
+	// ... otherwise proceed with adding a new client entry
+
 	// If we did not return until here, then this client is definitely new
 	// Store ID
 	int clientID = counters->clients;
@@ -218,6 +224,9 @@ int findClientID(const char *client)
 	// to be done separately to be non-blocking
 	clients[clientID].new = true;
 	clients[clientID].namepos = 0;
+	// No query seen so far
+	clients[clientID].lastQuery = 0;
+	clients[clientID].numQueriesARP = 0;
 
 	// Create new overTime client data
 	newOverTimeClient(clientID);

dnsmasq_interface.c

@@ -17,7 +17,7 @@
 void print_flags(unsigned int flags);
 void save_reply_type(unsigned int flags, int queryID, struct timeval response);
 unsigned long converttimeval(struct timeval time);
-static void block_single_domain(char *domain);
+static void block_single_domain_regex(char *domain);
 static void detect_blocked_IP(unsigned short flags, char* answer, int queryID);
 static void query_externally_blocked(int i);
 static int findQueryID(int id);
@@ -25,7 +25,7 @@ static int findQueryID(int id);
 unsigned char* pihole_privacylevel = &config.privacylevel;
 char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "};
 
-void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type)
+void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -61,7 +61,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	else
 	{
 		// Return early to avoid accessing querytypedata out of bounds
-		if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id);
+		if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping unknown query type: %s (%i)", types, id);
 		unlock_shm();
 		return;
 	}
@@ -69,7 +69,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	// Skip AAAA queries if user doesn't want to have them analyzed
 	if(!config.analyze_AAAA && querytype == TYPE_AAAA)
 	{
-		if(debug) logg("Not analyzing AAAA query");
+		if(config.debug & DEBUG_QUERIES) logg("Not analyzing AAAA query");
 		unlock_shm();
 		return;
 	}
@@ -112,7 +112,8 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 
 	// Log new query if in debug mode
 	char *proto = (type == UDP) ? "UDP" : "TCP";
-	if(debug) logg("**** new %s %s \"%s\" from %s (ID %i, FTL %i)", proto, types, domain, client, id, queryID);
+	if(config.debug & DEBUG_QUERIES)
+		logg("**** new %s %s \"%s\" from %s (ID %i, FTL %i, %s:%i)", proto, types, domain, client, id, queryID, file, line);
 
 	// Update counters
 	int timeidx = findOverTimeID(overTimetimestamp);
@@ -125,7 +126,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA)
 	{
 		// Don't process this query further here, we already counted it
-		if(debug) logg("Notice: Skipping new query: %s (%i)", types, id);
+		if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping new query: %s (%i)", types, id);
 		free(domain);
 		free(domainbuffer);
 		free(client);
@@ -137,7 +138,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	int domainID = findDomainID(domain);
 
 	// Go through already knows clients and see if it is one of them
-	int clientID = findClientID(client);
+	int clientID = findClientID(client, true);
 
 	// Save everything
 	validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__);
@@ -177,6 +178,10 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	// Update overTime data structure with the new client
 	overTimeClientData[clientID][timeidx]++;
 
+	// Set lastQuery timer and add one query for network table
+	clients[clientID].lastQuery = querytimestamp;
+	clients[clientID].numQueriesARP++;
+
 	// Try blocking regex if configured
 	validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__);
 	if(domains[domainID].regexmatch == REGEX_UNKNOWN && blockingstatus != BLOCKING_DISABLED)
@@ -194,7 +199,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 		if(match_regex(domainbuffer) && !in_whitelist(domainbuffer))
 		{
 			// We have to block this domain
-			block_single_domain(domainbuffer);
+			block_single_domain_regex(domainbuffer);
 			domains[domainID].regexmatch = REGEX_BLOCKED;
 		}
 		else
@@ -237,7 +242,7 @@ static int findQueryID(int id)
 	return -1;
 }
 
-void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id)
+void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -255,7 +260,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id
 	strtolower(forward);
 
 	// Debug logging
-	if(debug) logg("**** forwarded %s to %s (ID %i)", name, forward, id);
+	if(config.debug & DEBUG_QUERIES) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, forward, id, file, line);
 
 	// Save status and forwardID in corresponding query identified by dnsmasq's ID
 	int i = findQueryID(id);
@@ -368,9 +373,12 @@ void FTL_dnsmasq_reload(void)
 	// Reread regex.list
 	free_regex();
 	read_regex_from_file();
+
+	// Reread pihole-FTL.conf to see which debugging flags are set
+	read_debuging_settings(NULL);
 }
 
-void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
+void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -395,9 +403,9 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 	else if(flags & F_NEG)
 		answer = "(NODATA)";
 
-	if(debug)
+	if(config.debug & DEBUG_QUERIES)
 	{
-		logg("**** got reply %s is %s (ID %i)", name, answer, id);
+		logg("**** got reply %s is %s (ID %i, %s:%i)", name, answer, id, file, line);
 		print_flags(flags);
 	}
 
@@ -410,7 +418,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 	if(i < 0)
 	{
 		// This may happen e.g. if the original query was "pi.hole"
-		if(debug) logg("FTL_reply(): Query %i has not been found", id);
+		if(config.debug & DEBUG_QUERIES) logg("FTL_reply(): Query %i has not been found", id);
 		unlock_shm();
 		return;
 	}
@@ -585,7 +593,7 @@ static void query_externally_blocked(int i)
 	queries[i].status = QUERY_EXTERNAL_BLOCKED;
 }
 
-void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id)
+void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -614,8 +622,11 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 	free(domain);
 
 	// Debug logging
-	if(debug) logg("**** got cache answer for %s / %s / %s (ID %i)", name, dest, arg, id);
-	if(debug) print_flags(flags);
+	if(config.debug & DEBUG_QUERIES)
+	{
+		logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line);
+		print_flags(flags);
+	}
 
 	// Get response time
 	struct timeval response;
@@ -731,7 +742,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 	unlock_shm();
 }
 
-void FTL_dnssec(int status, int id)
+void _FTL_dnssec(int status, int id, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -749,11 +760,11 @@ void FTL_dnssec(int status, int id)
 	}
 
 	// Debug logging
-	if(debug)
+	if(config.debug & DEBUG_QUERIES)
 	{
 		int domainID = queries[i].domainID;
 		validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
-		logg("**** got DNSSEC details for %s: %i (ID %i)", getstr(domains[domainID].domainpos), status, id);
+		logg("**** got DNSSEC details for %s: %i (ID %i, %s:%i)", getstr(domains[domainID].domainpos), status, id, file, line);
 	}
 
 	// Iterate through possible values
@@ -767,7 +778,7 @@ void FTL_dnssec(int status, int id)
 	unlock_shm();
 }
 
-void FTL_header_analysis(unsigned char header4, unsigned int rcode, int id)
+void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const int id, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -797,11 +808,11 @@ void FTL_header_analysis(unsigned char header4, unsigned int rcode, int id)
 		return;
 	}
 
-	if(debug)
+	if(config.debug & DEBUG_QUERIES)
 	{
 		int domainID = queries[queryID].domainID;
 		validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
-		logg("**** %s externally blocked (ID %i, FTL %i)", getstr(domains[domainID].domainpos), id, queryID);
+		logg("**** %s externally blocked (ID %i, FTL %i, %s:%i)", getstr(domains[domainID].domainpos), id, queryID, file, line);
 	}
 
 
@@ -822,6 +833,11 @@ void print_flags(unsigned int flags)
 {
 	// Debug function, listing resolver flags in clear text
 	// e.g. "Flags: F_FORWARD F_NEG F_IPV6"
+
+	// Only print flags if corresponding debugging flag is set
+	if(!(config.debug & DEBUG_FLAGS))
+		return;
+
 	unsigned int i;
 	char *flagstr = calloc(256,sizeof(char));
 	for(i = 0; i < sizeof(flags)*8; i++)
@@ -974,7 +990,7 @@ void getCacheInformation(int *sock)
 	// which hasn't been looked up for the longest time is evicted.
 }
 
-void FTL_forwarding_failed(struct server *server)
+void _FTL_forwarding_failed(struct server *server, const char* file, const int line)
 {
 	// Don't analyze anything if in PRIVACY_NOSTATS mode
 	if(config.privacylevel >= PRIVACY_NOSTATS)
@@ -993,7 +1009,7 @@ void FTL_forwarding_failed(struct server *server)
 	strtolower(forward);
 	int forwardID = findForwardID(forward, false);
 
-	if(debug) logg("**** forwarding to %s (ID %i) failed", dest, forwardID);
+	if(config.debug & DEBUG_QUERIES) logg("**** forwarding to %s (ID %i, %s:%i) failed", dest, forwardID, file, line);
 
 	forwarded[forwardID].failed++;
 
@@ -1056,14 +1072,15 @@ void rehash(int size);
 // This routine adds one domain to the resolver's cache. Depending on the configured blocking mode it may create
 // a single entry valid for IPv4 & IPv6 or two entries one for IPv4 and one for IPv6.
 // When IPv6 is not available on the machine, we do not add IPv6 cache entries (likewise for IPv4)
-static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *addr6, bool has_IPv4, bool has_IPv6,
-                                    char *domain, struct crec **rhash, int hashsz, unsigned int index)
+static int add_blocked_domain(struct all_addr *addr4, struct all_addr *addr6, bool has_IPv4, bool has_IPv6,
+                              char *domain, int len, struct crec **rhash, int hashsz, unsigned int index)
 {
 	int name_count = 0;
 	struct crec *cache4,*cache6;
-	// Add IPv4 record
+	// Add IPv4 record, allocate enough space for cache entry including arbitrary domain name length
+	// (the domain name is stored at the end of struct crec)
 	if(has_IPv4 &&
-	   (cache4 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME)))
+	   (cache4 = malloc(sizeof(struct crec) + len+1-SMALLDNAME)))
 	{
 		strcpy(cache4->name.sname, domain);
 		cache4->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV4;
@@ -1093,7 +1110,7 @@ static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *add
 	}
 	// Add IPv6 record only if we respond with a non-NULL IP address to blocked domains
 	if(has_IPv6 && (config.blockingmode == MODE_IP || config.blockingmode == MODE_IP_NODATA_AAAA) &&
-	   (cache6 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME)))
+	   (cache6 = malloc(sizeof(struct crec) + len+1-SMALLDNAME)))
 	{
 		strcpy(cache6->name.sname, domain);
 		cache6->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV6;
@@ -1102,11 +1119,15 @@ static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *add
 		add_hosts_entry(cache6, addr6, IN6ADDRSZ, index, rhash, hashsz);
 		name_count++;
 	}
+
+	// Return 1 if only one cache slot was allocated (IPv4) or 2 if two slots were allocated (IPv4 + IPv6)
 	return name_count;
 }
 
 // Add a single domain to resolver's cache. This respects the configured blocking mode
-static void block_single_domain(char *domain)
+// Note: This routine is meant for adding a single domain at a time. It should not be
+//       invoked for batch processing
+static void block_single_domain_regex(char *domain)
 {
 	struct all_addr addr4 = {{{ 0 }}}, addr6 = {{{ 0 }}};
 	bool has_IPv4 = false, has_IPv6 = false;
@@ -1114,9 +1135,9 @@ static void block_single_domain(char *domain)
 	// Get IPv4/v6 addresses for blocking depending on user configures blocking mode
 	prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6);
 	regexlistname = files.regexlist;
-	add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, SRC_REGEX);
+	add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, strlen(domain), NULL, 0, SRC_REGEX);
 
-	if(debug) logg("Added %s to cache", domain);
+	if(config.debug & DEBUG_QUERIES) logg("Added %s to cache", domain);
 
 	return;
 }
@@ -1142,7 +1163,8 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s
 	// Get IPv4/v6 addresses for blocking depending on user configured blocking mode
 	prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6);
 
-	// If we have neither a valid IPv4 nor a valid IPv6, then we cannot add any entries here
+	// If we have neither a valid IPv4 nor a valid IPv6 but the user asked for
+	// blocking modes MODE_IP or MODE_IP_NODATA_AAAA then we cannot add any entries here
 	if(!has_IPv4 && !has_IPv6)
 	{
 		logg("ERROR: found neither a valid IPV4_ADDRESS nor IPV6_ADDRESS in setupVars.conf");
@@ -1163,7 +1185,8 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s
 
 		// Check for spaces or tabs
 		// If found, then this list is still in HOSTS format and we
-		// don't analyze it here.
+		// don't analyze it here. We only check the first line for
+		// efficiency reasons (strstr() is slow)
 		if(firstline &&
 		   (strstr(domain, " ") != NULL || strstr(domain, "\t") != NULL))
 		{
@@ -1175,12 +1198,16 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s
 		firstline = false;
 
 		// Skip empty lines
-		if(strlen(domain) == 0)
+		int len = strlen(domain);
+		if(len == 0)
 			continue;
 
 		// Strip newline character at the end of line we just read
-		if(domain[strlen(domain)-1] == '\n')
-			domain[strlen(domain)-1] = '\0';
+		if(domain[len-1] == '\n')
+		{
+			domain[len-1] = '\0';
+			len -= 1;
+		}
 
 		// As of here we assume the entry to be valid
 		// Rehash every 1000 valid names
@@ -1190,11 +1217,17 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s
 			cache_size = name_count;
 		}
 
-		name_count += add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, rhash, hashsz, index);
+		// Add domain
+		name_count += add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, len, rhash, hashsz, index);
+
 		// Count added domain
 		added++;
 	}
 
+	// Rehash after having read all entries
+	if(rhash)
+		rehash(name_count);
+
 	// Free allocated memory
 	if(buffer != NULL)
 	{

dnsmasq_interface.h

@@ -11,15 +11,27 @@ extern int socketfd, telnetfd4, telnetfd6;
 extern unsigned char* pihole_privacylevel;
 enum { TCP, UDP };
 
-void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type);
-void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id);
-void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id);
-void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id);
-void FTL_dnssec(int status, int id);
+#define FTL_new_query(flags, name, addr, types, id, type) _FTL_new_query(flags, name, addr, types, id, type, __FILE__, __LINE__)
+void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line);
+
+#define FTL_forwarded(flags, name, addr, id) _FTL_forwarded(flags, name, addr, id, __FILE__, __LINE__)
+void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line);
+
+#define FTL_reply(flags, name, addr, id) _FTL_reply(flags, name, addr, id, __FILE__, __LINE__)
+void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line);
+
+#define FTL_cache(flags, name, addr, arg, id) _FTL_cache(flags, name, addr, arg, id, __FILE__, __LINE__)
+void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id, const char* file, const int line);
+
+#define FTL_dnssec(status, id) _FTL_dnssec(status, id, __FILE__, __LINE__)
+void _FTL_dnssec(int status, int id, const char* file, const int line);
+
+#define FTL_header_analysis(header4, rcode, id) _FTL_header_analysis(header4, rcode, id, __FILE__, __LINE__)
+void _FTL_header_analysis(unsigned char header4, unsigned int rcode, int id, const char* file, const int line);
+
+#define FTL_forwarding_failed(server) _FTL_forwarding_failed(server, __FILE__, __LINE__)
+void _FTL_forwarding_failed(struct server *server, const char* file, const int line);
+
 void FTL_dnsmasq_reload(void);
 void FTL_fork_and_bind_sockets(struct passwd *ent_pw);
-
-void FTL_header_analysis(unsigned char header4, unsigned int rcode, int id);
-
-void FTL_forwarding_failed(struct server *server);
 int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz);

docker/aarch64/Dockerfile

@@ -3,6 +3,6 @@ FROM debian:stretch
 RUN dpkg --add-architecture arm64 && \
     apt-get update && \
     apt-get install -y --no-install-recommends nettle-dev:arm64 gcc-aarch64-linux-gnu libc-dev-arm64-cross \
-        make file wget netcat-traditional sqlite3 git ca-certificates ssh
+        make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:arm64
 
 ENV CC aarch64-linux-gnu-gcc

docker/armhf/Dockerfile

@@ -3,6 +3,6 @@ FROM debian:stretch
 RUN dpkg --add-architecture armhf && \
     apt-get update && \
     apt-get install -y --no-install-recommends nettle-dev:armhf gcc-arm-linux-gnueabihf libc6-dev-armhf-cross \
-        make file wget netcat-traditional sqlite3 git ca-certificates ssh
+        make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:armhf
 
 ENV CC arm-linux-gnueabihf-gcc

docker/x86_32/Dockerfile

@@ -3,6 +3,6 @@ FROM debian:stretch
 RUN dpkg --add-architecture i386 && \
     apt-get update && \
     apt-get install -y --no-install-recommends nettle-dev:i386 gcc gcc-multilib \
-        make file wget netcat-traditional sqlite3 git ca-certificates ssh
+        make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:i386
 
-ENV CC gcc
+ENV CC "gcc -m32"

docker/x86_64/Dockerfile

@@ -2,6 +2,6 @@ FROM debian:stretch
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends nettle-dev gcc libc-dev \
-        make file wget netcat-traditional sqlite3 git ca-certificates ssh
+        make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev
 
 ENV CC gcc

gc.c

@@ -37,11 +37,11 @@ void *GC_thread(void *val)
 			// Get minimum time stamp to keep
 			time_t mintime = time(NULL) - config.maxlogage;
 
-			if(debug) timer_start(GC_TIMER);
+			if(config.debug & DEBUG_GC) timer_start(GC_TIMER);
 
 			long int i;
 			int removed = 0;
-			if(debug) logg("GC starting, mintime: %u %s", mintime, ctime(&mintime));
+			if(config.debug & DEBUG_GC) logg("GC starting, mintime: %u %s", mintime, ctime(&mintime));
 
 			// Process all queries
 			for(i=0; i < counters->queries; i++)
@@ -51,7 +51,6 @@ void *GC_thread(void *val)
 				if(queries[i].timestamp > mintime)
 					break;
 
-
 				// Adjust total counters and total over time data
 				// We cannot edit counters->queries directly as it is used
 				// as max ID for the queries[] struct
@@ -154,11 +153,13 @@ void *GC_thread(void *val)
 
 			// Update queries counter
 			counters->queries -= removed;
+			// Update DB index as total number of queries reduced
+			lastdbindex -= removed;
 
 			// Zero out remaining memory (marked as "F" in the above example)
 			memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries));
 
-			if(debug) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER));
+			if(config.debug & DEBUG_GC) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER));
 
 			// Release thread lock
 			unlock_shm();

grep.c

@@ -142,5 +142,5 @@ void check_blocking_status(void)
 		message = "disabled";
 	}
 
-	if(debug) logg("Blocking status is %s", message);
+	logg("Blocking status is %s", message);
 }

log.c

@@ -92,7 +92,7 @@ void logg(const char *format, ...)
 		va_end(args);
 		fputc('\n',logfile);
 	}
-	else if(debug)
+	else if(!daemonmode)
 	{
 		printf("!!! WARNING: Writing to FTL\'s log file failed!\n");
 		syslog(LOG_ERR, "Writing to FTL\'s log file failed!");

main.c

@@ -69,7 +69,11 @@ int main (int argc, char* argv[])
 	log_counter_info();
 	check_setupVarsconf();
 
-	// Preparations done - start the resolver
+	// Check for availability of advanced capabilities
+	// immediately before starting the resolver.
+	check_capabilities();
+
+	// Start the resolver
 	main_dnsmasq(argc_dnsmasq, argv_dnsmasq);
 
 	logg("Shutting down...");

memory.c

@@ -20,6 +20,7 @@ FTLFileNamesStruct FTLfiles = {
 	NULL,
 	NULL,
 	NULL,
+	NULL,
 	NULL
 };
 

networktable.c

@@ -0,0 +1,336 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Network table routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "shmem.h"
+#include "sqlite3.h"
+#define ARPCACHE "/proc/net/arp"
+
+// Private prototypes
+static char* getMACVendor(const char* hwaddr);
+
+bool create_network_table(void)
+{
+	bool ret;
+	// Create network table in the database
+	ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \
+	                                     "ip TEXT NOT NULL, " \
+	                                     "hwaddr TEXT NOT NULL, " \
+	                                     "interface TEXT NOT NULL, " \
+	                                     "name TEXT, " \
+	                                     "firstSeen INTEGER NOT NULL, " \
+	                                     "lastQuery INTEGER NOT NULL, " \
+	                                     "numQueries INTEGER NOT NULL," \
+	                                     "macVendor TEXT);");
+	if(!ret){ dbclose(); return false; }
+
+	// Update database version to 3
+	ret = db_set_FTL_property(DB_VERSION, 3);
+	if(!ret){ dbclose(); return false; }
+
+	return true;
+}
+
+// Read kernel's ARP cache using procfs
+void parse_arp_cache(void)
+{
+	FILE* arpfp = NULL;
+	// Try to access the kernel's ARP cache
+	if((arpfp = fopen(ARPCACHE, "r")) == NULL)
+	{
+		logg("WARN: Opening of %s failed!", ARPCACHE);
+		logg("      Message: %s", strerror(errno));
+		return;
+	}
+
+	// Open database file
+	if(!dbopen())
+	{
+		logg("read_arp_cache() - Failed to open DB");
+		fclose(arpfp);
+		return;
+	}
+
+	// Start ARP timer
+	if(config.debug & DEBUG_ARP) timer_start(ARP_TIMER);
+
+	// Prepare buffers
+	char * linebuffer = NULL;
+	size_t linebuffersize = 0;
+	char ip[100], mask[100], hwaddr[100], iface[100];
+	int type, flags, entries = 0;
+	time_t now = time(NULL);
+
+	// Start collecting database commands
+	dbquery("BEGIN TRANSACTION");
+
+	// Read ARP cache line by line
+	while(getline(&linebuffer, &linebuffersize, arpfp) != -1)
+	{
+		int num = sscanf(linebuffer, "%99s 0x%x 0x%x %99s %99s %99s\n",
+		                 ip, &type, &flags, hwaddr, mask, iface);
+
+		// Skip header and empty lines
+		if (num < 4)
+			continue;
+
+		// Skip incomplete entires, i.e., entries without C (complete) flag
+		if(!(flags & 0x02))
+			continue;
+
+		// Get ID of this device in our network database. If it cannot be found, then this is a new device
+		// We match both IP *and* MAC address
+		// Same MAC, two IPs: Non-deterministic DHCP server, treat as two entries
+		// Same IP, two MACs: Either non-deterministic DHCP server or (almost) full DHCP address pool
+		// We can run this SELECT inside the currently active transaction as only the
+		// changed to the database are collected for latter commitment. Read-only access
+		// such as this SELECT command will be executed immediately on the database.
+		char* querystr = NULL;
+		int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
+		if(querystr == NULL || ret < 0)
+		{
+			logg("Memory allocation failed in parse_arp_cache (%i)", ret);
+			break;
+		}
+
+		// Perform SQL query
+		int dbID = db_query_int(querystr);
+		free(querystr);
+
+		if(dbID == DB_FAILED)
+		{
+			// SQLite error
+			break;
+		}
+
+		// If we reach this point, we can check if this client
+		// is known to pihole-FTL
+		// false = do not create a new record if the client is
+		//         unknown (only DNS requesting clients do this)
+		lock_shm();
+		int clientID = findClientID(ip, false);
+		unlock_shm();
+
+		// This client is known (by its IP address) to pihole-FTL if
+		// findClientID() returned a non-negative index
+		bool clientKnown = clientID >= 0;
+
+		// Get hostname of this client if the client is known
+		char *hostname = "";
+		if(clientKnown)
+		{
+			validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
+			hostname = getstr(clients[clientID].namepos);
+		}
+
+		// Device not in database, add new entry
+		if(dbID == DB_NODATA)
+		{
+			char* macVendor = getMACVendor(hwaddr);
+			dbquery("INSERT INTO network "\
+			        "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\
+			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\", \"%s\");",\
+			        ip, hwaddr, iface, now,
+			        clientKnown ? clients[clientID].lastQuery : 0L,
+			        clientKnown ? clients[clientID].numQueriesARP : 0u,
+			        hostname,
+			        macVendor);
+			free(macVendor);
+		}
+		// Device in database AND client known to Pi-hole
+		else if(clientKnown)
+		{
+			// Update lastQuery. Only use new value if larger
+			// clients[clientID].lastQuery may be zero if this
+			// client is only known from a database entry but has
+			// not been seen since then
+			dbquery("UPDATE network "\
+			        "SET lastQuery = MAX(lastQuery, %ld) "\
+			        "WHERE id = %i;",\
+			        clients[clientID].lastQuery, dbID);
+
+			// Update numQueries. Add queries seen since last update
+			// and reset counter afterwards
+			dbquery("UPDATE network "\
+			        "SET numQueries = numQueries + %u "\
+			        "WHERE id = %i;",\
+			        clients[clientID].numQueriesARP, dbID);
+			clients[clientID].numQueriesARP = 0;
+
+			// Store hostname if available
+			if(strlen(hostname) > 0)
+			{
+				// Store host name
+				dbquery("UPDATE network "\
+				        "SET name = \"%s\" "\
+				        "WHERE id = %i;",\
+				        hostname, dbID);
+			}
+		}
+		// else:
+		// Device in database but not known to Pi-hole: No action required
+
+		// Count number of processed ARP cache entries
+		entries++;
+	}
+
+	// Actually update the database
+	dbquery("COMMIT");
+
+	// Debug logging
+	if(config.debug & DEBUG_ARP) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER));
+
+	// Close file handle
+	fclose(arpfp);
+
+	// Close database connection
+	dbclose();
+}
+
+static char* getMACVendor(const char* hwaddr)
+{
+	struct stat st;
+	if(stat(FTLfiles.macvendordb, &st) != 0)
+	{
+		// File does not exist
+		if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb);
+		return strdup("");
+	}
+	else if(strlen(hwaddr) != 17)
+	{
+		// MAC address is incomplete
+		if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr));
+		return strdup("");
+	}
+
+	sqlite3 *macdb;
+	int rc = sqlite3_open_v2(FTLfiles.macvendordb, &macdb, SQLITE_OPEN_READONLY, NULL);
+	if( rc ){
+		logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
+		sqlite3_close(macdb);
+		return strdup("");
+	}
+
+	char *querystr = NULL;
+	// Only keep "XX:YY:ZZ" (8 characters)
+	char * hwaddrshort = strdup(hwaddr);
+	hwaddrshort[8] = '\0';
+	rc = asprintf(&querystr, "SELECT vendor FROM macvendor WHERE mac LIKE \"%s\";", hwaddrshort);
+	if(rc < 1)
+	{
+		logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc);
+		sqlite3_close(macdb);
+		return strdup("");
+	}
+	free(hwaddrshort);
+
+	sqlite3_stmt* stmt;
+	rc = sqlite3_prepare_v2(macdb, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb));
+		sqlite3_close(macdb);
+		return strdup("");
+	}
+	free(querystr);
+
+	char *vendor = NULL;
+	rc = sqlite3_step(stmt);
+	if(rc == SQLITE_ROW)
+	{
+		vendor = strdup((char*)sqlite3_column_text(stmt, 0));
+	}
+	else
+	{
+		// Not found
+		vendor = strdup("");
+	}
+
+	if(rc != SQLITE_DONE && rc != SQLITE_ROW)
+	{
+		// Error
+		logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
+	}
+
+	sqlite3_finalize(stmt);
+	sqlite3_close(macdb);
+
+	return vendor;
+}
+
+void updateMACVendorRecords()
+{
+	struct stat st;
+	if(stat(FTLfiles.macvendordb, &st) != 0)
+	{
+		// File does not exist
+		if(config.debug & DEBUG_ARP) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb);
+		return;
+	}
+
+	sqlite3 *db;
+	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
+	if( rc ){
+		logg("updateMACVendorRecords() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return;
+	}
+
+	sqlite3_stmt* stmt;
+	const char* querystr = "SELECT id,hwaddr FROM network;";
+	rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", querystr, rc, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return;
+	}
+
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		const int id = sqlite3_column_int(stmt, 0);
+		char* hwaddr = strdup((char*)sqlite3_column_text(stmt, 1));
+
+		// Get vendor for MAC
+		char* vendor = getMACVendor(hwaddr);
+		free(hwaddr);
+		hwaddr = NULL;
+
+		// Prepare UPDATE statement
+		char *querystr = NULL;
+		if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1)
+		{
+			logg("updateMACVendorRecords() - Allocation error 2");
+			free(vendor);
+			break;
+		}
+
+		// Execute prepared statement
+		char *zErrMsg = NULL;
+		rc = sqlite3_exec(db, querystr, NULL, NULL, &zErrMsg);
+		if( rc != SQLITE_OK ){
+			logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg);
+			sqlite3_free(zErrMsg);
+			free(querystr);
+			free(vendor);
+			break;
+		}
+
+		// Free allocated memory
+		free(querystr);
+		free(vendor);
+	}
+	if(rc != SQLITE_DONE)
+	{
+		// Error
+		logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+	}
+
+	sqlite3_finalize(stmt);
+	sqlite3_close(db);
+}

regex.c

@@ -40,7 +40,7 @@ static bool init_regex(const char *regexin, int index)
 	}
 
 	// Store compiled regex string in buffer if in regex debug mode
-	if(config.regex_debugmode)
+	if(config.debug & DEBUG_REGEX)
 	{
 		regexbuffer[index] = strdup(regexin);
 	}
@@ -98,8 +98,8 @@ bool match_regex(char *input)
 			matched = true;
 
 			// Print match message when in regex debug mode
-			if(config.regex_debugmode)
-				logg("DEBUG: Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input);
+			if(config.debug & DEBUG_REGEX)
+				logg("Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input);
 			break;
 		}
 		else if (errcode != REG_NOMATCH)
@@ -134,7 +134,7 @@ void free_regex(void)
 			regfree(&regex[index]);
 
 			// Also free buffered regex strings if in regex debug mode
-			if(config.regex_debugmode)
+			if(config.debug & DEBUG_REGEX)
 			{
 				free(regexbuffer[index]);
 				regexbuffer[index] = NULL;
@@ -245,7 +245,7 @@ void read_regex_from_file(void)
 	regexconfigured = calloc(num_regex, sizeof(bool));
 
 	// Buffer strings if in regex debug mode
-	if(config.regex_debugmode)
+	if(config.debug & DEBUG_REGEX)
 		regexbuffer = calloc(num_regex, sizeof(char*));
 
 	// Search through file

request.c

@@ -168,6 +168,12 @@ void process_request(char *client_message, int *sock)
 		read_regex_from_file();
 		unlock_shm();
 	}
+	else if(command(client_message, ">update-mac-vendor"))
+	{
+		processed = true;
+		logg("Received API request to update vendors in network table");
+		updateMACVendorRecords();
+	}
 
 	// Test only at the end if we want to quit or kill
 	// so things can be processed before

routines.h

@@ -29,7 +29,7 @@ void strtolower(char *str);
 int findOverTimeID(int overTimetimestamp);
 int findForwardID(const char * forward, bool count);
 int findDomainID(const char *domain);
-int findClientID(const char *client);
+int findClientID(const char *client, bool addNew);
 bool isValidIPv4(const char *addr);
 bool isValidIPv6(const char *addr);
 char *getDomainString(int queryID);
@@ -66,13 +66,16 @@ bool getSetupVarsBool(char * input);
 
 void parse_args(int argc, char* argv[]);
 
+// setupVars.c
 char* find_equals(const char* s);
+void trim_whitespace(char *string);
 
 // config.c
 void getLogFilePath(void);
 void read_FTLconf(void);
 void get_privacy_level(FILE *fp);
 void get_blocking_mode(FILE *fp);
+void read_debuging_settings(FILE *fp);
 
 // gc.c
 void *GC_thread(void *val);
@@ -83,6 +86,11 @@ void *DB_thread(void *val);
 int get_number_of_queries_in_DB(void);
 void save_to_DB(void);
 void read_data_from_DB(void);
+bool db_set_FTL_property(unsigned int ID, int value);
+bool dbquery(const char *format, ...);
+bool dbopen(void);
+void dbclose(void);
+int db_query_int(const char*);
 
 // memory.c
 void memory_check(int which);
@@ -126,3 +134,11 @@ void newOverTimeClient(int clientID);
  * This also updates `overTimeClientData`.
  */
 void addOverTimeClientSlot();
+
+// capabilities.c
+bool check_capabilities(void);
+
+// networktable.c
+bool create_network_table(void);
+void parse_arp_cache(void);
+void updateMACVendorRecords(void);

setupVars.c

@@ -38,6 +38,26 @@ char* find_equals(const char* s)
 	return (char*)s;
 }
 
+void trim_whitespace(char *string)
+{
+	// isspace(char*) man page:
+	// checks for white-space  characters. In the "C" and "POSIX"
+	// locales, these are: space, form-feed ('\f'), newline ('\n'),
+	// carriage return ('\r'), horizontal tab ('\t'), and vertical tab
+	// ('\v').
+	char *original = string, *modified = string;
+	// Trim any whitespace characters (see above) at the beginning by increasing the pointer address
+	while (isspace((unsigned char)*original))
+		original++;
+	// Copy the content of original into modified as long as there is something in original
+	while ((*modified = *original++) != '\0')
+		modified++;
+	// Trim any whitespace characters (see above) at the end of the string by overwriting it
+	// with the zero character (marking the end of a C string)
+	while (modified > string && isspace((unsigned char)*--modified))
+		*modified = '\0';
+}
+
 // This will hold the read string
 // in memory and will serve the space
 // we will point to in the rest of the

shmem.c

@@ -11,6 +11,9 @@
 #include "FTL.h"
 #include "shmem.h"
 
+/// The version of shared memory used
+#define SHARED_MEMORY_VERSION 1
+
 /// The name of the shared memory. Use this when connecting to the shared memory.
 #define SHARED_LOCK_NAME "/FTL-lock"
 #define SHARED_STRINGS_NAME "/FTL-strings"
@@ -20,6 +23,7 @@
 #define SHARED_QUERIES_NAME "/FTL-queries"
 #define SHARED_FORWARDED_NAME "/FTL-forwarded"
 #define SHARED_OVERTIME_NAME "/FTL-overTime"
+#define SHARED_SETTINGS_NAME "/FTL-settings"
 #define SHARED_OVERTIMECLIENT_PREFIX "/FTL-client-"
 
 /// The pointer in shared memory to the shared string buffer
@@ -31,6 +35,7 @@ static SharedMemory shm_clients = { 0 };
 static SharedMemory shm_queries = { 0 };
 static SharedMemory shm_forwarded = { 0 };
 static SharedMemory shm_overTime = { 0 };
+static SharedMemory shm_settings = { 0 };
 
 static SharedMemory *shm_overTimeClients = NULL;
 
@@ -54,7 +59,12 @@ unsigned long long addstr(const char *str)
 	// Get string length
 	size_t len = strlen(str);
 
-	if(debug) logg("Adding \"%s\" (len %i) to buffer at pos %u", str, len, next_pos);
+	// If this is an empty string, use the one at position zero
+	if(len == 0) {
+		return 0;
+	}
+
+	if(config.debug & DEBUG_SHMEM) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos);
 
 	// Reserve additional memory if necessary
 	size_t required_size = next_pos + len + 1;
@@ -157,11 +167,13 @@ void _lock_shm(const char* function, const int line, const char * file) {
 	// Signal that FTL is waiting for a lock
 	shmLock->waitingForLock = true;
 
-	if(debug) logg("Waiting for lock in %s() (%s:%i)", function, file, line);
+	if(config.debug & DEBUG_LOCKS)
+		logg("Waiting for lock in %s() (%s:%i)", function, file, line);
 
 	int result = pthread_mutex_lock(&shmLock->lock);
 
-	if(debug) logg("Obtained lock for %s() (%s:%i)", function, file, line);
+	if(config.debug & DEBUG_LOCKS)
+		logg("Obtained lock for %s() (%s:%i)", function, file, line);
 
 	// Turn off the waiting for lock signal to notify everyone who was
 	// deferring to FTL that they can jump in the lock queue.
@@ -180,7 +192,8 @@ void _lock_shm(const char* function, const int line, const char * file) {
 void _unlock_shm(const char* function, const int line, const char * file) {
 	int result = pthread_mutex_unlock(&shmLock->lock);
 
-	if(debug) logg("Removed lock in %s() (%s:%i)", function, file, line);
+	if(config.debug & DEBUG_LOCKS)
+		logg("Removed lock in %s() (%s:%i)", function, file, line);
 
 	if(result != 0)
 		logg("Failed to unlock SHM lock: %s", strerror(result));
@@ -257,6 +270,14 @@ bool init_shmem(void)
 	overTime = (overTimeDataStruct*)shm_overTime.ptr;
 	counters->overTime_MAX = pagesize;
 
+	/****************************** shared settings struct ******************************/
+	// Try to create shared memory object
+	shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings));
+	if(shm_settings.ptr == NULL)
+		return false;
+	ShmSettings *settings = (ShmSettings*)shm_settings.ptr;
+	settings->version = SHARED_MEMORY_VERSION;
+
 	return true;
 }
 
@@ -276,6 +297,7 @@ void destroy_shmem(void)
 	delete_shm(&shm_queries);
 	delete_shm(&shm_forwarded);
 	delete_shm(&shm_overTime);
+	delete_shm(&shm_settings);
 
 	// Don't use counters->clients because it's been freed
 	for(int i = 0; i < clientCount; i++) {
@@ -286,7 +308,8 @@ void destroy_shmem(void)
 
 SharedMemory create_shm(char *name, size_t size)
 {
-	if(debug) logg("Creating shared memory with name \"%s\" and size %zu", name, size);
+	if(config.debug & DEBUG_SHMEM)
+		logg("Creating shared memory with name \"%s\" and size %zu", name, size);
 
 	SharedMemory sharedMemory = {
 		.name = name,
@@ -394,7 +417,8 @@ void *enlarge_shmem_struct(char type)
 }
 
 bool realloc_shm(SharedMemory *sharedMemory, size_t size) {
-	logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size);
+	if(config.debug & DEBUG_SHMEM)
+		logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size);
 
 	int result = munmap(sharedMemory->ptr, sharedMemory->size);
 	if(result != 0)

socket.c

@@ -552,14 +552,14 @@ bool ipv6_available(void)
 			{
 				iface[addr->sa_family == AF_INET6 ? 1 : 0]++;
 
-				// For now unused debug statement
-				// logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4");
+				if(config.debug & DEBUG_NETWORKING)
+					logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4");
 			}
 		}
 		freeifaddrs(allInterfaces);
 	}
 
-	if(debug)
+	if(config.debug & DEBUG_NETWORKING)
 	{
 		logg("Found %i IPv4 and %i IPv6 capable interfaces", iface[0], iface[1]);
 	}