Merge remote-tracking branch 'origin/development' into new/API

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>

.travis.yml

@@ -70,7 +70,7 @@ before_script:
 - mkdir -p "${DEST_DIR}"
 - make clean
 script:
-- make pihole-FTL CC="${BUILD_CC}" CFLAGS="${CFLAGS}"
+- make CC="${BUILD_CC}" CFLAGS="${CFLAGS}"
 - file pihole-FTL
 - if [[ "${BIN_NAME}" == "pihole-FTL-linux-x86_64" ]]; then test/run.sh; fi
 - mv pihole-FTL "${DEST_DIR}/${BIN_NAME}"

FTL.h

@@ -41,6 +41,9 @@
 #include "sqlite3.h"
 // tolower()
 #include <ctype.h>
+// Unix socket
+#include <sys/un.h>
+
 
 #include "routines.h"
 
@@ -77,6 +80,7 @@ typedef struct {
 	const char* pid;
 	const char* port;
 	char* db;
+	const char* socketfile;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -111,10 +115,12 @@ typedef struct {
 	int overTime;
 	int IPv4;
 	int IPv6;
-	int PTR;
-	int SRV;
 	int wildcarddomains;
 	int forwardedqueries;
+	int reply_NODATA;
+	int reply_NXDOMAIN;
+	int reply_CNAME;
+	int reply_IP;
 } countersStruct;
 
 typedef struct {
@@ -142,6 +148,11 @@ typedef struct {
 	int forwardID;
 	bool valid;
 	bool db;
+	// the ID is a (signed) int in dnsmasq, so no need for a long int here
+	int id;
+	bool complete;
+	unsigned char reply;
+	int generation;
 } queriesDataStruct;
 
 typedef struct {
@@ -164,6 +175,7 @@ typedef struct {
 	int blockedcount;
 	char *domain;
 	bool wildcard;
+	unsigned char dnssec;
 } domainsDataStruct;
 
 typedef struct {
@@ -194,6 +206,7 @@ typedef struct {
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD, AUTHDATA };
 enum { SOCKET, API, APIH };
 enum { WHITELIST, BLACKLIST, WILDLIST };
+enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 
 logFileNamesStruct files;
 FTLFileNamesStruct FTLfiles;

Makefile

@@ -46,7 +46,7 @@ _DEPS = $(patsubst %,$(IDIR)/%,$(DEPS))
 
 _OBJ = $(patsubst %,$(ODIR)/%,$(OBJ))
 
-all: pihole-FTL
+all: pihole-FTL socket-test
 
 $(ODIR)/%.o: %.c $(_DEPS) | $(ODIR)
 	$(CC) -c -o $@ $< -g3 $(CCFLAGS)
@@ -60,6 +60,9 @@ $(ODIR)/sqlite3.o: sqlite3.c
 pihole-FTL: $(_OBJ) $(ODIR)/sqlite3.o
 	$(CC) -v $(CCFLAGS) -o $@ $^ $(LIBS)
 
+socket-test: socket_client.c
+	$(CC) -o $@ $< $(CCFLAGS)
+
 .PHONY: clean force install
 
 clean:

README.md

@@ -187,7 +187,7 @@ unique_clients 3
 2 10.0 5.6.7.8 some.other.dns.com
 ```
  Variant: `>forward-dest unsorted` to show forward destinations in unsorted order (equivalent to using `>forward-names`)
-```
+
 
 - `>querytypes` : get collected query types percentage
  ```

api.h

@@ -26,6 +26,7 @@ void getVersion(int *sock, char type);
 void getDBstats(int *sock, char type);
 void getClientsOverTime(int *sock);
 void getClientNames(int *sock);
+void getUnknownQueries(int *sock);
 
 // Endpoints under /dns/
 void getList(int *sock, char type, char list_type);

api_stats.c

@@ -845,28 +845,28 @@ void getAllQueries(char *client_message, int *sock, char type)
 			if(!privacymode)
 			{
 				if(strlen(clients[queries[i].clientID].name) > 0)
-					ssend(*sock,"%i %s %s %s %i\n",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].name,queries[i].status);
+					ssend(*sock,"%i %s %s %s %i %i\n",queries[i].timestamp,type,domains[queries[i].domainID].domain,clients[queries[i].clientID].name,queries[i].status,domains[queries[i].domainID].dnssec);
 				else
-					ssend(*sock,"%i %s %s %s %i\n",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].ip,queries[i].status);
+					ssend(*sock,"%i %s %s %s %i %i\n",queries[i].timestamp,type,domains[queries[i].domainID].domain,clients[queries[i].clientID].ip,queries[i].status,domains[queries[i].domainID].dnssec);
 			}
 			else
-				ssend(*sock,"%i %s %s hidden %i\n",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,queries[i].status);
+				ssend(*sock,"%i %s %s hidden %i %i\n",queries[i].timestamp,type,domains[queries[i].domainID].domain,queries[i].status,domains[queries[i].domainID].dnssec);
 		}
 		else
 		{
-			// {"data":[["1497351662","IPv4","clients4.google.com","10.8.0.2","2"],
+			// {"data":[["1497351662","IPv4","clients4.google.com","10.8.0.2",2,1],
 			if(!first) ssend(*sock, ",");
 			first = false;
 
 			if(!privacymode)
 			{
 				if(strlen(clients[queries[i].clientID].name) > 0)
-					ssend(*sock,"[%i,\"%s\",\"%s\",\"%s\",%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].name,queries[i].status);
+					ssend(*sock,"[%i,\"%s\",\"%s\",\"%s\",%i,%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].name,queries[i].status,domains[queries[i].domainID].dnssec);
 				else
-					ssend(*sock,"[%i,\"%s\",\"%s\",\"%s\",%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].ip,queries[i].status);
+					ssend(*sock,"[%i,\"%s\",\"%s\",\"%s\",%i,%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,clients[queries[i].clientID].ip,queries[i].status,domains[queries[i].domainID].dnssec);
 			}
 			else
-				ssend(*sock,"[%i,\"%s\",\"%s\",\"hidden\",%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,queries[i].status);
+				ssend(*sock,"[%i,\"%s\",\"%s\",\"hidden\",%i,%i]",queries[i].timestamp,qtype,domains[queries[i].domainID].domain,queries[i].status,domains[queries[i].domainID].dnssec);
 		}
 	}
 
@@ -1160,11 +1160,19 @@ void getQueryTypesOverTime(int *sock, char type)
 
 void getVersion(int *sock, char type)
 {
-	char version[] = GIT_VERSION;
+	const char * version = GIT_VERSION;
+	const char * branch = GIT_BRANCH;
+	// Travis CI pulls on a tag basis, not by branch.
+	// Hence, it may happen that the master binary isn't aware of its branch.
+	// We check if this is the case and if there is a "vX.YY" like tag on the
+	// binary are print out branch "master" if we find that this is the case
+	if(strstr(branch, "(no branch)") != NULL && strstr(version, ".") != NULL)
+		branch = "master";
+
 	if(strstr(version, ".") != NULL)
-		ssend(*sock,"version %s\ntag %s\nbranch %s\ndate %s\n", GIT_VERSION, GIT_TAG, GIT_BRANCH, GIT_DATE);
+		ssend(*sock,"version %s\ntag %s\nbranch %s\ndate %s\n", version, GIT_TAG, branch, GIT_DATE);
 	else
-		ssend(*sock,"version vDev-%s\ntag %s\nbranch %s\ndate %s\n", GIT_HASH, GIT_TAG, GIT_BRANCH, GIT_DATE);
+		ssend(*sock,"version vDev-%s\ntag %s\nbranch %s\ndate %s\n", GIT_HASH, GIT_TAG, branch, GIT_DATE);
 
 	if(debugclients)
 		logg("Sent version info to client, ID: %i", *sock);
@@ -1303,3 +1311,35 @@ void getClientNames(int *sock)
 	if(excludeclients != NULL)
 		clearSetupVarsArray();
 }
+
+void getUnknownQueries(int *sock)
+{
+	int i;
+	for(i=0; i < counters.queries; i++)
+	{
+		validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__);
+		// Check if this query has been removed due to garbage collection
+		if(queries[i].status != 0 && queries[i].complete) continue;
+
+		char type[5];
+		if(queries[i].type == 1)
+		{
+			strcpy(type,"IPv4");
+		}
+		else
+		{
+			strcpy(type,"IPv6");
+		}
+
+		validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+		validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__);
+
+		if(strlen(clients[queries[i].clientID].name) > 0)
+			ssend(*sock,"%i %i %i %s %s %s %i %s\n",queries[i].timestamp,i,queries[i].id,type,domains[queries[i].domainID].domain,clients[queries[i].clientID].name,queries[i].status,queries[i].complete ?"true":"false");
+		else
+			ssend(*sock,"%i %i %i %s %s %s %i %s\n",queries[i].timestamp,i,queries[i].id,type,domains[queries[i].domainID].domain,clients[queries[i].clientID].ip,queries[i].status,queries[i].complete?"true":"false");
+	}
+
+	if(debugclients)
+		logg("Sent unknown queries data to client, ID: %i", *sock);
+}

args.c

@@ -70,7 +70,7 @@ void parse_args(int argc, char* argv[])
 		if(strcmp(argv[i], "-v") == 0 ||
 		   strcmp(argv[i], "version") == 0)
 		{
-			char version[] = GIT_VERSION;
+			const char * version = GIT_VERSION;
 			// Check if version is of format vX.YY
 			// '.' can never be part of a commit hash
 			if(strstr(version, ".") != NULL)
@@ -90,7 +90,15 @@ void parse_args(int argc, char* argv[])
 		if(strcmp(argv[i], "-b") == 0 ||
 		   strcmp(argv[i], "branch") == 0)
 		{
-			printf("%s\n",GIT_BRANCH);
+			const char * branch = GIT_BRANCH;
+			const char * version = GIT_VERSION;
+			// Travis CI pulls on a tag basis, not by branch.
+			// Hence, it may happen that the master binary isn't aware of its branch.
+			// We check if this is the case and if there is a "vX.YY" like tag on the
+			// binary are print out branch "master" if we find that this is the case
+			if(strstr(branch, "(no branch)") != NULL && strstr(version, ".") != NULL)
+				branch = "master";
+			printf("%s\n",branch);
 			exit(EXIT_SUCCESS);
 		}
 
@@ -118,13 +126,13 @@ void parse_args(int argc, char* argv[])
 			FTLfiles.log = "pihole-FTL.log";
 			// FTLfiles.db will be set to "pihole-FTL.db" via config file on Travis
 			FTLfiles.conf = "pihole-FTL.conf";
+			FTLfiles.socketfile = "pihole-FTL.sock";
 			files.log = "pihole.log";
 			ok = true;
 		}
 
 		// List of implemented arguments
-		if(strcmp(argv[i], "-h") == 0 ||
-		   strcmp(argv[i], "help") == 0)
+		if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0)
 		{
 			printf("pihole-FTL - The Pi-hole FTL engine\n\n");
 			printf("Usage:    sudo service pihole-FTL <action>\n");

database.c

@@ -36,7 +36,12 @@ void check_database(int rc)
 
 void dbclose(void)
 {
-	sqlite3_close(db);
+	int rc = sqlite3_close(db);
+	// Report any error
+	if( rc )
+		logg("dbclose() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+
+	// Unlock mutex on the database
 	pthread_mutex_unlock(&dblock);
 }
 
@@ -169,6 +174,9 @@ void db_init(void)
 		}
 	}
 
+	// Close database to prevent having it opened all time
+	sqlite3_close(db);
+
 	if (pthread_mutex_init(&dblock, NULL) != 0)
 	{
 		logg("FATAL: DB mutex init failed\n");
@@ -309,14 +317,19 @@ void save_to_DB(void)
 		return;
 	}
 
+	int currenttimestamp = time(NULL);
 	for(i = lastdbindex; i < counters.queries; i++)
 	{
 		validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__);
-		if(queries[i].timestamp <= lasttimestamp || queries[i].db == true)
-		{
-			// Already in database
-			// logg("Skipping %i",i);
+		if(queries[i].timestamp <= lasttimestamp || queries[i].db)
+			// Already in database or not yet complete
 			continue;
+
+		if(!queries[i].complete && queries[i].timestamp > currenttimestamp-2)
+		{
+			// Break if a brand new query (age < 2 seconds) is not yet completed
+			// giving it a chance to be stored next time
+			break;
 		}
 
 		// Memory checks
@@ -385,8 +398,8 @@ void save_to_DB(void)
 
 	// Finish prepared statement
 	ret = dbquery("END TRANSACTION");
-	if(!ret){ dbclose(); return; }
-	sqlite3_finalize(stmt);
+	int ret2 = sqlite3_finalize(stmt);
+	if(!ret || ret2 != SQLITE_OK){ dbclose(); return; }
 
 	// Store index for next loop interation round and update last time stamp
 	// in the database only if all queries have been saved successfully

gc.c

@@ -69,6 +69,28 @@ void *GC_thread(void *val)
 					counters.forwardedqueries--;
 					validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__);
 					forwarded[queries[i].forwardID].count--;
+					// Maybe we have to adjust total counters depending on the reply type
+					switch(queries[i].reply)
+					{
+						case 1: // NODATA(-IPv6)
+						counters.reply_NODATA--;
+						break;
+
+						case 2: // NXDOMAIN
+						counters.reply_NXDOMAIN--;
+						break;
+
+						case 3: // <CNAME>
+						counters.reply_CNAME--;
+						break;
+
+						case 4: // valid IP
+						counters.reply_IP--;
+						break;
+
+						default: // Incomplete query, do nothing
+						break;
+					}
 					break;
 				case 3:
 					// Answered from local cache _or_ local config

grep.c

@@ -191,10 +191,38 @@ int countlineswith(const char* str, const char* fname)
 	}
 
 	// Search through file
-	// getline reads a string from the specified file up to either a newline character or EOF
+	// getline reads a string from the specified file up to either a
+	// newline character or EOF
 	while(getline(&buffer, &size, fp) != -1)
-		if((strstr(buffer, str)) != NULL)
+	{
+		// Strip potential newline character at the end of line we just read
+		if(buffer[strlen(buffer)-1] == '\n')
+			buffer[strlen(buffer)-1] = '\0';
+
+		// Search for exact match
+		if(strcmp(buffer, str) == 0)
+		{
 			found++;
+			continue;
+		}
+
+		// If line starts with *, search for partial match of
+		// needle "buffer+1" in haystack "str"
+		if(buffer[0] == '*')
+		{
+			char * buf = strstr(str, buffer+1);
+			// The  strstr() function finds the first occurrence of
+			// the substring buffer+1 in the string str.
+			// These functions return a pointer to the beginning of
+			// the located substring, or NULL if the substring is not
+			// found. Hence, we compare the length of the substring to
+			// the wildcard entry to rule out the possiblity that
+			// there is anything behind the wildcard. This avoids that given
+			// "*example.com" "example.com.xxxxx" would also match.
+			if(buf != NULL && strlen(buf) == strlen(buffer+1))
+				found++;
+		}
+	}
 
 	// Free allocated memory
 	if(buffer != NULL)

log.c

@@ -135,10 +135,13 @@ void log_counter_info(void)
 {
 	logg(" -> Total DNS queries: %i", counters.queries);
 	logg(" -> Cached DNS queries: %i", counters.cached);
-	logg(" -> Blocked DNS queries: %i", counters.blocked);
+	logg(" -> Forwarded DNS queries: %i", counters.forwardedqueries);
+	logg(" -> Exactly blocked DNS queries: %i", counters.blocked);
+	logg(" -> Wildcard blocked DNS queries: %i", counters.wildcardblocked);
 	logg(" -> Unknown DNS queries: %i", counters.unknown);
 	logg(" -> Unique domains: %i", counters.domains);
 	logg(" -> Unique clients: %i", counters.clients);
+	logg(" -> Known forward destinations: %i", counters.forwarded);
 }
 
 void log_FTL_version(void)

main.c

@@ -72,11 +72,19 @@ int main (int argc, char* argv[]) {
 	}
 	sleepms(100);
 
+	// Start TELNET thread
+	pthread_t telnet_listenthread;
+	if(pthread_create( &telnet_listenthread, &attr, telnet_listening_thread, NULL ) != 0)
+	{
+		logg("Unable to open telnet listening thread. Exiting...");
+		killed = 1;
+	}
+
 	// Start SOCKET thread
 	pthread_t socket_listenthread;
 	if(pthread_create( &socket_listenthread, &attr, socket_listening_thread, NULL ) != 0)
 	{
-		logg("Unable to open socket listening thread. Exiting...");
+		logg("Unable to open Unix socket listening thread. Exiting...");
 		killed = 1;
 	}
 	sleepms(100);
@@ -157,15 +165,18 @@ int main (int argc, char* argv[]) {
 			// Have to re-read gravity files
 			rereadgravity = false;
 			read_gravity_files();
+			log_counter_info();
 			disable_thread_lock("pihole_main_thread");
 		}
 	}
 
-
 	logg("Shutting down...");
 	pthread_cancel(piholelogthread);
+	pthread_cancel(telnet_listenthread);
 	pthread_cancel(socket_listenthread);
-	close_socket(SOCKET);
+	close_telnet_socket();
+	close_unix_socket();
+	close_api_socket();
 	removepid();
 	logg("########## FTL terminated! ##########");
 	return 1;

parser.c

@@ -23,6 +23,7 @@ long int lastpos = 0;
 int lastqueryID = 0;
 bool flush = false;
 char timestamp[16] = "";
+int loggeneration = 0;
 
 void initial_log_parsing(void)
 {
@@ -49,6 +50,45 @@ long int checkLogForChanges(void)
 	return difference;
 }
 
+int getTimeIndex(char * readbuffer)
+{
+	int querytimestamp, overTimetimestamp, timeidx = -1, i;
+	extracttimestamp(readbuffer, &querytimestamp, &overTimetimestamp);
+
+	bool found = false;
+	// Check struct size
+	memory_check(OVERTIME);
+	for(i=0; i < counters.overTime; i++)
+	{
+		validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__);
+		if(overTime[i].timestamp == overTimetimestamp)
+		{
+			found = true;
+			timeidx = i;
+			break;
+		}
+	}
+	if(!found)
+	{
+		timeidx = counters.overTime;
+		validate_access("overTime", timeidx, false, __LINE__, __FUNCTION__, __FILE__);
+		overTime[timeidx].magic = MAGICBYTE;
+		overTime[timeidx].timestamp = overTimetimestamp;
+		overTime[timeidx].total = 0;
+		overTime[timeidx].blocked = 0;
+		overTime[timeidx].cached = 0;
+		overTime[timeidx].forwardnum = 0;
+		overTime[timeidx].forwarddata = NULL;
+		overTime[timeidx].querytypedata = calloc(2, sizeof(int));
+		overTime[timeidx].clientnum = 0;
+		overTime[timeidx].clientdata = NULL;
+		memory.querytypedata += 2*sizeof(int);
+		counters.overTime++;
+	}
+
+	return timeidx;
+}
+
 void open_pihole_log(void)
 {
 	FILE * fp;
@@ -144,12 +184,47 @@ void *pihole_log_thread(void *val)
 	return NULL;
 }
 
+int getID(char * readbuffer)
+{
+	// Get query ID from a string like
+	// "Dec 20 21:16:22 dnsmasq[19372]: 4 10.8.0.2/34596 query[A] pi.hole from 10.8.0.2"
+	int dnsmasqID = -1;
+	if(!sscanf(readbuffer, "%*[^]]]: %i", &dnsmasqID))
+	{
+		if(debug) logg("Error getting ID for query \"%s\" %u", readbuffer, dnsmasqID);
+	}
+	return dnsmasqID;
+}
+
+bool checkQuery(char * readbuffer, const char * type)
+{
+	// Check if this domain names contains only printable characters
+	// if not: skip analysis of this log line
+	if(strstr(readbuffer,"<name unprintable>") != NULL)
+	{
+		if(debug) logg("Ignoring <name unprintable> domain (%s)", type);
+		return false;
+	}
+
+	// Check if this domain name contains quotes
+	if(strstr(readbuffer, "\"") != NULL)
+	{
+		if(debug) logg("Ignoring \" domain (%s)", type);
+		return false;
+	}
+
+	// Check if this is a PTR query (we don't analyze them)
+	if(strstr(readbuffer,"in-addr.arpa") != NULL)
+		return false;
+
+	return true;
+}
+
 void process_pihole_log(int file)
 {
 	int i;
 	char *readbuffer = NULL;
-	char *readbuffer2 = NULL;
-	size_t size1 = 0, size2 = 0;
+	size_t size1 = 0;
 	FILE *fp;
 
 	if(file == 0)
@@ -200,23 +275,12 @@ void process_pihole_log(int file)
 		}
 
 		// Test if the read line is a query line
-		if(strstr(readbuffer,"]: query[A") != NULL)
+		if(strstr(readbuffer," query[A") != NULL)
 		{
-			// Check if this domain names contains only printable characters
-			// if not: skip analysis of this log line
-			if(strstr(readbuffer,"<name unprintable>") != NULL)
-			{
-				if(debug) logg("Ignoring <name unprintable> domain (query)");
+			if(!checkQuery(readbuffer, "query"))
 				continue;
-			}
 
-			if(strstr(readbuffer, "\"") != NULL)
-			{
-				if(debug) logg("Ignoring \" domain (query)");
-				continue;
-			}
-
-			if(!config.analyze_AAAA && strstr(readbuffer,"]: query[AAAA]") != NULL)
+			if(!config.analyze_AAAA && strstr(readbuffer," query[AAAA]") != NULL)
 			{
 				if(debug) logg("Not analyzing AAAA query");
 				continue;
@@ -304,6 +368,12 @@ void process_pihole_log(int file)
 				continue;
 			}
 
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
 			// Get domain
 			// domainstart = pointer to | in "query[AAAA] |host.name from ww.xx.yy.zz\n"
 			const char *domainstart = strstr(readbuffer, "] ");
@@ -333,11 +403,8 @@ void process_pihole_log(int file)
 			}
 
 			char *domain = calloc(domainlen+1,sizeof(char));
-			char *domainwithspaces = calloc(domainlen+3,sizeof(char));
 			// strncat() NULL-terminates the copied string (strncpy() doesn't!)
 			strncat(domain,domainstart+2,domainlen);
-			// Copy string into buffer surrounded by spaces
-			sprintf(domainwithspaces," %s ",domain);
 			// Convert domain to lower case
 			strtolower(domain);
 
@@ -346,7 +413,6 @@ void process_pihole_log(int file)
 				// domain is "pi.hole", skip this query
 				// free memory already allocated here
 				free(domain);
-				free(domainwithspaces);
 				continue;
 			}
 
@@ -359,7 +425,6 @@ void process_pihole_log(int file)
 				logg("Notice: Skipping malformated log line (client end missing): %s", readbuffer);
 				// Skip this line, free memory already allocated here
 				free(domain);
-				free(domainwithspaces);
 				continue;
 			}
 
@@ -369,7 +434,6 @@ void process_pihole_log(int file)
 				logg("Notice: Skipping malformated log line (client length < 1): %s", readbuffer);
 				// Skip this line, free memory already allocated here
 				free(domain);
-				free(domainwithspaces);
 				continue;
 			}
 
@@ -396,95 +460,6 @@ void process_pihole_log(int file)
 				overTime[timeidx].querytypedata[1]++;
 			}
 
-			// Save current file pointer position
-			long int fpos = ftell(fp);
-			unsigned char status = 0;
-
-			// Try to find either a matching
-			// - "gravity.list" + domain
-			// - "forwarded" + domain
-			// - "cached" + domain
-			// - "black.list" + domain
-			// in the following up to 200 lines
-			bool firsttime = true;
-			int forwardID = -1;
-			for(i=0; i<200; i++)
-			{
-				if(getline(&readbuffer2, &size2, fp) != -1)
-				{
-					// Process only matching lines
-					if(strstr(readbuffer2, domainwithspaces) != NULL)
-					{
-						// Blocked by gravity.list ?
-						if(strstr(readbuffer2,"gravity.list ") != NULL)
-						{
-							status = 1;
-							break;
-						}
-						// Forwarded to upstream server?
-						else if(strstr(readbuffer2,": forwarded ") != NULL)
-						{
-							status = 2;
-							// Get ID of forward destination, create new forward destination record
-							// if not found in current data structure
-							forwardID = getforwardID(readbuffer2, false);
-							if(forwardID == -2)
-								continue;
-							break;
-						}
-						// Answered by local cache?
-						else if((strstr(readbuffer2,"cached ") != NULL) ||
-						        (strstr(readbuffer2,"local.list") != NULL) ||
-						        (strstr(readbuffer2,"hostname.list") != NULL) ||
-						        (strstr(readbuffer2,"DHCP ") != NULL) ||
-						        (strstr(readbuffer2,"/etc/hosts") != NULL))
-						{
-							status = 3;
-							break;
-						}
-						// wildcard blocking?
-						else if((strstr(readbuffer2,"config ") != NULL))
-						{
-							status = detectStatus(domain);
-							break;
-						}
-						// Blocked by black.list ?
-						else if(strstr(readbuffer2,"black.list ") != NULL)
-						{
-							status = 5;
-							break;
-						}
-					}
-				}
-				else
-				{
-					if(firsttime)
-					{
-						// Reached EOF without finding the action
-						// wait 100msec and try again to read dnsmasq's response
-						i = 0;
-						fseek(fp, fpos, SEEK_SET);
-						firsttime = false;
-						sleepms(100);
-					}
-					else
-					{
-						// Failed second time
-						break;
-					}
-				}
-			}
-
-			// Return to previous file pointer position
-			fseek(fp, fpos, SEEK_SET);
-
-			// Free memory allocated by readline
-			if(readbuffer2 != NULL)
-			{
-				free(readbuffer2);
-				readbuffer2 = NULL;
-			}
-
 			// Go through already knows domains and see if it is one of them
 			// Check struct size
 			memory_check(DOMAINS);
@@ -496,7 +471,7 @@ void process_pihole_log(int file)
 				domainID = counters.domains;
 				// // Debug output
 				if(debug)
-					logg("New domain: %s (%i - %i/%i)", domain, status, domainID, counters.domains_MAX);
+					logg("New domain: %s (%i: %i/%i)", domain, dnsmasqID, domainID, counters.domains_MAX);
 				validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__);
 				// Set magic byte
 				domains[domainID].magic = MAGICBYTE;
@@ -509,6 +484,8 @@ void process_pihole_log(int file)
 				// Store domain name
 				domains[domainID].domain = strdup(domain);
 				memory.domainnames += (strlen(domain) + 1) * sizeof(char);
+				// Store DNSSEC result for this domain
+				domains[domainID].dnssec = DNSSEC_UNSPECIFIED;
 				// Increase counter by one
 				counters.domains++;
 			}
@@ -529,10 +506,10 @@ void process_pihole_log(int file)
 				{
 					// Convert hostname to lower case
 					strtolower(hostname);
-					logg("New client: %s %s (%i/%i)", client, hostname, clientID, counters.clients_MAX);
+					logg("New client: %s %s (%i: %i/%i)", client, hostname, dnsmasqID, clientID, counters.clients_MAX);
 				}
 				else
-					logg("New client: %s (%i/%i)", client, clientID, counters.clients_MAX);
+					logg("New client: %s (%i: %i/%i)", client, dnsmasqID, clientID, counters.clients_MAX);
 
 				validate_access("clients", clientID, false, __LINE__, __FUNCTION__, __FILE__);
 				// Set magic byte
@@ -555,69 +532,27 @@ void process_pihole_log(int file)
 			queries[queryID].magic = MAGICBYTE;
 			queries[queryID].timestamp = querytimestamp;
 			queries[queryID].type = type;
-			queries[queryID].status = status;
+			queries[queryID].status = 0;
 			queries[queryID].domainID = domainID;
 			queries[queryID].clientID = clientID;
 			queries[queryID].timeidx = timeidx;
-			queries[queryID].forwardID = forwardID;
 			queries[queryID].valid = true;
 			queries[queryID].db = false;
+			queries[queryID].id = dnsmasqID;
+			queries[queryID].complete = false;
+			queries[queryID].reply = 0;
+			queries[queryID].generation = loggeneration;
 
 			// Increase DNS queries counter
 			counters.queries++;
+			// Count this query as unknown as long as no reply has
+			// been found and analyzed
+			counters.unknown++;
 
 			// Update overTime data
 			validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
 			overTime[timeidx].total++;
 
-			// Decide what to increment depending on status
-			switch(status)
-			{
-				case 0:
-					// Unknown (?)
-					counters.unknown++;
-					break;
-				case 1:
-					// Blocked by Pi-hole's blocking lists
-					counters.blocked++;
-					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
-					overTime[timeidx].blocked++;
-					validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
-					domains[domainID].blockedcount++;
-					break;
-				case 2:
-					// Forwarded to an upstream DNS server
-					counters.forwardedqueries++;
-					break;
-				case 3:
-					// Answered from local cache _or_ local config
-					counters.cached++;
-					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
-					overTime[timeidx].cached++;
-					break;
-				case 4:
-					// Blocked due to a matching wildcard rule
-					counters.wildcardblocked++;
-					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
-					overTime[timeidx].blocked++;
-					validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
-					domains[domainID].blockedcount++;
-					domains[domainID].wildcard = true;
-					break;
-				case 5:
-					// Blocked by user's black list
-					counters.blocked++;
-					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
-					overTime[timeidx].blocked++;
-					validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
-					domains[domainID].blockedcount++;
-					break;
-				default:
-					/* That cannot happen */
-					logg("Found unexpected status %i",status);
-					break;
-			}
-
 			// Determine if there is enough space for saving the current
 			// clientID in the overTime data structure, allocate space otherwise
 			validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
@@ -642,63 +577,101 @@ void process_pihole_log(int file)
 			// Free allocated memory
 			free(client);
 			free(domain);
-			free(domainwithspaces);
 		}
-		else if(strstr(readbuffer,": forwarded") != NULL)
+		// is this a "gravity.list" line?
+		else if(strstr(readbuffer,"/gravity.list ") != NULL && strstr(readbuffer," is ") != NULL)
 		{
-			// Check if this domain names contains only printable characters
-			// if not: skip analysis of this log line
-			if(strstr(readbuffer,"<name unprintable>") != NULL)
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "gravity.list"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Save status in corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
 			{
-				if(debug) logg("Ignoring <name unprintable> domain (forwarded)");
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					queries[i].status = 1;
+					found = true;
+					break;
+				}
+			}
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
 				continue;
 			}
 
-			// Check if this is a PTR query
-			// if so: skip analysis of this log line
-			if(strstr(readbuffer,"in-addr.arpa") != NULL)
+			if(!queries[i].complete)
+			{
+				// This query is no longer unknown
+				counters.unknown--;
+				// ... but got blocked
+				counters.blocked++;
+				// Hereby, this query is now fully determined
+				queries[i].complete = true;
+
+				// Get time index
+				int timeidx = getTimeIndex(readbuffer);
+				validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+				overTime[timeidx].blocked++;
+				validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+				domains[queries[i].domainID].blockedcount++;
+			}
+		}
+		// is this a "forwarded" line?
+		else if(strstr(readbuffer," forwarded ") != NULL && strstr(readbuffer," to ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "forwarded"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
 				continue;
 
 			// Get ID of forward destination, create new forward destination record
 			// if not found in current data structure
 			int forwardID = getforwardID(readbuffer, true);
 			if(forwardID == -2)
-				continue;
-
-			// Get timestamp
-			int querytimestamp, overTimetimestamp, timeidx = -1, i;
-			extracttimestamp(readbuffer, &querytimestamp, &overTimetimestamp);
-
-			bool found = false;
-			// Check struct size
-			memory_check(OVERTIME);
-			for(i=0; i < counters.overTime; i++)
 			{
-				validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__);
-				if(overTime[i].timestamp == overTimetimestamp)
+				if(debug) logg("Skipping malformated forwarded line");
+				continue;
+			}
+
+			// Save status and forwardID in corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
 				{
+					queries[i].status = 2;
+					queries[i].forwardID = forwardID;
 					found = true;
-					timeidx = i;
 					break;
 				}
 			}
 			if(!found)
 			{
-				timeidx = counters.overTime;
-				validate_access("overTime", timeidx, false, __LINE__, __FUNCTION__, __FILE__);
-				overTime[timeidx].magic = MAGICBYTE;
-				overTime[timeidx].timestamp = overTimetimestamp;
-				overTime[timeidx].total = 0;
-				overTime[timeidx].blocked = 0;
-				overTime[timeidx].cached = 0;
-				overTime[timeidx].forwardnum = 0;
-				overTime[timeidx].forwarddata = NULL;
-				overTime[timeidx].querytypedata = calloc(2, sizeof(int));
-				overTime[timeidx].clientnum = 0;
-				overTime[timeidx].clientdata = NULL;
-				memory.querytypedata += 2*sizeof(int);
-				counters.overTime++;
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
 			}
+
+			// Get time index
+			int timeidx = getTimeIndex(readbuffer);
+
 			// Determine if there is enough space for saving the current
 			// forwardID in the overTime data structure, allocate space otherwise
 			validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
@@ -707,9 +680,10 @@ void process_pihole_log(int file)
 				// Reallocate more space for forwarddata
 				overTime[timeidx].forwarddata = realloc(overTime[timeidx].forwarddata, (forwardID+1)*sizeof(*overTime[timeidx].forwarddata));
 				// Initialize new data fields with zeroes
-				for(i = overTime[timeidx].forwardnum; i <= forwardID; i++)
+				int j;
+				for(j = overTime[timeidx].forwardnum; j <= forwardID; j++)
 				{
-					overTime[timeidx].forwarddata[i] = 0;
+					overTime[timeidx].forwarddata[j] = 0;
 					memory.forwarddata++;
 				}
 				// Update counter
@@ -719,7 +693,321 @@ void process_pihole_log(int file)
 			// Update overTime data structure with the new forwarder
 			validate_access_oTfd(timeidx, forwardID, __LINE__, __FUNCTION__, __FILE__);
 			overTime[timeidx].forwarddata[forwardID]++;
+
+			if(!queries[i].complete)
+			{
+				// This query is no longer unknown ...
+				counters.unknown--;
+				// ... but got forwarded
+				counters.forwardedqueries++;
+				// Hereby, this query is now fully determined
+				queries[i].complete = true;
+			}
 		}
+		// is this a "cached" line?
+		else if((strstr(readbuffer," cached ") != NULL || \
+			     strstr(readbuffer,"/local.list ") != NULL || \
+			     strstr(readbuffer,"/hostname.list ") != NULL || \
+			     strstr(readbuffer," DHCP ") != NULL || \
+			     strstr(readbuffer," /etc/hosts ") != NULL) \
+			  && strstr(readbuffer," is ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "cached"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Save status in corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					queries[i].status = 3;
+					found = true;
+					break;
+				}
+			}
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
+			}
+
+			if(!queries[i].complete)
+			{
+				// This query is no longer unknown ...
+				counters.unknown--;
+				// ... but answered from local cache _or_ local config
+				counters.cached++;
+				// Hereby, this query is now fully determined
+				queries[i].complete = true;
+
+				// Get time index
+				int timeidx = getTimeIndex(readbuffer);
+				validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+				overTime[timeidx].cached++;
+			}
+		}
+		// is this a "wildcard" line?
+		else if(strstr(readbuffer," config ") != NULL && strstr(readbuffer," is ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "config"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Save status in corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					queries[i].status = detectStatus(domains[queries[i].domainID].domain);
+					found = true;
+					break;
+				}
+			}
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
+			}
+
+			if(!queries[i].complete)
+			{
+				// This query is no longer unknown
+				counters.unknown--;
+				// Hereby, this query is now fully determined
+				queries[i].complete = true;
+
+				// Get time index
+				int timeidx = getTimeIndex(readbuffer);
+
+				// Decide what to do depening on the result of detectStatus()
+				if(queries[i].status == 4)
+				{
+					// Blocked due to a matching wildcard rule
+					counters.wildcardblocked++;
+
+					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+					overTime[timeidx].blocked++;
+					validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+					domains[queries[i].domainID].blockedcount++;
+					domains[queries[i].domainID].wildcard = true;
+				}
+				else
+				{
+					// Answered from a custom (user provided) cache file
+					counters.cached++;
+
+					validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+					overTime[timeidx].cached++;
+				}
+			}
+		}
+
+		// is this a "black.list" line?
+		else if(strstr(readbuffer,"/black.list ") != NULL && strstr(readbuffer," is ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "black.list"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Save status in corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					queries[i].status = 5;
+					found = true;
+					break;
+				}
+			}
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
+			}
+
+			if(!queries[i].complete)
+			{
+				// This query is no longer unknown
+				counters.unknown--;
+				// ... but got blocked by user's black list
+				counters.blocked++;
+				// Hereby, this query is now fully determined
+				queries[i].complete = true;
+
+				// Get time index
+				int timeidx = getTimeIndex(readbuffer);
+				validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+				overTime[timeidx].blocked++;
+				validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+				domains[queries[i].domainID].blockedcount++;
+			}
+		}
+		// Is dnsmasq restarted? If yes, then we have to increment the log generation
+		// as the following UUIDs will again start at zero...
+		else if(strstr(readbuffer,"IPv6") != NULL &&
+		        strstr(readbuffer,"DBus") != NULL &&
+		        strstr(readbuffer,"i18n") != NULL &&
+		        strstr(readbuffer,"DHCP") != NULL)
+		{
+			loggeneration++;
+			if(debug) logg("Detected restart of dnsmasq, increasing loggeneration to %i", loggeneration);
+		}
+
+		// is this a "reply" line?
+		else if(strstr(readbuffer," reply ") != NULL && strstr(readbuffer," is ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "reply"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Search for corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					found = true;
+					break;
+				}
+			}
+
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
+			}
+
+			if(queries[i].reply != 0)
+			{
+				// Skip this entry if we already found the first result for this query before
+				continue;
+			}
+
+			// Iterate through possible values
+			if(strstr(readbuffer," is NODATA") != NULL)
+			{
+				// NODATA(-IPv6)
+				queries[i].reply = 1;
+				counters.reply_NODATA++;
+			}
+			else if(strstr(readbuffer," is NXDOMAIN") != NULL)
+			{
+				// NXDOMAIN
+				queries[i].reply = 2;
+				counters.reply_NXDOMAIN++;
+			}
+			else if(strstr(readbuffer," is <CNAME>") != NULL)
+			{
+				// <CNAME>
+				queries[i].reply = 3;
+				counters.reply_CNAME++;
+			}
+			else
+			{
+				// Valid IP
+				queries[i].reply = 4;
+				counters.reply_IP++;
+				// const char * dest = strstr(readbuffer," is ");
+				// char * result;
+				// sscanf(dest, " is %ms", &result);
+				// printf("reply is IP: %s\n",result);
+				// free(result);
+			}
+		}
+		// is this a "validaton" line? -- DNSSEC
+		else if(strstr(readbuffer," validation ") != NULL && strstr(readbuffer," is ") != NULL)
+		{
+			// Check query for invalid characters
+			if(!checkQuery(readbuffer, "DNSSEC"))
+				continue;
+
+			// Get dnsmasq's ID for this transaction
+			int dnsmasqID = getID(readbuffer);
+			// Skip invalid lines
+			if(dnsmasqID < 0)
+				continue;
+
+			// Search for corresponding query indentified by dnsmasq's ID
+			bool found = false;
+			for(i=0; i<counters.queries; i++)
+			{
+				// Check both UUID and generation of this query
+				if(queries[i].id == dnsmasqID && queries[i].generation == loggeneration)
+				{
+					found = true;
+					break;
+				}
+			}
+
+			if(!found)
+			{
+				// This may happen e.g. if the original query was a PTR query or "pi.hole"
+				// as we ignore them altogether
+				continue;
+			}
+
+			validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+			// Iterate through possible values
+			if(strstr(readbuffer,"is SECURE") != NULL)
+			{
+				domains[queries[i].domainID].dnssec = DNSSEC_SECURE;
+			}
+			else if(strstr(readbuffer,"is INSECURE") != NULL)
+			{
+				domains[queries[i].domainID].dnssec = DNSSEC_INSECURE;
+			}
+			else if(strstr(readbuffer,"is BOGUS") != NULL)
+			{
+				domains[queries[i].domainID].dnssec = DNSSEC_BOGUS;
+			}
+			else if(strstr(readbuffer,"is ABANDONED") != NULL)
+			{
+				domains[queries[i].domainID].dnssec = DNSSEC_ABANDONED;
+			}
+			else
+			{
+				// Unknown
+				domains[queries[i].domainID].dnssec = DNSSEC_UNKNOWN;
+				if(debug) logg("Unknown DNSSEC reply: %i\n\"%s\"",dnsmasqID,readbuffer);
+			}
+		}
+
 
 		// Save file pointer position, because we might have to repeat
 		// reading the next line if dnsmasq hasn't finished writing it

request.c

@@ -104,6 +104,11 @@ void process_socket_request(char *client_message, int *sock)
 		processed = true;
 		getClientNames(sock);
 	}
+	else if(command(client_message, ">unknown"))
+	{
+		processed = true;
+		getUnknownQueries(sock);
+	}
 
 	// Test only at the end if we want to quit or kill
 	// so things can be processed before

routines.h

@@ -38,9 +38,12 @@ void pihole_log_flushed(bool message);
 
 void memory_check(int which);
 
-void close_socket(char type);
+void close_telnet_socket(void);
+void close_unix_socket(void);
+void close_api_socket(void);
 void seom(int sock);
 void ssend(int sock, const char *format, ...);
+void *telnet_listening_thread(void *args);
 void *socket_listening_thread(void *args);
 void *api_listening_thread(void *args);
 

socket.c

@@ -12,7 +12,7 @@
 
 // The backlog argument defines the maximum length
 // to which the queue of pending connections for
-// socketfd may grow. If a connection request arrives
+// telnetfd may grow. If a connection request arrives
 // when the queue is full, the client may receive an
 // error with an indication of ECONNREFUSED or, if
 // the underlying protocol supports retransmission,
@@ -21,7 +21,7 @@
 #define BACKLOG 5
 
 // File descriptors
-int socketfd, apifd;
+int telnetfd, socketfd, apifd;
 
 void saveport(int port)
 {
@@ -38,13 +38,13 @@ void saveport(int port)
 	}
 }
 
-void bind_to_port(char type, int *socketdescriptor)
+void bind_to_telnet_port(char type, int *socketdescriptor)
 {
 	*socketdescriptor = socket(AF_INET, SOCK_STREAM, 0);
 
 	if(*socketdescriptor < 0)
 	{
-		logg("Error opening socket");
+		logg("Error opening telnet socket");
 		exit(EXIT_FAILURE);
 	}
 
@@ -133,6 +133,43 @@ void bind_to_port(char type, int *socketdescriptor)
 	}
 }
 
+
+void bind_to_unix_socket(int *socketdescriptor)
+{
+	*socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0);
+
+	if(*socketdescriptor < 0)
+	{
+		logg("Error opening unix socket");
+		exit(EXIT_FAILURE);
+	}
+
+	// Make sure unix socket file handle does not exist, if it exists, remove it
+	unlink(FTLfiles.socketfile);
+
+	struct sockaddr_un address;
+	address.sun_family = AF_LOCAL;
+	strcpy(address.sun_path, FTLfiles.socketfile);
+
+	// Bild to Unix socket handle
+	errno = 0;
+	if(bind(*socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) {
+		logg("Error on binding on unix socket %s", FTLfiles.socketfile);
+		logg("Reason: %s (%i)", strerror(errno), errno);
+		exit(EXIT_FAILURE);
+	}
+
+	// The listen system call allows the process to listen on the Unix socket for connections
+	if(listen(*socketdescriptor, BACKLOG) == -1)
+	{
+		logg("Error on listening");
+		exit(EXIT_FAILURE);
+	}
+
+	logg("Listening on Unix socket");
+}
+
+
 // Called from main() at graceful shutdown
 void removeport(void)
 {
@@ -191,25 +228,84 @@ int listener(int sockfd)
 	return clientsocket;
 }
 
-void close_socket(char type)
+void close_telnet_socket(void)
 {
-	switch(type)
-	{
-		case SOCKET:
-			removeport();
-			// Using global variable here
-			close(socketfd);
-			break;
-		case API:
-			close(apifd);
-			break;
-		default:
-			logg("Incompatible socket type %i, cannot close",(int)type);
-			exit(EXIT_FAILURE);
-			break;
-	}
+	removeport();
+	// Using global variable here
+	close(telnetfd);
 }
 
+void close_unix_socket(void)
+{
+	// The process has to take care of unlinking the socket file description on exit
+	unlink(FTLfiles.socketfile);
+	// Using global variable here
+	close(socketfd);
+}
+
+void close_api_socket(void)
+{
+	close(apifd);
+}
+
+void *telnet_connection_handler_thread(void *socket_desc)
+{
+	//Get the socket descriptor
+	int sock = *(int*)socket_desc;
+	// Store copy only for displaying the debug messages
+	int sockID = sock;
+	char client_message[SOCKETBUFFERLEN] = "";
+
+	// Set thread name
+	char threadname[16];
+	sprintf(threadname,"telnet-%i",sockID);
+	prctl(PR_SET_NAME,threadname,0,0,0);
+	//Receive from client
+	ssize_t n;
+	while((n = recv(sock,client_message,SOCKETBUFFERLEN-1, 0)))
+	{
+		if (n > 0)
+		{
+			char *message = calloc(strlen(client_message)+1,sizeof(char));
+			strcpy(message, client_message);
+
+			// Clear client message receive buffer
+			memset(client_message, 0, sizeof client_message);
+
+			// Lock FTL data structure, since it is likely that it will be changed here
+			// Requests should not be processed/answered when data is about to change
+			enable_thread_lock(threadname);
+
+			process_socket_request(message, &sock);
+			free(message);
+
+			// Release thread lock
+			disable_thread_lock(threadname);
+
+			if(sock == 0)
+			{
+				// Client disconnected by sending EOT or ">quit"
+				break;
+			}
+		}
+		else if(n == -1)
+		{
+			if(debugclients) logg("Telnet connection interrupted (%s), ID: %i", strerror(errno), sockID);
+			break;
+		}
+	}
+	if(debugclients)
+		logg("Telnet disconnected, ID: %i", sockID);
+
+	//Free the socket pointer
+	if(sock != 0)
+		close(sock);
+	free(socket_desc);
+
+	return 0;
+}
+
+
 void *socket_connection_handler_thread(void *socket_desc)
 {
 	//Get the socket descriptor
@@ -246,18 +342,17 @@ void *socket_connection_handler_thread(void *socket_desc)
 
 			if(sock == 0)
 			{
-				// Client disconnected by sending EOT or ">quit"
+				// Socket connection interrupted by sending EOT or ">quit"
 				break;
 			}
 		}
 		else if(n == -1)
 		{
-			if(debugclients)
-				logg("Client connection interrupted, ID: %i", sockID);
+			if(debugclients) logg("Unix socket connection interrupted (%s), ID: %i", strerror(errno), sockID);
+			break;
 		}
 	}
-	if(debugclients)
-		logg("Client disconnected, ID: %i", sockID);
+	if(debugclients) logg("Socket disconnected, ID: %i", sockID);
 
 	//Free the socket pointer
 	if(sock != 0)
@@ -272,9 +367,46 @@ void *socket_connection_handler_thread(void *socket_desc)
 	return 0;
 }
 
+void *telnet_listening_thread(void *args)
+{
+	// We will use the attributes object later to start all threads in detached mode
+	pthread_attr_t attr;
+	// Initialize thread attributes object with default attribute values
+	pthread_attr_init(&attr);
+	// When a detached thread terminates, its resources are automatically released back to
+	// the system without the need for another thread to join with the terminated thread
+	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+
+	// Set thread name
+	prctl(PR_SET_NAME,"telnet listener",0,0,0);
+
+	// Initialize sockets only after initial log parsing in listenting_thread
+	bind_to_telnet_port(SOCKET, &telnetfd);
+
+	// Listen as long as FTL is not killed
+	while(!killed)
+	{
+		// Look for new clients that want to connect
+		int csck = listener(telnetfd);
+
+		// Allocate memory used to transport client socket ID to client listening thread
+		int *newsock;
+		newsock = calloc(1,sizeof(int));
+		*newsock = csck;
+
+		pthread_t telnet_connection_thread;
+		// Create a new thread
+		if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0)
+		{
+			// Log the error code description
+			logg("WARNING: Unable to open telnet processing thread, error: %s", strerror(errno));
+		}
+	}
+	return 0;
+}
+
 void *socket_listening_thread(void *args)
 {
-	int *newsock;
 	// We will use the attributes object later to start all threads in detached mode
 	pthread_attr_t attr;
 	// Initialize thread attributes object with default attribute values
@@ -287,7 +419,7 @@ void *socket_listening_thread(void *args)
 	prctl(PR_SET_NAME,"socket listener",0,0,0);
 
 	// Initialize sockets only after initial log parsing in listening_thread
-	bind_to_port(SOCKET, &socketfd);
+	bind_to_unix_socket(&socketfd);
 
 	// Listen as long as FTL is not killed
 	while(!killed)
@@ -297,6 +429,7 @@ void *socket_listening_thread(void *args)
 		if(csck < 0) continue;
 
 		// Allocate memory used to transport client socket ID to client listening thread
+		int *newsock;
 		newsock = calloc(1,sizeof(int));
 		*newsock = csck;
 
@@ -305,12 +438,13 @@ void *socket_listening_thread(void *args)
 		if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0)
 		{
 			// Log the error code description
-			logg("WARNING: Unable to open client socket thread, error: %s", strerror(errno));
+			logg("WARNING: Unable to open socket processing thread, error: %s", strerror(errno));
 		}
 	}
 	return 0;
 }
 
+
 void *api_connection_handler_thread(void *socket_desc)
 {
 	//Get the socket descriptor
@@ -442,7 +576,7 @@ void *api_listening_thread(void *args)
 	prctl(PR_SET_NAME,"API listener",0,0,0);
 
 	// Initialize sockets only after initial log parsing in listening_thread
-	bind_to_port(API, &apifd);
+	bind_to_telnet_port(API, &apifd);
 
 	// Listen as long as FTL is not killed
 	while(!killed)

socket_client.c

@@ -0,0 +1,77 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Unix socket connection test program
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#define BUF 1024
+
+int main (int argc, char **argv) {
+	int socketfd;
+	char *buffer = malloc (BUF);
+	struct sockaddr_un address;
+	int size, ret;
+
+	// Create socket
+	socketfd = socket(PF_LOCAL, SOCK_STREAM, 0);
+	if(socketfd <= 0)
+	{
+		printf("Error creating socket!\n");
+		exit(EXIT_FAILURE);
+	}
+	printf("Socket created\n");
+
+	// Set socket family to local socket (not an Internet socket)
+	address.sun_family = AF_LOCAL;
+
+	// Set socket file location (respect special location on the CI system Travis)
+	if(argc == 2 && strcmp(argv[1], "travis") == 0)
+		strcpy(address.sun_path,"pihole-FTL.sock");
+	else
+		strcpy(address.sun_path,"/var/run/pihole/FTL.sock");
+
+	// Connect to the socket provided by pihole-FTL
+	ret = connect(socketfd, (struct sockaddr *) &address, sizeof (address));
+	if (ret != 0)
+	{
+		printf("Error establishing connection!\n");
+		exit(EXIT_FAILURE);
+	}
+	printf("Connection established\n");
+
+	// As an example, we query the current statistics from FTL through the socket here
+	sprintf(buffer, ">stats");
+	send(socketfd, buffer, strlen (buffer), 0);
+
+	// Try to receive data until either recv() fails or we see "--EOM--"
+	while((size = recv(socketfd, buffer, BUF-1, 0)) > -1)
+	{
+		// Zero-terminate incoming message
+		if(size > 0)
+			buffer[size] = '\0';
+
+		// Print received data to stdout
+		printf("%s", buffer);
+
+		// Exit on End Of Message
+		if(strstr(buffer, "--EOM--") != NULL)
+			break;
+
+	}
+
+	// Close Unix socket connection
+	close(socketfd);
+	return EXIT_SUCCESS;
+}

structs.c

@@ -16,7 +16,8 @@ FTLFileNamesStruct FTLfiles = {
 	"/var/log/pihole-FTL.log",
 	"/var/run/pihole-FTL.pid",
 	"/var/run/pihole-FTL.port",
-	NULL
+	NULL,
+	"/var/run/pihole/FTL.sock"
 };
 
 logFileNamesStruct files = {

test/run.sh

@@ -8,39 +8,39 @@ dnsmasq_pre() {
 # Prepare FTL's files
 ts="$(dnsmasq_pre)"
 cat <<EOT >> pihole.log
-${ts} query[AAAA] raspberrypi from 127.0.0.1
-${ts} /etc/pihole/local.list raspberrypi is fda2:2001:5647:0:ba27:ebff:fe37:4205
-${ts} query[A] ChEcKiP.DyNdNs.OrG from 127.0.0.1
-${ts} forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::9249:d69b
-${ts} forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::1c04:b12f
-${ts} forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccd::2
-${ts} forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccc::2
-${ts} reply ChEcKiP.DyNdNs.OrG is <CNAME>
-${ts} reply checkip.dyndns.com is 216.146.38.70
-${ts} reply checkip.dyndns.com is 216.146.43.71
-${ts} reply checkip.dyndns.com is 91.198.22.70
-${ts} reply checkip.dyndns.com is 216.146.43.70
-${ts} query[A] pi.hole from 10.8.0.2
-${ts} /etc/pihole/local.list pi.hole is 192.168.2.10
-${ts} query[A] example.com from 10.8.0.2
-${ts} /etc/pihole/local.list example.com is 192.168.2.10
-${ts} query[A] play.google.com from 192.168.2.208
-${ts} forwarded play.google.com to 2001:1608:10:25::9249:d69b
-${ts} forwarded play.google.com to 2001:1608:10:25::1c04:b12f
-${ts} forwarded play.google.com to 2620:0:ccd::2
-${ts} forwarded play.google.com to 2620:0:ccc::2
-${ts} reply play.google.com is <CNAME>
-${ts} reply play.l.google.com is 216.58.208.110
-${ts} reply play.l.google.com is 216.58.208.110
-${ts} reply play.l.google.com is 216.58.208.110
-${ts} reply play.google.com is <CNAME>
-${ts} query[AAAA] play.google.com from 192.168.2.208
-${ts} forwarded play.google.com to 2620:0:ccd::2
-${ts} reply play.l.google.com is 2a00:1450:4017:802::200e
-${ts} query[A] blacklisted.com from 192.168.2.208
-${ts} /etc/pihole/black.list blacklisted.com is 1.2.3.4
-${ts} query[A] addomain.com from 192.168.2.208
-${ts} /etc/pihole/gravity.list addomain.com is 1.2.3.4
+${ts} 1 1270.0.01/1234 query[AAAA] raspberrypi from 127.0.0.1
+${ts} 1 1270.0.01/1234 /etc/pihole/local.list raspberrypi is fda2:2001:5647:0:ba27:ebff:fe37:4205
+${ts} 2 1270.0.01/1234 query[A] ChEcKiP.DyNdNs.OrG from 127.0.0.1
+${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::9249:d69b
+${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::1c04:b12f
+${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccd::2
+${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccc::2
+${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is <CNAME>
+${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.38.70
+${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.43.71
+${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 91.198.22.70
+${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.43.70
+${ts} 3 1270.0.01/1234 query[A] pi.hole from 10.8.0.2
+${ts} 3 1270.0.01/1234 /etc/pihole/local.list pi.hole is 192.168.2.10
+${ts} 4 1270.0.01/1234 query[A] example.com from 10.8.0.2
+${ts} 4 1270.0.01/1234 /etc/pihole/local.list example.com is 192.168.2.10
+${ts} 5 1270.0.01/1234 query[A] play.google.com from 192.168.2.208
+${ts} 5 1270.0.01/1234 forwarded play.google.com to 2001:1608:10:25::9249:d69b
+${ts} 5 1270.0.01/1234 forwarded play.google.com to 2001:1608:10:25::1c04:b12f
+${ts} 5 1270.0.01/1234 forwarded play.google.com to 2620:0:ccd::2
+${ts} 5 1270.0.01/1234 forwarded play.google.com to 2620:0:ccc::2
+${ts} 5 1270.0.01/1234 reply play.google.com is <CNAME>
+${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110
+${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110
+${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110
+${ts} 5 1270.0.01/1234 reply play.google.com is <CNAME>
+${ts} 6 1270.0.01/1234 query[AAAA] play.google.com from 192.168.2.208
+${ts} 6 1270.0.01/1234 forwarded play.google.com to 2620:0:ccd::2
+${ts} 6 1270.0.01/1234 reply play.l.google.com is 2a00:1450:4017:802::200e
+${ts} 7 1270.0.01/1234 query[A] blacklisted.com from 192.168.2.208
+${ts} 7 1270.0.01/1234 /etc/pihole/black.list blacklisted.com is 1.2.3.4
+${ts} 8 1270.0.01/1234 query[A] addomain.com from 192.168.2.208
+${ts} 8 1270.0.01/1234 /etc/pihole/gravity.list addomain.com is 1.2.3.4
 EOT
 touch "pihole-FTL.log"
 

test/test_suite.sh

@@ -242,6 +242,24 @@ load 'libs/bats-support/load'
   [[ ${lines[0]} == "pihole-FTL - The Pi-hole FTL engine" ]]
 }
 
+@test "Unix socket returning data" {
+  run bash -c './socket-test travis'
+  echo "output: ${lines[@]}"
+  [[ ${lines[0]} == "Socket created" ]]
+  [[ ${lines[1]} == "Connection established" ]]
+  [[ ${lines[2]} == "domains_being_blocked -1" ]]
+  [[ ${lines[3]} == "dns_queries_today 7" ]]
+  [[ ${lines[4]} == "ads_blocked_today 2" ]]
+  [[ ${lines[5]} == "ads_percentage_today 28.571428" ]]
+  [[ ${lines[6]} == "unique_domains 6" ]]
+  [[ ${lines[7]} == "queries_forwarded 3" ]]
+  [[ ${lines[8]} == "queries_cached 2" ]]
+  [[ ${lines[9]} == "clients_ever_seen 3" ]]
+  [[ ${lines[10]} == "unique_clients 3" ]]
+  [[ ${lines[11]} == "status unknown" ]]
+  [[ ${lines[12]} == "---EOM---" ]]
+}
+
 @test "Final part of the tests: Killing pihole-FTL process" {
   run bash -c 'echo ">kill" | nc -v 127.0.0.1 4711'
   echo "output: ${lines[@]}"