Merge branch 'development' into new/shmem

# Conflicts:
#	api.c
#	dnsmasq_interface.c

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 
 .job_template: &job_template
   docker:
-    - image: thepihole/ftl-build:$CIRCLE_JOB
+    - image: pihole/ftl-build:$CIRCLE_JOB
   steps:
     - checkout
     - run:
@@ -12,17 +12,20 @@ version: 2
     - run:
         name: "Build"
         command: |
-          make CFLAGS="${CFLAGS}" GIT_BRANCH="${CIRCLE_BRANCH}" GIT_TAG="${CIRCLE_TAG}"
+          BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master")
+
+          make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}"
           file pihole-FTL
     - run:
         name: "Upload"
         command: |
+          FOLDER=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "$CIRCLE_TAG")
           mv pihole-FTL "${BIN_NAME}"
           sha1sum pihole-FTL-* > ${BIN_NAME}.sha1
           wget https://ftl.pi-hole.net:8080/FTL-client
           chmod +x ./FTL-client
-          [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${CIRCLE_BRANCH}" "${BIN_NAME}" "${FTL_SECRET}"
-          [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${CIRCLE_BRANCH}" "${BIN_NAME}.sha1" "${FTL_SECRET}"
+          [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}" "${FTL_SECRET}"
+          [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}.sha1" "${FTL_SECRET}"
           rm ./FTL-client
           ls -lah .
 
@@ -62,9 +65,25 @@ workflows:
   version: 2
   build:
     jobs:
-      - arm
-      - armhf
-      - aarch64
-      - x86_64
+      - arm:
+          filters:
+            tags:
+              only: /^v.*/
+      - armhf:
+          filters:
+            tags:
+              only: /^v.*/
+      - aarch64:
+          filters:
+            tags:
+              only: /^v.*/
+      - x86_64:
+          filters:
+            tags:
+              only: /^v.*/
 #      - x86_64-musl
-      - x86_32
+      - x86_32:
+          filters:
+            tags:
+              only: /^v.*/
+

FTL.h

@@ -78,10 +78,10 @@
 enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER };
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
-enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST };
+enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED };
 enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX };
-enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP };
-enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM };
+enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME };
+enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };
 enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA };
 enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED };
 enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN };
@@ -130,6 +130,7 @@ typedef struct {
 	int reply_NXDOMAIN;
 	int reply_CNAME;
 	int reply_IP;
+	int reply_domain;
 } countersStruct;
 
 typedef struct {
@@ -145,6 +146,8 @@ typedef struct {
 	bool ignore_localhost;
 	unsigned char blockingmode;
 	bool regex_debugmode;
+	bool analyze_only_A_AAAA;
+	bool DBimport;
 } ConfigStruct;
 
 // Dynamic structs
@@ -164,6 +167,7 @@ typedef struct {
 	unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.)
 	unsigned char reply;
 	unsigned char dnssec;
+	bool AD;
 } queriesDataStruct;
 
 typedef struct {

api.c

@@ -578,6 +578,7 @@ void getQueryTypes(int *sock)
 	}
 }
 
+char *querytypes[8] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","UNKN"};
 
 void getAllQueries(char *client_message, int *sock)
 {
@@ -591,14 +592,74 @@ void getAllQueries(char *client_message, int *sock)
 
 	char *domainname = NULL;
 	bool filterdomainname = false;
+	int domainid = -1;
 
 	char *clientname = NULL;
 	bool filterclientname = false;
+	int clientid = -1;
+
+	int querytype = 0;
+
+	char *forwarddest = NULL;
+	bool filterforwarddest = false;
+	int forwarddestid = 0;
 
 	// Time filtering?
 	if(command(client_message, ">getallqueries-time")) {
 		sscanf(client_message, ">getallqueries-time %i %i",&from, &until);
 	}
+
+	// Query type filtering?
+	if(command(client_message, ">getallqueries-qtype")) {
+		// Get query type we want to see only
+		sscanf(client_message, ">getallqueries-qtype %i", &querytype);
+		if(querytype < 1 || querytype >= TYPE_MAX)
+		{
+			// Invalid query type requested
+			return;
+		}
+	}
+
+	// Forward destination filtering?
+	if(command(client_message, ">getallqueries-forward")) {
+		// Get forward destination name we want to see only (limit length to 255 chars)
+		forwarddest = calloc(256, sizeof(char));
+		if(forwarddest == NULL) return;
+		sscanf(client_message, ">getallqueries-forward %255s", forwarddest);
+		filterforwarddest = true;
+
+		if(strcmp(forwarddest, "cache") == 0)
+			forwarddestid = -1;
+		else if(strcmp(forwarddest, "blocklist") == 0)
+			forwarddestid = -2;
+		else
+		{
+			// Iterate through all known forward destinations
+			int i;
+			validate_access("forwards", MAX(0,counters.forwarded-1), true, __LINE__, __FUNCTION__, __FILE__);
+			forwarddestid = -3;
+			for(i = 0; i < counters.forwarded; i++)
+			{
+				// Try to match the requested string against their IP addresses and
+				// (if available) their host names
+				if(strcmp(forwarded[i].ip, forwarddest) == 0 ||
+				   (forwarded[i].name != NULL &&
+				    strcmp(forwarded[i].name, forwarddest) == 0))
+				{
+					forwarddestid = i;
+					break;
+				}
+			}
+			if(forwarddestid < 0)
+			{
+				// Requested forward destination has not been found, we directly
+				// exit here as there is no data to be returned
+				free(forwarddest);
+				return;
+			}
+		}
+	}
+
 	// Domain filtering?
 	if(command(client_message, ">getallqueries-domain")) {
 		// Get domain name we want to see only (limit length to 255 chars)
@@ -606,7 +667,27 @@ void getAllQueries(char *client_message, int *sock)
 		if(domainname == NULL) return;
 		sscanf(client_message, ">getallqueries-domain %255s", domainname);
 		filterdomainname = true;
+		// Iterate through all known domains
+		int i;
+		validate_access("domains", MAX(0,counters.domains-1), true, __LINE__, __FUNCTION__, __FILE__);
+		for(i = 0; i < counters.domains; i++)
+		{
+			// Try to match the requested string
+			if(strcmp(domains[i].domain, domainname) == 0)
+			{
+				domainid = i;
+				break;
+			}
+		}
+		if(domainid < 0)
+		{
+			// Requested domain has not been found, we directly
+			// exit here as there is no data to be returned
+			free(domainname);
+			return;
+		}
 	}
+
 	// Client filtering?
 	if(command(client_message, ">getallqueries-client")) {
 		// Get client name we want to see only (limit length to 255 chars)
@@ -614,6 +695,27 @@ void getAllQueries(char *client_message, int *sock)
 		if(clientname == NULL) return;
 		sscanf(client_message, ">getallqueries-client %255s", clientname);
 		filterclientname = true;
+		// Iterate through all known clients
+		int i;
+		validate_access("clients", MAX(0,counters.clients-1), true, __LINE__, __FUNCTION__, __FILE__);
+		for(i = 0; i < counters.clients; i++)
+		{
+			// Try to match the requested string
+			if(strcmp(clients[i].ip, clientname) == 0 ||
+			   (clients[i].name != NULL &&
+			    strcmp(clients[i].name, clientname) == 0))
+			{
+				clientid = i;
+				break;
+			}
+		}
+		if(clientid < 0)
+		{
+			// Requested client has not been found, we directly
+			// exit here as there is no data to be returned
+			free(clientname);
+			return;
+		}
 	}
 
 	int ibeg = 0, num;
@@ -654,7 +756,7 @@ void getAllQueries(char *client_message, int *sock)
 		validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
 		validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__);
 
-		char *qtype = (queries[i].type == TYPE_A)? "A" : "AAAA";
+		char *qtype = querytypes[queries[i].type - TYPE_A];
 
 		// 1 = gravity.list, 4 = wildcard, 5 = black.list
 		if((queries[i].status == QUERY_GRAVITY ||
@@ -670,18 +772,30 @@ void getAllQueries(char *client_message, int *sock)
 		if((from > queries[i].timestamp && from != 0) || (queries[i].timestamp > until && until != 0))
 			continue;
 
-		if(filterdomainname)
-		{
-			// Skip if domain name is not identical with what the user wants to see
-			if(strcmp(getstr(domains[queries[i].domainID].domainpos), domainname) != 0)
-				continue;
-		}
+		// Skip if domain is not identical with what the user wants to see
+		if(filterdomainname && queries[i].domainID != domainid)
+			continue;
 
-		if(filterclientname)
+		// Skip if client name and IP are not identical with what the user wants to see
+		if(filterclientname && queries[i].clientID != clientid)
+			continue;
+
+		// Skip if query type is not identical with what the user wants to see
+		if(querytype != 0 && querytype != queries[i].type)
+			continue;
+
+		if(filterforwarddest)
 		{
-			// Skip if client name and IP are not identical with what the user wants to see
-			if(strcmp(getstr(clients[queries[i].clientID].ippos), clientname) != 0 &&
-			  (strcmp(getstr(clients[queries[i].clientID].namepos), clientname) != 0))
+			// Does the user want to see queries answered from blocking lists?
+			if(forwarddestid == -2 && queries[i].status != QUERY_GRAVITY
+			                       && queries[i].status != QUERY_WILDCARD
+			                       && queries[i].status != QUERY_BLACKLIST)
+				continue;
+			// Does the user want to see queries answered from local cache?
+			else if(forwarddestid == -1 && queries[i].status != QUERY_CACHE)
+				continue;
+			// Does the user want to see queries answered by an upstream server?
+			else if(forwarddestid >= 0 && forwarddestid != queries[i].forwardID)
 				continue;
 		}
 
@@ -724,6 +838,9 @@ void getAllQueries(char *client_message, int *sock)
 
 	if(filterdomainname)
 		free(domainname);
+
+	if(filterforwarddest)
+		free(forwarddest);
 }
 
 void getRecentBlocked(char *client_message, int *sock)

config.c

@@ -167,8 +167,9 @@ void read_FTLconf(void)
 	// PRIVACY_HIDE_DOMAINS (1) = show and store all domains as "hidden", return nothing for Top Domains + Top Ads
 	// PRIVACY_HIDE_DOMAINS_CLIENTS (2) = as above, show all domains as "hidden" and all clients as "127.0.0.1"
 	//                                    (or "::1"), return nothing for any Top Lists
-	// PRIVACY_MAXIMUM (3) = Disabled basically everything except the anonymous stastics, there will be no entries
+	// PRIVACY_MAXIMUM (3) = Disabled basically everything except the anonymous statistics, there will be no entries
 	//                       added to the database, no entries visible in the query log and no Top Item Lists
+	// PRIVACY_NOSTATS (4) = Disable any analysis on queries. No counters are available in this mode.
 	// defaults to: PRIVACY_SHOW_ALL
 	config.privacylevel = PRIVACY_SHOW_ALL;
 	get_privacy_level(fp);
@@ -219,6 +220,30 @@ void read_FTLconf(void)
 	else
 		logg("   REGEX_DEBUGMODE: Inactive");
 
+	// ANALYZE_ONLY_A_AND_AAAA
+	// defaults to: No
+	config.analyze_only_A_AAAA = false;
+	buffer = parse_FTLconf(fp, "ANALYZE_ONLY_A_AND_AAAA");
+
+	if(buffer != NULL && strcasecmp(buffer, "true") == 0)
+		config.analyze_only_A_AAAA = true;
+
+	if(config.analyze_only_A_AAAA)
+		logg("   ANALYZE_ONLY_A_AND_AAAA: Enabled. Analyzing only A and AAAA queries");
+	else
+		logg("   ANALYZE_ONLY_A_AND_AAAA: Disabled. Analyzing all queries");
+
+	// DBIMPORT
+	// defaults to: Yes
+	config.DBimport = true;
+	buffer = parse_FTLconf(fp, "DBIMPORT");
+	if(buffer != NULL && strcasecmp(buffer, "no") == 0)
+		config.DBimport = false;
+	if(config.DBimport)
+		logg("   DBIMPORT: Importing history from database");
+	else
+		logg("   DBIMPORT: Not importing history from database");
+
 	logg("Finished config file parsing");
 
 	// Release memory
@@ -303,7 +328,7 @@ void get_privacy_level(FILE *fp)
 		// Check for change and validity of privacy level (set in FTL.h)
 		if(value != config.privacylevel &&
 		   value >= PRIVACY_SHOW_ALL &&
-		   value <= PRIVACY_MAXIMUM)
+		   value <= PRIVACY_NOSTATS)
 		{
 			logg("Notice: Changing privacy level from %i to %i", config.privacylevel, value);
 			config.privacylevel = value;

database.c

@@ -337,6 +337,10 @@ int get_number_of_queries_in_DB(void)
 
 void save_to_DB(void)
 {
+	// Don't save anything to the database if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Start database timer
 	if(debug) timer_start(DATABASE_WRITE_TIMER);
 
@@ -452,9 +456,10 @@ void save_to_DB(void)
 
 		// Total counter information (delta computation)
 		total++;
-		if(queries[i].status == 1 ||
-		   queries[i].status == 4 ||
-		   queries[i].status == 5)
+		if(queries[i].status == QUERY_GRAVITY ||
+		   queries[i].status == QUERY_BLACKLIST ||
+		   queries[i].status == QUERY_WILDCARD ||
+		   queries[i].status == QUERY_EXTERNAL_BLOCKED)
 			blocked++;
 
 		// Update lasttimestamp variable with timestamp of the latest stored query
@@ -574,6 +579,10 @@ void *DB_thread(void *val)
 // Get most recent 24 hours data from long-term database
 void read_data_from_DB(void)
 {
+	// Don't try to load anything to the database if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Open database file
 	if(!dbopen())
 	{
@@ -628,9 +637,9 @@ void read_data_from_DB(void)
 		}
 
 		int type = sqlite3_column_int(stmt, 2);
-		if(type != TYPE_A && type != TYPE_AAAA)
+		if(type < TYPE_A || type >= TYPE_MAX)
 		{
-			logg("DB warn: TYPE should be either 1 or 2 but not %i", type);
+			logg("DB warn: TYPE should not be %i", type);
 			continue;
 		}
 		// Don't import AAAA queries from database if the user set
@@ -641,9 +650,9 @@ void read_data_from_DB(void)
 		}
 
 		int status = sqlite3_column_int(stmt, 3);
-		if(status < QUERY_UNKNOWN || status > QUERY_BLACKLIST)
+		if(status < QUERY_UNKNOWN || status > QUERY_EXTERNAL_BLOCKED)
 		{
-			logg("DB warn: STATUS should be within [0,5] but is %i", status);
+			logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_EXTERNAL_BLOCKED, status);
 			continue;
 		}
 
@@ -704,6 +713,7 @@ void read_data_from_DB(void)
 		queries[queryID].id = 0; // This is dnsmasq's internal ID. We don't store it in the database
 		queries[queryID].complete = true; // Mark as all information is avaiable
 		queries[queryID].response = 0;
+		queries[queryID].AD = false;
 		lastDBimportedtimestamp = queryTimeStamp;
 
 		// Handle type counters
@@ -733,6 +743,7 @@ void read_data_from_DB(void)
 			case QUERY_GRAVITY: // Blocked by gravity.list
 			case QUERY_WILDCARD: // Blocked by regex filter
 			case QUERY_BLACKLIST: // Blocked by black.list
+			case QUERY_EXTERNAL_BLOCKED: // Blocked by external provider
 				counters.blocked++;
 				overTime[timeidx].blocked++;
 				domains[domainID].blockedcount++;

dnsmasq/cache.c

@@ -1534,6 +1534,10 @@ char *record_source(unsigned int index)
     return "config";
   else if (index == SRC_HOSTS)
     return HOSTSFILE;
+  /*----- Pi-hole modification -----*/
+  else if (index == SRC_REGEX)
+    return (char*)regexlistname;
+  /*--------------------------------*/
 
   for (ah = daemon->addn_hosts; ah; ah = ah->next)
     if (ah->index == index)

dnsmasq/dnsmasq.c

@@ -570,7 +570,7 @@ int main_dnsmasq (int argc, char **argv)
 	}
     }
 
-    FTL_fork_and_bind_sockets();
+    FTL_fork_and_bind_sockets(ent_pw);
 
    log_err = log_start(ent_pw, err_pipe[1]);
 

dnsmasq/dnsmasq.h

@@ -464,7 +464,10 @@ struct crec {
 #define SRC_CONFIG    1
 #define SRC_HOSTS     2
 #define SRC_AH        3
-
+/*----- Pi-hole modification -----*/
+#define SRC_REGEX     4
+const char *regexlistname;
+/*--------------------------------*/
 
 /* struct sockaddr is not large enough to hold any address,
    and specifically not big enough to hold an IPv6 address.

dnsmasq/forward.c

@@ -648,6 +648,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
 	}
     }
 
+  FTL_header_ADbit(header->hb4, daemon->log_display_id);
+
   /* RFC 4035 sect 4.6 para 3 */
   if (!is_sign && !option_bool(OPT_DNSSEC_PROXY))
      header->hb4 &= ~HB4_AD;

dnsmasq/option.c

@@ -2212,7 +2212,9 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
 		      char *star;
 		      new->next = daemon->synth_domains;
 		      daemon->synth_domains = new;
-		      if ((star = strrchr(new->prefix, '*')) && *(star+1) == 0)
+		      if (new->prefix &&
+			  (star = strrchr(new->prefix, '*'))
+			  && *(star+1) == 0)
 			{
 			  *star = 0;
 			  new->indexed = 1;

dnsmasq/rfc1035.c

@@ -1341,6 +1341,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		      unsigned long ttl = daemon->local_ttl;
 		      int ok = 1;
 		      log_query(F_CONFIG | F_RRNAME, name, NULL, "<TXT>");
+		      FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "<TXT>", daemon->log_display_id);
 #ifndef NO_ID
 		      /* Dynamically generate stat record */
 		      if (t->stat != 0)
@@ -1372,6 +1373,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		if (!dryrun)
 		  {
 		    log_query(F_CONFIG | F_RRNAME, name, NULL, "<RR>");
+		    FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "<RR>", daemon->log_display_id);
 		    if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 					    daemon->local_ttl, NULL,
 					    t->class, C_IN, "t", t->len, t->txt))
@@ -1430,6 +1432,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		  if (!dryrun)
 		    {
 		      log_query(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL);
+		      FTL_cache(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL, daemon->log_display_id);
 		      if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 					      daemon->local_ttl, NULL,
 					      T_PTR, C_IN, "d", intr->name))
@@ -1443,6 +1446,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		  if (!dryrun)
 		    {
 		      log_query(F_CONFIG | F_RRNAME, name, NULL, "<PTR>");
+		      FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "<PTR>", daemon->log_display_id);
 		      for (ptr = daemon->ptr; ptr; ptr = ptr->next)
 			if (hostname_isequal(name, ptr->name) &&
 			    add_resource_record(header, limit, &trunc, nameoffset, &ansp,
@@ -1479,6 +1483,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 				nxdomain = 1;
 			      if (!dryrun)
 				log_query(crecp->flags & ~F_FORWARD, name, &addr, NULL);
+				FTL_cache(crecp->flags & ~F_FORWARD, name, &addr, NULL, daemon->log_display_id);
 			    }
 			  else
 			    {
@@ -1488,6 +1493,8 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 				{
 				  log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr,
 					    record_source(crecp->uid));
+				  FTL_cache(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr,
+					    record_source(crecp->uid), daemon->log_display_id);
 
 				  if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 							  crec_ttl(crecp, now), NULL,
@@ -1505,6 +1512,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		  if (!dryrun)
 		    {
 		      log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL);
+		      FTL_cache(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL, daemon->log_display_id);
 
 		      if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 					      daemon->local_ttl, NULL,
@@ -1548,8 +1556,12 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		      sec_data = 0;
 		      nxdomain = 1;
 		      if (!dryrun)
+		      {
 			log_query(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN,
 				  name, &addr, NULL);
+			FTL_cache(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN,
+				  name, &addr, NULL, daemon->log_display_id);
+		      }
 		    }
 		}
 	    }
@@ -1619,6 +1631,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 				{
 				  gotit = 1;
 				  log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL);
+				  FTL_cache(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL, daemon->log_display_id);
 				  if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 							  daemon->local_ttl, NULL, type, C_IN,
 							  type == T_A ? "4" : "6", &addrlist->addr))
@@ -1628,7 +1641,10 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 		      }
 
 		  if (!dryrun && !gotit)
+		  {
 		    log_query(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL);
+		    FTL_cache(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL, daemon->log_display_id);
+		  }
 
 		  continue;
 		}
@@ -1673,6 +1689,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
 			    if (!dryrun)
 			      {
 				log_query(crecp->flags, name, NULL, record_source(crecp->uid));
+				FTL_cache(crecp->flags, name, NULL, record_source(crecp->uid), daemon->log_display_id);
 				if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
 							crec_ttl(crecp, now), &nameoffset,
 							T_CNAME, C_IN, "d", cname_target))

dnsmasq_interface.c

@@ -17,13 +17,20 @@ void print_flags(unsigned int flags);
 void save_reply_type(unsigned int flags, int queryID, struct timeval response);
 unsigned long converttimeval(struct timeval time);
 static void block_single_domain(char *domain);
+static void query_externally_blocked(int i);
+static int findQueryID(int id);
 
 char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "};
 
 void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Create new query in data structure
 	enable_thread_lock();
+
 	// Get timestamp
 	int querytimestamp, overTimetimestamp;
 	gettimestamp(&querytimestamp, &overTimetimestamp);
@@ -50,7 +57,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 		querytype = TYPE_TXT;
 	else
 	{
-		// Return early if query type is not known
+		// Return early to avoid accessing querytypedata out of bounds
 		if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id);
 		disable_thread_lock();
 		return;
@@ -81,6 +88,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 		return;
 	}
 
+	// Store plain text domain in buffer for regex validation
+	char *domainbuffer = strdup(domain);
+
 	// Check and apply possible privacy level rules
 	// We do this immediately on the raw data to avoid any possible leaking
 	get_privacy_level(NULL);
@@ -101,6 +111,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	   (strcmp(client, "127.0.0.1") == 0 || strcmp(client, "::1") == 0))
 	{
 		free(domain);
+		free(domainbuffer);
 		free(client);
 		disable_thread_lock();
 		return;
@@ -124,12 +135,14 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	overTime[timeidx].querytypedata[querytype-1]++;
 	counters.querytype[querytype-1]++;
 
-	// Skip rest of the analyis if this query is not of type A or AAAA
-	if(querytype != TYPE_A && querytype != TYPE_AAAA)
+	// Skip rest of the analysis if this query is not of type A or AAAA
+	// but user wants to see only A and AAAA queries (pre-v4.1 behavior)
+	if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA)
 	{
 		// Don't process this query further here, we already counted it
 		if(debug) logg("Notice: Skipping new query: %s (%i)", types, id);
 		free(domain);
+		free(domainbuffer);
 		free(client);
 		disable_thread_lock();
 		return;
@@ -159,6 +172,8 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	queries[queryID].reply = REPLY_UNKNOWN;
 	// Store DNSSEC result for this domain
 	queries[queryID].dnssec = DNSSEC_UNSPECIFIED;
+	// AD has not yet been received for this query
+	queries[queryID].AD = false;
 
 	// Increase DNS queries counter
 	counters.queries++;
@@ -188,10 +203,10 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 		// of a specific domain. The logic herein is:
 		// If matched, then compare against whitelist
 		// If in whitelist, negate matched so this function returns: not-to-be-blocked
-		if(match_regex(domain) && !in_whitelist(domain))
+		if(match_regex(domainbuffer) && !in_whitelist(domainbuffer))
 		{
 			// We have to block this domain
-			block_single_domain(domain);
+			block_single_domain(domainbuffer);
 			domains[domainID].regexmatch = REGEX_BLOCKED;
 		}
 		else
@@ -205,13 +220,41 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	// Free allocated memory
 	free(client);
 	free(domain);
+	free(domainbuffer);
 
 	// Release thread lock
 	disable_thread_lock();
 }
 
+static int findQueryID(int id)
+{
+	// Loop over all queries - we loop in reverse order (start from the most recent query and
+	// continuously walk older queries while trying to find a match. Ideally, we should always
+	// find the correct query with zero iterations, but it may happen that queries are processed
+	// asynchronously, e.g. for slow upstream relies to a huge amount of requests.
+	// We iterate from the most recent query down to at most MAXITER queries in the past to avoid
+	// iterating through the entire array of queries
+	// MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful)
+
+	// Validate access only once for the maximum index (all lower will work)
+	validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
+	int until = MAX(0, counters.queries-MAXITER);
+	int i;
+	// Check UUIDs of queries
+	for(i = counters.queries-1; i >= until; i--)
+		if(queries[i].id == id)
+			return i;
+
+	// If not found
+	return -1;
+}
+
 void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Save that this query got forwarded to an upstream server
 	enable_thread_lock();
 
@@ -226,31 +269,8 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id
 	if(debug) logg("**** forwarded %s to %s (ID %i)", name, forward, id);
 
 	// Save status and forwardID in corresponding query identified by dnsmasq's ID
-	bool found = false;
-	int i;
-	// Loop over all queries - we loop in reverse order (start from the most recent query and
-	// continuously walk older queries while trying to find a match. Ideally, we should always
-	// find the correct query with zero iterations, but it may happen that queries are processed
-	// asynchronously, e.g. for slow upstream relies to a huge amount of requests.
-	// We iterate from the most recent query down to at most MAXITER queries in the past to avoid
-	// iterating through the entire array of queries when queries that have not been recorded
-	// (like PTR queries, etc.) are processed.
-	// MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful)
-
-	// Validate access only once for the maximum index (all lower will work)
-	validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
-	int until = MAX(0, counters.queries-MAXITER);
-	for(i = counters.queries-1; i >= until; i--)
-	{
-		// Check UUID of this query
-		if(queries[i].id == id)
-		{
-			queries[i].status = QUERY_FORWARDED;
-			found = true;
-			break;
-		}
-	}
-	if(!found)
+	int i = findQueryID(id);
+	if(i < 0)
 	{
 		// This may happen e.g. if the original query was a PTR query or "pi.hole"
 		// as we ignore them altogether
@@ -259,6 +279,9 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id
 		return;
 	}
 
+	// Set query status
+	queries[i].status = QUERY_FORWARDED;
+
 	// Proceed only if
 	// - current query has not been marked as replied to so far
 	//   (it could be that answers from multiple forward
@@ -333,8 +356,9 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id
 
 void FTL_dnsmasq_reload(void)
 {
-	// This funtion is called by the dnsmasq code on receive of SIGHUP
+	// This function is called by the dnsmasq code on receive of SIGHUP
 	// *before* clearing the cache and rereading the lists
+	// This is the only hook that is not skipped in PRIVACY_NOSTATS mode
 
 	// Called when dnsmasq re-reads its config and hosts files
 	// Reset number of blocked domains
@@ -346,9 +370,9 @@ void FTL_dnsmasq_reload(void)
 
 	// Reread pihole-FTL.conf to see which blocking mode the user wants to use
 	// It is possible to change the blocking mode here as we anyhow clear the
-	// cahce and reread all blocking lists
+	// cache and reread all blocking lists
 	// Passing NULL to this function means it has to open the config file on
-	// its own behalf (on initial reading, the confg file is already opened)
+	// its own behalf (on initial reading, the config file is already opened)
 	get_blocking_mode(NULL);
 
 	// Reread regex.list
@@ -358,6 +382,10 @@ void FTL_dnsmasq_reload(void)
 
 void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Interpret hosts files that have been read by dnsmasq
 	enable_thread_lock();
 	// Determine returned result if available
@@ -367,16 +395,18 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 		inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN);
 	}
 
+	// Extract answer (used e.g. for detecting if a local config is a user-defined
+	// wildcard blocking entry in form "server=/tobeblocked.com/")
+	char *answer = dest;
+	if(flags & F_CNAME)
+		answer = "(CNAME)";
+	else if((flags & F_NEG) && (flags & F_NXDOMAIN))
+		answer = "(NXDOMAIN)";
+	else if(flags & F_NEG)
+		answer = "(NODATA)";
+
 	if(debug)
 	{
-		char *answer = dest;
-		if(flags & F_CNAME)
-			answer = "(CNAME)";
-		else if((flags & F_NEG) && (flags & F_NXDOMAIN))
-			answer = "(NXDOMAIN)";
-		else if(flags & F_NEG)
-			answer = "(NODATA)";
-
 		logg("**** got reply %s is %s (ID %i)", name, answer, id);
 		print_flags(flags);
 	}
@@ -385,101 +415,102 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 	struct timeval response;
 	gettimeofday(&response, 0);
 
-	if(flags & F_CONFIG)
+	// Save status in corresponding query identified by dnsmasq's ID
+	int i = findQueryID(id);
+	if(i < 0)
 	{
-		// Answered from local configuration, might be a wildcard or user-provided
-		// Save status in corresponding query identified by dnsmasq's ID
-		bool found = false;
-		int i;
-
-		// Search match in known queries
-		// See comments in FTL_forwarded() for further details about this loop
-		validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
-		int until = MAX(0, counters.queries-MAXITER);
-		for(i = counters.queries-1; i >= until; i--)
-		{
-			// Check UUID of this query
-			if(queries[i].id == id)
-			{
-				found = true;
-				break;
-			}
-		}
-
-		if(!found)
-		{
-			// This may happen e.g. if the original query was a PTR query or "pi.hole"
-			// as we ignore them altogether
-			disable_thread_lock();
-			return;
-		}
-
-		if(!queries[i].complete)
-		{
-			// This query is no longer unknown
-			counters.unknown--;
-			// Answered from a custom (user provided) cache file
-			counters.cached++;
-			queries[i].status = QUERY_CACHE;
-
-			// Get time index
-			int querytimestamp, overTimetimestamp;
-			gettimestamp(&querytimestamp, &overTimetimestamp);
-			int timeidx = findOverTimeID(overTimetimestamp);
-			validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
-
-			overTime[timeidx].cached++;
-
-			// Save reply type and update individual reply counters
-			save_reply_type(flags, i, response);
-
-			// Hereby, this query is now fully determined
-			queries[i].complete = true;
-		}
-
-		// We are done here
+		// This may happen e.g. if the original query was "pi.hole"
+		if(debug) logg("FTL_reply(): Query %i has not been found", id);
 		disable_thread_lock();
 		return;
 	}
+
+	if(queries[i].reply != REPLY_UNKNOWN)
+	{
+		// Nothing to be done here
+		disable_thread_lock();
+		return;
+	}
+
+	if(flags & F_CONFIG)
+	{
+		// Answered from local configuration, might be a wildcard or user-provided
+		// This query is no longer unknown
+		counters.unknown--;
+		// Answered from a custom (user provided) cache file
+		counters.cached++;
+
+		// Detect user-defined blocking rules
+		if(strcmp(answer, "(NXDOMAIN)") == 0 ||
+		   strcmp(answer, "0.0.0.0") == 0 ||
+		   strcmp(answer, "::") == 0)
+			queries[i].status = QUERY_WILDCARD;
+		else
+			queries[i].status = QUERY_CACHE;
+
+		// Get time index
+		int querytimestamp, overTimetimestamp;
+		gettimestamp(&querytimestamp, &overTimetimestamp);
+		int timeidx = findOverTimeID(overTimetimestamp);
+		validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+
+		overTime[timeidx].cached++;
+
+		// Save reply type and update individual reply counters
+		save_reply_type(flags, i, response);
+
+		// Hereby, this query is now fully determined
+		queries[i].complete = true;
+	}
 	else if(flags & F_FORWARD)
 	{
-		// Search for corresponding query identified by dnsmasq's ID
-		bool found = false;
-		int i;
-
-		// Search match in known queries
-		// See comments in FTL_forwarded() for further details about this loop
-		validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
-		int until = MAX(0, counters.queries-MAXITER);
-		for(i = counters.queries-1; i >= until; i--)
-		{
-			// Check UUID of this query
-			if(queries[i].id == id)
-			{
-				found = true;
-				break;
-			}
-		}
-
-		if(!found)
-		{
-			// This may happen e.g. if the original query was a PTR query or "pi.hole"
-			// as we ignore them altogether
-			disable_thread_lock();
-			return;
-		}
-
 		int domainID = queries[i].domainID;
 		validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__);
+
 		if(strcmp(getstr(domains[domainID].domainpos), name) == 0)
 		{
 			// Save reply type and update individual reply counters
 			save_reply_type(flags, i, response);
+
+			// If received NXDOMAIN and AD bit is set, Quad9 may have blocked this query
+			if(flags & F_NXDOMAIN && queries[i].AD)
+			{
+				query_externally_blocked(i);
+			}
+
+			// If received one of the following IPs as reply, OpenDNS
+			// (Cisco Umbrella) blocked this query
+			// See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses-
+			// for a full list of these IP addresses
+			else if(flags & F_IPV4 && answer != NULL &&
+				(strcmp("146.112.61.104", answer) == 0 ||
+				 strcmp("146.112.61.105", answer) == 0 ||
+				 strcmp("146.112.61.106", answer) == 0 ||
+				 strcmp("146.112.61.107", answer) == 0 ||
+				 strcmp("146.112.61.108", answer) == 0 ||
+				 strcmp("146.112.61.109", answer) == 0 ||
+				 strcmp("146.112.61.110", answer) == 0 ))
+			{
+					query_externally_blocked(i);
+			}
+
+			else if(flags & F_IPV6 && answer != NULL &&
+				(strcmp("::ffff:146.112.61.104", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.105", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.106", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.107", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.108", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.109", answer) == 0 ||
+				 strcmp("::ffff:146.112.61.110", answer) == 0 ))
+			{
+					query_externally_blocked(i);
+			}
 		}
 	}
 	else if(flags & F_REVERSE)
 	{
-		if(debug) logg("Skipping result of PTR query");
+		// Save reply type and update individual reply counters
+		save_reply_type(flags, i, response);
 	}
 	else
 	{
@@ -490,8 +521,31 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id)
 	disable_thread_lock();
 }
 
+static void query_externally_blocked(int i)
+{
+	// Correct counters as we won't count this as forwarded ...
+	counters.forwardedqueries--;
+	overTime[queries[i].timeidx].forwarded--;
+	validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__);
+	forwarded[queries[i].forwardID].count--;
+
+	// ... but as blocked
+	counters.blocked++;
+	overTime[queries[i].timeidx].blocked++;
+	validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__);
+	domains[queries[i].domainID].blockedcount++;
+	validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__);
+	clients[queries[i].clientID].blockedcount++;
+
+	queries[i].status = QUERY_EXTERNAL_BLOCKED;
+}
+
 void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Save that this query got answered from cache
 	enable_thread_lock();
 	char dest[ADDRSTRLEN]; dest[0] = '\0';
@@ -522,7 +576,11 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 	struct timeval response;
 	gettimeofday(&response, 0);
 
-	if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || ((flags & F_NAMEP) && (flags & F_DHCP)) || (flags & F_FORWARD))
+	if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) ||
+	   ((flags & F_NAMEP) && (flags & F_DHCP)) ||
+	   (flags & F_FORWARD) ||
+	   (flags & F_REVERSE) ||
+	   (flags & F_RRNAME))
 	{
 		// List data: /etc/pihole/gravity.list, /etc/pihole/black.list, /etc/pihole/local.list, etc.
 		// or
@@ -547,28 +605,18 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 			requesttype = QUERY_CACHE;
 		else if(flags & F_FORWARD) // cached answer to previously forwarded request
 			requesttype = QUERY_CACHE;
+		else if(flags & F_REVERSE) // cached answer to reverse request (PTR)
+			requesttype = QUERY_CACHE;
+		else if(flags & F_RRNAME) // cached answer to TXT query
+			requesttype = QUERY_CACHE;
 		else
 		{
 			logg("*************************** unknown CACHE reply (1) ***************************");
 			print_flags(flags);
 		}
 
-		bool found = false;
-		int i;
-		// Search match in known queries
-		// See comments in FTL_forwarded() for further details about this loop
-		validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
-		int until = MAX(0, counters.queries-MAXITER);
-		for(i = counters.queries-1; i >= until; i--)
-		{
-			// Check UUID of this query
-			if(queries[i].id == id)
-			{
-				found = true;
-				break;
-			}
-		}
-		if(!found)
+		int i = findQueryID(id);
+		if(i < 0)
 		{
 			// This may happen e.g. if the original query was a PTR query or "pi.hole"
 			// as we ignore them altogether
@@ -633,26 +681,15 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg,
 
 void FTL_dnssec(int status, int id)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Process DNSSEC result for a domain
 	enable_thread_lock();
-	// Search for corresponding query indentified by ID
-	bool found = false;
-	int i;
-	// Search match in known queries
-	// See comments in FTL_forwarded() for further details about this loop
-	validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__);
-	int until = MAX(0, counters.queries-MAXITER);
-	for(i = counters.queries-1; i >= until; i--)
-	{
-		// Check both UUID and generation of this query
-		if(queries[i].id == id)
-		{
-			found = true;
-			break;
-		}
-	}
-
-	if(!found)
+	// Search for corresponding query identified by ID
+	int i = findQueryID(id);
+	if(i < 0)
 	{
 		// This may happen e.g. if the original query was an unhandled query type
 		disable_thread_lock();
@@ -678,6 +715,36 @@ void FTL_dnssec(int status, int id)
 	disable_thread_lock();
 }
 
+void FTL_header_ADbit(unsigned char header4, int id)
+{
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
+	enable_thread_lock();
+	// Check if AD bit is set in DNS header
+	if(!(header4 & 0x20))
+	{
+		// AD bit not set
+		disable_thread_lock();
+		return;
+	}
+
+	// Search for corresponding query identified by ID
+	int i = findQueryID(id);
+	if(i < 0)
+	{
+		// This may happen e.g. if the original query was an unhandled query type
+		disable_thread_lock();
+		return;
+	}
+
+	// Store AD bit in query data
+	queries[i].AD = true;
+
+	disable_thread_lock();
+}
+
 void print_flags(unsigned int flags)
 {
 	// Debug function, listing resolver flags in clear text
@@ -716,6 +783,17 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response)
 		queries[queryID].reply = REPLY_CNAME;
 		counters.reply_CNAME++;
 	}
+	else if(flags & F_REVERSE)
+	{
+		// reserve lookup
+		queries[queryID].reply = REPLY_DOMAIN;
+		counters.reply_domain++;
+	}
+	else if(flags & F_RRNAME)
+	{
+		// TXT query
+		queries[queryID].reply = REPLY_RRNAME;
+	}
 	else
 	{
 		// Valid IP
@@ -734,7 +812,7 @@ pthread_t socket_listenthread;
 pthread_t DBthread;
 pthread_t GCthread;
 
-void FTL_fork_and_bind_sockets(void)
+void FTL_fork_and_bind_sockets(struct passwd *ent_pw)
 {
 	if(!debug && daemonmode)
 		go_daemon();
@@ -786,6 +864,16 @@ void FTL_fork_and_bind_sockets(void)
 		logg("Unable to open GC thread. Exiting...");
 		exit(EXIT_FAILURE);
 	}
+
+	// Chown files if FTL started as user root but a dnsmasq config option
+	// states to run as a different user/group (e.g. "nobody")
+	if(ent_pw != NULL && getuid() == 0)
+	{
+		if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1)
+			logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno);
+		if(database && chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1)
+			logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, strerror(errno), errno);
+	}
 }
 
 // int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c
@@ -806,6 +894,10 @@ void getCacheInformation(int *sock)
 
 void FTL_forwarding_failed(struct server *server)
 {
+	// Don't analyze anything if in PRIVACY_NOSTATS mode
+	if(config.privacylevel >= PRIVACY_NOSTATS)
+		return;
+
 	// Save that this query got forwarded to an upstream server
 	enable_thread_lock();
 	char dest[ADDRSTRLEN];
@@ -929,7 +1021,8 @@ static void block_single_domain(char *domain)
 
 	// Get IPv4/v6 addresses for blocking depending on user configures blocking mode
 	prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6);
-	add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, 0);
+	regexlistname = files.regexlist;
+	add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, SRC_REGEX);
 
 	if(debug) logg("Added %s to cache", domain);
 

dnsmasq_interface.h

@@ -16,7 +16,9 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id);
 void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id);
 void FTL_dnssec(int status, int id);
 void FTL_dnsmasq_reload(void);
-void FTL_fork_and_bind_sockets(void);
+void FTL_fork_and_bind_sockets(struct passwd *ent_pw);
+
+void FTL_header_ADbit(unsigned char header4, int id);
 
 void FTL_forwarding_failed(struct server *server);
 int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz);

gc.c

@@ -97,8 +97,9 @@ void *GC_thread(void *val)
 						counters.cached--;
 						overTime[timeidx].cached--;
 						break;
-					case QUERY_BLACKLIST:
-						// Blocked by user's black list
+					case QUERY_BLACKLIST: // exact blocked
+					case QUERY_WILDCARD: // regex blocked (fall through)
+					case QUERY_EXTERNAL_BLOCKED: // blocked by upstream provider (fall through)
 						counters.blocked--;
 						overTime[timeidx].blocked--;
 						domains[domainID].blockedcount--;
@@ -128,7 +129,11 @@ void *GC_thread(void *val)
 					counters.reply_IP--;
 					break;
 
-					default: // Incomplete query, do nothing
+					case REPLY_DOMAIN: // reverse lookup
+					counters.reply_domain--;
+					break;
+
+					default: // Incomplete query or TXT, do nothing
 					break;
 				}
 
@@ -167,6 +172,12 @@ void *GC_thread(void *val)
 			// Have to this outside of the thread lock
 			// to prevent locking of the resolver
 			reresolveHostnames();
+
+			// After storing data in the database for the next time,
+			// we should scan for old entries, which will then be deleted
+			// to free up pages in the database and prevent it from growing
+			// ever larger and larger
+			DBdeleteoldqueries = true;
 		}
 		sleepms(100);
 	}

grep.c

@@ -124,27 +124,23 @@ int countlineswith(const char* str, const char* fname)
 
 void check_blocking_status(void)
 {
-	int disabled = countlineswith("#addn-hosts=/etc/pihole/gravity.list", files.dnsmasqconfig);
-	char *message = "";
+	char* blocking = read_setupVarsconf("BLOCKING_ENABLED");
+	char* message;
 
-	if(disabled < 0)
+	if(blocking == NULL || getSetupVarsBool(blocking))
 	{
-		// Failed to open file -> unknown status
-		blockingstatus = BLOCKING_UNKNOWN;
-		message = "unknown";
+		// Parameter either not present in setupVars.conf
+		// or explicitly set to true
+		blockingstatus = BLOCKING_ENABLED;
+		message = "enabled";
+		clearSetupVarsArray();
 	}
-	else if(disabled > 0)
+	else
 	{
 		// Disabled
 		blockingstatus = BLOCKING_DISABLED;
 		message = "disabled";
 	}
-	else
-	{
-		// Enabled
-		blockingstatus = BLOCKING_ENABLED;
-		message = "enabled";
-	}
 
 	if(debug) logg("Blocking status is %s", message);
 }

main.c

@@ -63,7 +63,7 @@ int main (int argc, char* argv[])
 		db_init();
 
 	// Try to import queries from long-term database if available
-	if(database)
+	if(database && config.DBimport)
 		read_data_from_DB();
 
 	log_counter_info();

regex.c

@@ -24,9 +24,8 @@ static void log_regex_error(char *where, int errcode, int index)
 	size_t length = regerror(errcode, &regex[index], NULL, 0);
 	char *buffer = calloc(length,sizeof(char));
 	(void) regerror (errcode, &regex[index], buffer, length);
-	logg("ERROR %s regex %i: %s (%i)", index, where, buffer, errcode);
+	logg("ERROR %s regex on line %i: %s (%i)", where, index+1, buffer, errcode);
 	free(buffer);
-	free_regex();
 }
 
 static bool init_regex(const char *regexin, int index)
@@ -283,11 +282,6 @@ void read_regex_from_file(void)
 
 		// Compile this regex
 		regexconfigured[i] = init_regex(buffer, i);
-		if(!regexconfigured[i])
-		{
-			logg("Error compiling regex on line %i", i+1);
-			errors++;
-		}
 	}
 
 	// Free allocated memory