From 872775d9654029a2cc5e57b8775d3593f4ad3246 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 30 Jul 2018 11:36:26 +0200 Subject: [PATCH 01/48] For privacylevel > 0, we obfuscate the domain early on by replacing it with "hidden". Unfortunately, this renders regex validation useless as it can only compare the filters against this "hidden" domain. This commit adds a buffer which keeps the domain in unobfuscated form. Signed-off-by: DL6ER --- dnsmasq_interface.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 5ec36fed..0dd655d0 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -57,6 +57,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * return; } + // Store plain text domain in buffer for regex validation + char *domainbuffer = strdup(domain); + // Check and apply possible privacy level rules // We do this immediately on the raw data to avoid any possible leaking get_privacy_level(NULL); @@ -77,6 +80,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * (strcmp(client, "127.0.0.1") == 0 || strcmp(client, "::1") == 0)) { free(domain); + free(domainbuffer); free(client); disable_thread_lock(); return; @@ -115,6 +119,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Return early to avoid accessing querytypedata out of bounds if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); free(domain); + free(domainbuffer); free(client); disable_thread_lock(); return; @@ -126,12 +131,13 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * overTime[timeidx].querytypedata[querytype-1]++; counters.querytype[querytype-1]++; - // Skip rest of the analyis if this query is not of type A or AAAA + // Skip rest of the analysis if this query is not of type A or AAAA if(querytype != TYPE_A && querytype != TYPE_AAAA) { // Don't process this query further here, we already counted it if(debug) logg("Notice: Skipping new query: %s (%i)", types, id); free(domain); + free(domainbuffer); free(client); disable_thread_lock(); return; @@ -190,10 +196,10 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // of a specific domain. The logic herein is: // If matched, then compare against whitelist // If in whitelist, negate matched so this function returns: not-to-be-blocked - if(match_regex(domain) && !in_whitelist(domain)) + if(match_regex(domainbuffer) && !in_whitelist(domainbuffer)) { // We have to block this domain - block_single_domain(domain); + block_single_domain(domainbuffer); domains[domainID].regexmatch = REGEX_BLOCKED; } else @@ -207,6 +213,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Free allocated memory free(client); free(domain); + free(domainbuffer); // Release thread lock disable_thread_lock(); From 805e373d6592439e225ba8897ece330b8e5ed04f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 31 Jul 2018 11:08:37 +0200 Subject: [PATCH 02/48] Run DB cleaning once GC is finished (defaults to once per hour) Signed-off-by: DL6ER --- gc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gc.c b/gc.c index 26a125e5..66225312 100644 --- a/gc.c +++ b/gc.c @@ -167,6 +167,12 @@ void *GC_thread(void *val) // Have to this outside of the thread lock // to prevent locking of the resolver reresolveHostnames(); + + // After storing data in the database for the next time, + // we should scan for old entries, which will then be deleted + // to free up pages in the database and prevent it from growing + // ever larger and larger + DBdeleteoldqueries = true; } sleepms(100); } From 201cdfc9be318dd454e0bc7597471a9e94f5ba27 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Aug 2018 14:19:33 +0200 Subject: [PATCH 03/48] Fix incorrect log_regex_error() subroutine Signed-off-by: DL6ER --- regex.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/regex.c b/regex.c index 8497bbfb..67acff56 100644 --- a/regex.c +++ b/regex.c @@ -24,9 +24,8 @@ static void log_regex_error(char *where, int errcode, int index) size_t length = regerror(errcode, ®ex[index], NULL, 0); char *buffer = calloc(length,sizeof(char)); (void) regerror (errcode, ®ex[index], buffer, length); - logg("ERROR %s regex %i: %s (%i)", index, where, buffer, errcode); + logg("ERROR %s regex %i: %s (%i)", where, index+1, buffer, errcode); free(buffer); - free_regex(); } static bool init_regex(const char *regexin, int index) From e7c7ff5f10d5be9c8ee41e57db2dbe32db4f6e63 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Aug 2018 14:26:17 +0200 Subject: [PATCH 04/48] Make more clear that the number is the line of the regex in regex.list Signed-off-by: DL6ER --- regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regex.c b/regex.c index 67acff56..9b8535f0 100644 --- a/regex.c +++ b/regex.c @@ -24,7 +24,7 @@ static void log_regex_error(char *where, int errcode, int index) size_t length = regerror(errcode, ®ex[index], NULL, 0); char *buffer = calloc(length,sizeof(char)); (void) regerror (errcode, ®ex[index], buffer, length); - logg("ERROR %s regex %i: %s (%i)", where, index+1, buffer, errcode); + logg("ERROR %s regex in line %i: %s (%i)", where, index+1, buffer, errcode); free(buffer); } From 12e6c6dd8a5ed775a1d704cdf0a258c8231ac93f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Aug 2018 14:31:24 +0200 Subject: [PATCH 05/48] it should be "regex on line ..." not "regex in line ..." + avoid double printing of the error Signed-off-by: DL6ER --- regex.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/regex.c b/regex.c index 9b8535f0..c2985583 100644 --- a/regex.c +++ b/regex.c @@ -24,7 +24,7 @@ static void log_regex_error(char *where, int errcode, int index) size_t length = regerror(errcode, ®ex[index], NULL, 0); char *buffer = calloc(length,sizeof(char)); (void) regerror (errcode, ®ex[index], buffer, length); - logg("ERROR %s regex in line %i: %s (%i)", where, index+1, buffer, errcode); + logg("ERROR %s regex on line %i: %s (%i)", where, index+1, buffer, errcode); free(buffer); } @@ -282,11 +282,6 @@ void read_regex_from_file(void) // Compile this regex regexconfigured[i] = init_regex(buffer, i); - if(!regexconfigured[i]) - { - logg("Error compiling regex on line %i", i+1); - errors++; - } } // Free allocated memory From 349ba432d98d1485f14ffaed6cbf38e589ba4888 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 5 Aug 2018 15:20:39 -0500 Subject: [PATCH 06/48] Run CI on version tags This fixes the v4.0 version issue, where a build was not triggered by the v4.0 tag. Signed-off-by: Mcat12 --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb118793..b0e96af1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,6 +61,9 @@ jobs: workflows: version: 2 build: + filters: + tags: + only: /^v.*/ jobs: - arm - armhf From 2e8f102092017bd3467abed9c0e5b12ad92e29c3 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 5 Aug 2018 15:36:21 -0500 Subject: [PATCH 07/48] Fix config Signed-off-by: Mcat12 --- .circleci/config.yml | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0e96af1..d4dd8127 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,13 +61,26 @@ jobs: workflows: version: 2 build: - filters: - tags: - only: /^v.*/ jobs: - - arm - - armhf - - aarch64 - - x86_64 + - arm: + filters: + tags: + only: /^v.*/ + - armhf: + filters: + tags: + only: /^v.*/ + - aarch64: + filters: + tags: + only: /^v.*/ + - x86_64: + filters: + tags: + only: /^v.*/ # - x86_64-musl - - x86_32 + - x86_32: + filters: + tags: + only: /^v.*/ + From c30356cc8410595d7530ab1b0a696ef6bc5a483a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 11:29:25 +0200 Subject: [PATCH 08/48] Implement detection for when a local configuration is a blocking instruction (in the legacy wildcard sense) Signed-off-by: DL6ER --- dnsmasq_interface.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 0dd655d0..5cb44d9e 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -376,16 +376,18 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); } + // Extract answer (used e.g. for detecting if a local config is a user-defined + // wildcard blocking entry in form "server=/tobeblocked.com/") + char *answer = dest; + if(flags & F_CNAME) + answer = "(CNAME)"; + else if((flags & F_NEG) && (flags & F_NXDOMAIN)) + answer = "(NXDOMAIN)"; + else if(flags & F_NEG) + answer = "(NODATA)"; + if(debug) { - char *answer = dest; - if(flags & F_CNAME) - answer = "(CNAME)"; - else if((flags & F_NEG) && (flags & F_NXDOMAIN)) - answer = "(NXDOMAIN)"; - else if(flags & F_NEG) - answer = "(NODATA)"; - logg("**** got reply %s is %s (ID %i)", name, answer, id); print_flags(flags); } @@ -429,7 +431,13 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) counters.unknown--; // Answered from a custom (user provided) cache file counters.cached++; - queries[i].status = QUERY_CACHE; + + if(strcmp(answer, "(NXDOMAIN)") == 0 || + strcmp(answer, "0.0.0.0") == 0 || + strcmp(answer, "::") == 0) + queries[i].status = QUERY_WILDCARD; + else + queries[i].status = QUERY_CACHE; // Get time index int querytimestamp, overTimetimestamp; From 55285056741dd3c2b350a89a7fec49671d5576e0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 11:46:15 +0200 Subject: [PATCH 09/48] Determine query type earlier on Signed-off-by: DL6ER --- dnsmasq_interface.c | 53 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 0dd655d0..58c106a4 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -32,8 +32,32 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * struct timeval request; gettimeofday(&request, 0); + // Determine query type + unsigned char querytype = 0; + if(strcmp(types,"query[A]") == 0) + querytype = TYPE_A; + else if(strcmp(types,"query[AAAA]") == 0) + querytype = TYPE_AAAA; + else if(strcmp(types,"query[ANY]") == 0) + querytype = TYPE_ANY; + else if(strcmp(types,"query[SRV]") == 0) + querytype = TYPE_SRV; + else if(strcmp(types,"query[SOA]") == 0) + querytype = TYPE_SOA; + else if(strcmp(types,"query[PTR]") == 0) + querytype = TYPE_PTR; + else if(strcmp(types,"query[TXT]") == 0) + querytype = TYPE_TXT; + else + { + // Return early to avoid accessing querytypedata out of bounds + if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); + disable_thread_lock(); + return; + } + // Skip AAAA queries if user doesn't want to have them analyzed - if(!config.analyze_AAAA && strcmp(types,"query[AAAA]") == 0) + if(!config.analyze_AAAA && querytype == TYPE_AAAA) { if(debug) logg("Not analyzing AAAA query"); disable_thread_lock(); @@ -98,33 +122,6 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * char *proto = (type == UDP) ? "UDP" : "TCP"; if(debug) logg("**** new %s %s \"%s\" from %s (ID %i)", proto, types, domain, client, id); - // Determine query type - unsigned char querytype = 0; - if(strcmp(types,"query[A]") == 0) - querytype = TYPE_A; - else if(strcmp(types,"query[AAAA]") == 0) - querytype = TYPE_AAAA; - else if(strcmp(types,"query[ANY]") == 0) - querytype = TYPE_ANY; - else if(strcmp(types,"query[SRV]") == 0) - querytype = TYPE_SRV; - else if(strcmp(types,"query[SOA]") == 0) - querytype = TYPE_SOA; - else if(strcmp(types,"query[PTR]") == 0) - querytype = TYPE_PTR; - else if(strcmp(types,"query[TXT]") == 0) - querytype = TYPE_TXT; - else - { - // Return early to avoid accessing querytypedata out of bounds - if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); - free(domain); - free(domainbuffer); - free(client); - disable_thread_lock(); - return; - } - // Update counters int timeidx = findOverTimeID(overTimetimestamp); validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); From 64a895d7207a2414d21d081064639794b103b111 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 11:57:41 +0200 Subject: [PATCH 10/48] Allow all query types (not only A and AAAA) during database import Signed-off-by: DL6ER --- database.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database.c b/database.c index 6700c94d..65c869ac 100644 --- a/database.c +++ b/database.c @@ -628,9 +628,9 @@ void read_data_from_DB(void) } int type = sqlite3_column_int(stmt, 2); - if(type != TYPE_A && type != TYPE_AAAA) + if(type < TYPE_A || type >= TYPE_MAX) { - logg("DB warn: TYPE should be either 1 or 2 but not %i", type); + logg("DB warn: TYPE should not be %i", type); continue; } // Don't import AAAA queries from database if the user set From a3ac3bc569845c46b8e0cc69bbe5853ec075be3b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 11:58:08 +0200 Subject: [PATCH 11/48] Be able to return any query type through API Signed-off-by: DL6ER --- api.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api.c b/api.c index 8a9f946d..7435f982 100644 --- a/api.c +++ b/api.c @@ -585,6 +585,7 @@ void getQueryTypes(int *sock) } } +char *querytypes[8] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","UNKN"}; void getAllQueries(char *client_message, int *sock) { @@ -661,7 +662,7 @@ void getAllQueries(char *client_message, int *sock) validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); - char *qtype = (queries[i].type == TYPE_A)? "A" : "AAAA"; + char *qtype = querytypes[queries[i].type - TYPE_A]; // 1 = gravity.list, 4 = wildcard, 5 = black.list if((queries[i].status == QUERY_GRAVITY || From 124f81ee9caea0b956450ea1acb89c8f0cd8d60b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 11:58:34 +0200 Subject: [PATCH 12/48] Remove limitation to only analyze A and AAAA queries Signed-off-by: DL6ER --- dnsmasq_interface.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 58c106a4..8196e6ae 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -128,18 +128,6 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * overTime[timeidx].querytypedata[querytype-1]++; counters.querytype[querytype-1]++; - // Skip rest of the analysis if this query is not of type A or AAAA - if(querytype != TYPE_A && querytype != TYPE_AAAA) - { - // Don't process this query further here, we already counted it - if(debug) logg("Notice: Skipping new query: %s (%i)", types, id); - free(domain); - free(domainbuffer); - free(client); - disable_thread_lock(); - return; - } - // Go through already knows domains and see if it is one of them int domainID = findDomainID(domain); From 835ae441fdfd0e880ec828d1b2f7a85ec1e94e89 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 12:01:20 +0200 Subject: [PATCH 13/48] Correct comment Signed-off-by: DL6ER --- dnsmasq_interface.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 8196e6ae..11d7f862 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -227,8 +227,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // find the correct query with zero iterations, but it may happen that queries are processed // asynchronously, e.g. for slow upstream relies to a huge amount of requests. // We iterate from the most recent query down to at most MAXITER queries in the past to avoid - // iterating through the entire array of queries when queries that have not been recorded - // (like PTR queries, etc.) are processed. + // iterating through the entire array of queries // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) // Validate access only once for the maximum index (all lower will work) @@ -629,7 +628,7 @@ void FTL_dnssec(int status, int id) { // Process DNSSEC result for a domain enable_thread_lock(); - // Search for corresponding query indentified by ID + // Search for corresponding query identified by ID bool found = false; int i; // Search match in known queries From 1edbd994b5552e94bee0a59f4d9009bbf71b8794 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 17:05:57 +0200 Subject: [PATCH 14/48] Also call FTL_cache(...) for the other query types (so far, we only called the routine for A and AAAA queries) Signed-off-by: DL6ER --- dnsmasq/rfc1035.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dnsmasq/rfc1035.c b/dnsmasq/rfc1035.c index ab34f351..ee61d411 100644 --- a/dnsmasq/rfc1035.c +++ b/dnsmasq/rfc1035.c @@ -1341,6 +1341,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, unsigned long ttl = daemon->local_ttl; int ok = 1; log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); #ifndef NO_ID /* Dynamically generate stat record */ if (t->stat != 0) @@ -1372,6 +1373,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, t->class, C_IN, "t", t->len, t->txt)) @@ -1430,6 +1432,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { log_query(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL); + FTL_cache(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL, daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, T_PTR, C_IN, "d", intr->name)) @@ -1443,6 +1446,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); for (ptr = daemon->ptr; ptr; ptr = ptr->next) if (hostname_isequal(name, ptr->name) && add_resource_record(header, limit, &trunc, nameoffset, &ansp, @@ -1479,6 +1483,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, nxdomain = 1; if (!dryrun) log_query(crecp->flags & ~F_FORWARD, name, &addr, NULL); + FTL_cache(crecp->flags & ~F_FORWARD, name, &addr, NULL, daemon->log_display_id); } else { @@ -1488,6 +1493,8 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, record_source(crecp->uid)); + FTL_cache(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, + record_source(crecp->uid), daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), NULL, @@ -1505,6 +1512,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL); + FTL_cache(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL, daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, @@ -1548,8 +1556,12 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, sec_data = 0; nxdomain = 1; if (!dryrun) + { log_query(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN, name, &addr, NULL); + FTL_cache(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN, + name, &addr, NULL, daemon->log_display_id); + } } } } @@ -1619,6 +1631,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { gotit = 1; log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL); + FTL_cache(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL, daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, type, C_IN, type == T_A ? "4" : "6", &addrlist->addr)) @@ -1628,7 +1641,10 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, } if (!dryrun && !gotit) + { log_query(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL); + FTL_cache(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL, daemon->log_display_id); + } continue; } @@ -1673,6 +1689,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { log_query(crecp->flags, name, NULL, record_source(crecp->uid)); + FTL_cache(crecp->flags, name, NULL, record_source(crecp->uid), daemon->log_display_id); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), &nameoffset, T_CNAME, C_IN, "d", cname_target)) From 2f4dd87c435c762993d59796379f7dde9891eb42 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 17:06:38 +0200 Subject: [PATCH 15/48] Remove more checks that skipped PTR replies Signed-off-by: DL6ER --- dnsmasq_interface.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 11d7f862..ac4c70bc 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -434,7 +434,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) disable_thread_lock(); return; } - else if(flags & F_FORWARD) + else if((flags & F_FORWARD) || (flags & F_REVERSE)) { // Search for corresponding query identified by dnsmasq's ID bool found = false; @@ -470,10 +470,6 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) save_reply_type(flags, i, response); } } - else if(flags & F_REVERSE) - { - if(debug) logg("Skipping result of PTR query"); - } else { logg("*************************** unknown REPLY ***************************"); @@ -515,7 +511,10 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, struct timeval response; gettimeofday(&response, 0); - if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || ((flags & F_NAMEP) && (flags & F_DHCP)) || (flags & F_FORWARD)) + if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || + ((flags & F_NAMEP) && (flags & F_DHCP)) || + (flags & F_FORWARD) || + (flags & F_REVERSE)) { // List data: /etc/pihole/gravity.list, /etc/pihole/black.list, /etc/pihole/local.list, etc. // or @@ -540,6 +539,8 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, requesttype = QUERY_CACHE; else if(flags & F_FORWARD) // cached answer to previously forwarded request requesttype = QUERY_CACHE; + else if(flags & F_REVERSE) // cached answer to reverse request (PTR) + requesttype = QUERY_CACHE; else { logg("*************************** unknown CACHE reply (1) ***************************"); From 111f1602c3ac3a45becac927164ab351b4692ee7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 17:13:53 +0200 Subject: [PATCH 16/48] Add new reply type: REPLY_DOMAIN Signed-off-by: DL6ER --- FTL.h | 3 ++- dnsmasq_interface.c | 6 ++++++ gc.c | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/FTL.h b/FTL.h index c5267aa1..cd841440 100644 --- a/FTL.h +++ b/FTL.h @@ -84,7 +84,7 @@ enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD }; enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN }; enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST }; enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX }; -enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP }; +enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN }; enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM }; enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA }; enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; @@ -134,6 +134,7 @@ typedef struct { int reply_NXDOMAIN; int reply_CNAME; int reply_IP; + int reply_domain; } countersStruct; typedef struct { diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index ac4c70bc..427c276c 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -710,6 +710,12 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response) queries[queryID].reply = REPLY_CNAME; counters.reply_CNAME++; } + else if(flags & F_REVERSE) + { + // reserve lookup + queries[queryID].reply = REPLY_DOMAIN; + counters.reply_domain++; + } else { // Valid IP diff --git a/gc.c b/gc.c index 66225312..bdf13d9d 100644 --- a/gc.c +++ b/gc.c @@ -128,6 +128,10 @@ void *GC_thread(void *val) counters.reply_IP--; break; + case REPLY_DOMAIN: // reverse lookup + counters.reply_domain--; + break; + default: // Incomplete query, do nothing break; } From 1f360dc3f92e3789338bb59f68b6583aed96be61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 17:14:13 +0200 Subject: [PATCH 17/48] Don't forget to GC wildcard blocked entries Signed-off-by: DL6ER --- gc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gc.c b/gc.c index bdf13d9d..c0d74803 100644 --- a/gc.c +++ b/gc.c @@ -97,8 +97,8 @@ void *GC_thread(void *val) counters.cached--; overTime[timeidx].cached--; break; - case QUERY_BLACKLIST: - // Blocked by user's black list + case QUERY_BLACKLIST: // exact blocked + case QUERY_WILDCARD: // regex blocked (fall through) counters.blocked--; overTime[timeidx].blocked--; domains[domainID].blockedcount--; From baeceeb7d87ba5ca2babb17563eb06d0358db7a3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 17:31:10 +0200 Subject: [PATCH 18/48] Extend FTL_reply(...). Meanwhile, I removed some code duplication and fixed the behavior to avoid multiple counting of query types if there are multiple replies (for instance, a PTR query can return multiple host names for a given IP address) Signed-off-by: DL6ER --- dnsmasq_interface.c | 126 +++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 73 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 427c276c..8e051380 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -378,90 +378,65 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) struct timeval response; gettimeofday(&response, 0); - if(flags & F_CONFIG) + // Save status in corresponding query identified by dnsmasq's ID + bool found = false; + int i; + + // Search match in known queries + // See comments in FTL_forwarded() for further details about this loop + validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); + int until = MAX(0, counters.queries-MAXITER); + for(i = counters.queries-1; i >= until; i--) { - // Answered from local configuration, might be a wildcard or user-provided - // Save status in corresponding query identified by dnsmasq's ID - bool found = false; - int i; - - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) + // Check UUID of this query + if(queries[i].id == id) { - // Check UUID of this query - if(queries[i].id == id) - { - found = true; - break; - } + found = true; + break; } + } - if(!found) - { - // This may happen e.g. if the original query was a PTR query or "pi.hole" - // as we ignore them altogether - disable_thread_lock(); - return; - } - - if(!queries[i].complete) - { - // This query is no longer unknown - counters.unknown--; - // Answered from a custom (user provided) cache file - counters.cached++; - queries[i].status = QUERY_CACHE; - - // Get time index - int querytimestamp, overTimetimestamp; - gettimestamp(&querytimestamp, &overTimetimestamp); - int timeidx = findOverTimeID(overTimetimestamp); - validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); - - overTime[timeidx].cached++; - - // Save reply type and update individual reply counters - save_reply_type(flags, i, response); - - // Hereby, this query is now fully determined - queries[i].complete = true; - } - - // We are done here + if(!found) + { + // This may happen e.g. if the original query was a PTR query or "pi.hole" + // as we ignore them altogether + if(debug) logg("FTL_reply(): Query %i has not been found", id); disable_thread_lock(); return; } - else if((flags & F_FORWARD) || (flags & F_REVERSE)) + + if(queries[i].reply != REPLY_UNKNOWN) { - // Search for corresponding query identified by dnsmasq's ID - bool found = false; - int i; + // Nothing to be done here + disable_thread_lock(); + return; + } - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check UUID of this query - if(queries[i].id == id) - { - found = true; - break; - } - } + if(flags & F_CONFIG) + { + // Answered from local configuration, might be a wildcard or user-provided + // This query is no longer unknown + counters.unknown--; + // Answered from a custom (user provided) cache file + counters.cached++; + queries[i].status = QUERY_CACHE; - if(!found) - { - // This may happen e.g. if the original query was a PTR query or "pi.hole" - // as we ignore them altogether - disable_thread_lock(); - return; - } + // Get time index + int querytimestamp, overTimetimestamp; + gettimestamp(&querytimestamp, &overTimetimestamp); + int timeidx = findOverTimeID(overTimetimestamp); + validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); + overTime[timeidx].cached++; + + // Save reply type and update individual reply counters + save_reply_type(flags, i, response); + + // Hereby, this query is now fully determined + queries[i].complete = true; + } + else if(flags & F_FORWARD) + { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); if(strcmp(domains[domainID].domain, name) == 0) @@ -470,6 +445,11 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) save_reply_type(flags, i, response); } } + else if(flags & F_REVERSE) + { + // Save reply type and update individual reply counters + save_reply_type(flags, i, response); + } else { logg("*************************** unknown REPLY ***************************"); From 877c81be7ca3698906d635abd3ce0a129ab58a8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 18:08:23 +0200 Subject: [PATCH 19/48] Add RRNAME reply type (needed for TXT queries) Signed-off-by: DL6ER --- FTL.h | 2 +- dnsmasq_interface.c | 10 +++++++++- gc.c | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/FTL.h b/FTL.h index cd841440..44817553 100644 --- a/FTL.h +++ b/FTL.h @@ -84,7 +84,7 @@ enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD }; enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN }; enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST }; enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX }; -enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN }; +enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME }; enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM }; enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA }; enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 8e051380..b3752ca7 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -494,7 +494,8 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || ((flags & F_NAMEP) && (flags & F_DHCP)) || (flags & F_FORWARD) || - (flags & F_REVERSE)) + (flags & F_REVERSE) || + (flags & F_RRNAME)) { // List data: /etc/pihole/gravity.list, /etc/pihole/black.list, /etc/pihole/local.list, etc. // or @@ -521,6 +522,8 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, requesttype = QUERY_CACHE; else if(flags & F_REVERSE) // cached answer to reverse request (PTR) requesttype = QUERY_CACHE; + else if(flags & F_RRNAME) // cached answer to TXT query + requesttype = QUERY_CACHE; else { logg("*************************** unknown CACHE reply (1) ***************************"); @@ -696,6 +699,11 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response) queries[queryID].reply = REPLY_DOMAIN; counters.reply_domain++; } + else if(flags & F_RRNAME) + { + // TXT query + queries[queryID].reply = REPLY_RRNAME; + } else { // Valid IP diff --git a/gc.c b/gc.c index c0d74803..878ec6f7 100644 --- a/gc.c +++ b/gc.c @@ -132,7 +132,7 @@ void *GC_thread(void *val) counters.reply_domain--; break; - default: // Incomplete query, do nothing + default: // Incomplete query or TXT, do nothing break; } From 1a242be979bd0f11677276d23b85fc68d303e812 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Aug 2018 20:28:00 +0200 Subject: [PATCH 20/48] Add "ANALYZE_ONLY_A_AND_AAAA" config option to obtain pre-v4.1 behavior (show and record only A and AAAA queries, ignore all others) Signed-off-by: DL6ER --- FTL.h | 1 + config.c | 13 +++++++++++++ dnsmasq_interface.c | 13 +++++++++++++ 3 files changed, 27 insertions(+) diff --git a/FTL.h b/FTL.h index 44817553..6c52d11f 100644 --- a/FTL.h +++ b/FTL.h @@ -150,6 +150,7 @@ typedef struct { bool ignore_localhost; unsigned char blockingmode; bool regex_debugmode; + bool analyze_only_A_AAAA; } ConfigStruct; // Dynamic structs diff --git a/config.c b/config.c index 4f98c459..fb20da46 100644 --- a/config.c +++ b/config.c @@ -219,6 +219,19 @@ void read_FTLconf(void) else logg(" REGEX_DEBUGMODE: Inactive"); + // ANALYZE_ONLY_A_AND_AAAA + // defaults to: No + config.analyze_only_A_AAAA = false; + buffer = parse_FTLconf(fp, "ANALYZE_ONLY_A_AND_AAAA"); + + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.analyze_only_A_AAAA = true; + + if(config.analyze_only_A_AAAA) + logg(" ANALYZE_ONLY_A_AND_AAAA: Enabled. Analyzing only A and AAAA queries"); + else + logg(" ANALYZE_ONLY_A_AND_AAAA: Disabled. Analyzing all queries"); + logg("Finished config file parsing"); // Release memory diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 42010c05..f7d5f69e 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -128,6 +128,19 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * overTime[timeidx].querytypedata[querytype-1]++; counters.querytype[querytype-1]++; + // Skip rest of the analysis if this query is not of type A or AAAA + // but user wants to see only A and AAAA queried (pre-v4.1 behavior) + if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA) + { + // Don't process this query further here, we already counted it + if(debug) logg("Notice: Skipping new query: %s (%i)", types, id); + free(domain); + free(domainbuffer); + free(client); + disable_thread_lock(); + return; + } + // Go through already knows domains and see if it is one of them int domainID = findDomainID(domain); From 77f3eda52ece16180358c97e94e2a9688c697dc1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Aug 2018 23:41:11 +0200 Subject: [PATCH 21/48] Show "regex.list" as source of regex blocked queries Signed-off-by: DL6ER --- dnsmasq/cache.c | 4 ++++ dnsmasq/dnsmasq.h | 5 ++++- dnsmasq_interface.c | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dnsmasq/cache.c b/dnsmasq/cache.c index 3c0f97e8..aa91057b 100644 --- a/dnsmasq/cache.c +++ b/dnsmasq/cache.c @@ -1534,6 +1534,10 @@ char *record_source(unsigned int index) return "config"; else if (index == SRC_HOSTS) return HOSTSFILE; + /*----- Pi-hole modification -----*/ + else if (index == SRC_REGEX) + return (char*)regexlistname; + /*--------------------------------*/ for (ah = daemon->addn_hosts; ah; ah = ah->next) if (ah->index == index) diff --git a/dnsmasq/dnsmasq.h b/dnsmasq/dnsmasq.h index 10ef7eec..ae6b2995 100644 --- a/dnsmasq/dnsmasq.h +++ b/dnsmasq/dnsmasq.h @@ -464,7 +464,10 @@ struct crec { #define SRC_CONFIG 1 #define SRC_HOSTS 2 #define SRC_AH 3 - +/*----- Pi-hole modification -----*/ +#define SRC_REGEX 4 +const char *regexlistname; +/*--------------------------------*/ /* struct sockaddr is not large enough to hold any address, and specifically not big enough to hold an IPv6 address. diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 5cb44d9e..89f9f62b 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -946,7 +946,8 @@ static void block_single_domain(char *domain) // Get IPv4/v6 addresses for blocking depending on user configures blocking mode prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6); - add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, 0); + regexlistname = files.regexlist; + add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, SRC_REGEX); if(debug) logg("Added %s to cache", domain); From 25042de8f8c232fa1628c6540fb7d8330d65ae65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Aug 2018 00:51:10 +0200 Subject: [PATCH 22/48] Review comments Signed-off-by: DL6ER --- dnsmasq_interface.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 213ba603..4da60aed 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -129,7 +129,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * counters.querytype[querytype-1]++; // Skip rest of the analysis if this query is not of type A or AAAA - // but user wants to see only A and AAAA queried (pre-v4.1 behavior) + // but user wants to see only A and AAAA queries (pre-v4.1 behavior) if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA) { // Don't process this query further here, we already counted it @@ -413,8 +413,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) if(!found) { - // This may happen e.g. if the original query was a PTR query or "pi.hole" - // as we ignore them altogether + // This may happen e.g. if the original query was "pi.hole" if(debug) logg("FTL_reply(): Query %i has not been found", id); disable_thread_lock(); return; @@ -435,7 +434,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) // Answered from a custom (user provided) cache file counters.cached++; - // Detect user-defined blocking rules in .conf files + // Detect user-defined blocking rules if(strcmp(answer, "(NXDOMAIN)") == 0 || strcmp(answer, "0.0.0.0") == 0 || strcmp(answer, "::") == 0) From 679f9d9340d9e3f23ae5f9e819d39730ed616ef6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Aug 2018 14:51:31 +0200 Subject: [PATCH 23/48] Determine blocking status enabled/disabled by reading the new variable BLOCKING in setupVars.conf Signed-off-by: DL6ER --- grep.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/grep.c b/grep.c index 2a656da6..3f3e2166 100644 --- a/grep.c +++ b/grep.c @@ -124,27 +124,23 @@ int countlineswith(const char* str, const char* fname) void check_blocking_status(void) { - int disabled = countlineswith("#addn-hosts=/etc/pihole/gravity.list", files.dnsmasqconfig); - char *message = ""; + char* blocking = read_setupVarsconf("BLOCKING"); + char* message; - if(disabled < 0) + if(blocking == NULL || getSetupVarsBool(blocking)) { - // Failed to open file -> unknown status - blockingstatus = BLOCKING_UNKNOWN; - message = "unknown"; + // Parameter either not present in setupVars.conf + // or explicitly set to true + blockingstatus = BLOCKING_ENABLED; + message = "enable"; + clearSetupVarsArray(); } - else if(disabled > 0) + else { // Disabled blockingstatus = BLOCKING_DISABLED; message = "disabled"; } - else - { - // Enabled - blockingstatus = BLOCKING_ENABLED; - message = "enabled"; - } if(debug) logg("Blocking status is %s", message); } From c97f490e249eab1ff792cfd7d2f7bdabb602b749 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Aug 2018 14:55:08 +0200 Subject: [PATCH 24/48] Fix tense Signed-off-by: DL6ER --- grep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grep.c b/grep.c index 3f3e2166..88fda536 100644 --- a/grep.c +++ b/grep.c @@ -132,7 +132,7 @@ void check_blocking_status(void) // Parameter either not present in setupVars.conf // or explicitly set to true blockingstatus = BLOCKING_ENABLED; - message = "enable"; + message = "enabled"; clearSetupVarsArray(); } else From 716c9782f734d9c6c5033e9434bab532728ef198 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Aug 2018 17:21:55 +0200 Subject: [PATCH 25/48] BLOCKING -> BLOCKING_ENABLED Signed-off-by: DL6ER --- grep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grep.c b/grep.c index 88fda536..95551e5b 100644 --- a/grep.c +++ b/grep.c @@ -124,7 +124,7 @@ int countlineswith(const char* str, const char* fname) void check_blocking_status(void) { - char* blocking = read_setupVarsconf("BLOCKING"); + char* blocking = read_setupVarsconf("BLOCKING_ENABLED"); char* message; if(blocking == NULL || getSetupVarsBool(blocking)) From a5a491b4f9e7bb4fdfd8c3023c97c4d09fba2924 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Aug 2018 15:26:59 +0200 Subject: [PATCH 26/48] Add query type forting for ">getallqueries" request Signed-off-by: DL6ER --- api.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api.c b/api.c index 7435f982..f01e7923 100644 --- a/api.c +++ b/api.c @@ -603,6 +603,8 @@ void getAllQueries(char *client_message, int *sock) char *clientname = NULL; bool filterclientname = false; + int querytype = 0; + // Time filtering? if(command(client_message, ">getallqueries-time")) { sscanf(client_message, ">getallqueries-time %i %i",&from, &until); @@ -623,6 +625,17 @@ void getAllQueries(char *client_message, int *sock) sscanf(client_message, ">getallqueries-client %255s", clientname); filterclientname = true; } + // Query type filtering? + if(command(client_message, ">getallqueries-qtype")) { + // Get query type we want to see only + sscanf(client_message, ">getallqueries-qtype %i", &querytype); + if(querytype < 1 || querytype >= TYPE_MAX) + { + // Invalid query type requested + querytype = 0; + } + logg("Requesting query type %i", querytype); + } int ibeg = 0, num; // Test for integer that specifies number of entries to be shown @@ -694,6 +707,9 @@ void getAllQueries(char *client_message, int *sock) continue; } + if(querytype != 0 && querytype != queries[i].type) + continue; + char *domain = domains[queries[i].domainID].domain; char *client; if(clients[queries[i].clientID].name != NULL && From 2b27f8a928ed5e6ea0a22b427e744ccdc052ab83 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Aug 2018 17:34:53 +0200 Subject: [PATCH 27/48] Implement forward destination filtering Signed-off-by: DL6ER --- api.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/api.c b/api.c index f01e7923..2758c7fd 100644 --- a/api.c +++ b/api.c @@ -605,10 +605,54 @@ void getAllQueries(char *client_message, int *sock) int querytype = 0; + char *forwarddest = NULL; + bool filterforwarddest = false; + int forwarddestid = 0; + // Time filtering? if(command(client_message, ">getallqueries-time")) { sscanf(client_message, ">getallqueries-time %i %i",&from, &until); } + + // Forward destination filtering? + if(command(client_message, ">getallqueries-forward")) { + // Get forward destination name we want to see only (limit length to 255 chars) + forwarddest = calloc(256, sizeof(char)); + if(forwarddest == NULL) return; + sscanf(client_message, ">getallqueries-forward %255s", forwarddest); + filterforwarddest = true; + + if(strcmp(forwarddest, "cache") == 0) + forwarddestid = -1; + else if(strcmp(forwarddest, "blocklist") == 0) + forwarddestid = -2; + else + { + // Iterate through all known forward destinations + int i; + forwarddestid = -3; + for(i = 0; i < counters.forwarded; i++) + { + // Try to match the requested string against their IP addresses and + // (if available) their host names + if(strcmp(forwarded[i].ip, forwarddest) == 0 || + (forwarded[i].name != NULL && + strcmp(forwarded[i].name, forwarddest) == 0)) + { + forwarddestid = i; + break; + } + } + if(forwarddestid < 0) + { + // Requested forward destination has not been found, we directly + // exit here as there is no data to be returned + free(forwarddest); + return; + } + } + } + // Domain filtering? if(command(client_message, ">getallqueries-domain")) { // Get domain name we want to see only (limit length to 255 chars) @@ -617,6 +661,7 @@ void getAllQueries(char *client_message, int *sock) sscanf(client_message, ">getallqueries-domain %255s", domainname); filterdomainname = true; } + // Client filtering? if(command(client_message, ">getallqueries-client")) { // Get client name we want to see only (limit length to 255 chars) @@ -625,6 +670,7 @@ void getAllQueries(char *client_message, int *sock) sscanf(client_message, ">getallqueries-client %255s", clientname); filterclientname = true; } + // Query type filtering? if(command(client_message, ">getallqueries-qtype")) { // Get query type we want to see only @@ -634,7 +680,6 @@ void getAllQueries(char *client_message, int *sock) // Invalid query type requested querytype = 0; } - logg("Requesting query type %i", querytype); } int ibeg = 0, num; @@ -710,6 +755,21 @@ void getAllQueries(char *client_message, int *sock) if(querytype != 0 && querytype != queries[i].type) continue; + if(filterforwarddest) + { + // Does the user want to see queries answered from blocking lists? + if(forwarddestid == -2 && queries[i].status != QUERY_GRAVITY + && queries[i].status != QUERY_WILDCARD + && queries[i].status != QUERY_BLACKLIST) + continue; + // Does the use want to see queries answered from local cache? + else if(forwarddestid == -1 && queries[i].status != QUERY_CACHE) + continue; + // Does the user want to see queries answered by an upstream server? + else if(forwarddestid >= 0 && forwarddestid != queries[i].forwardID) + continue; + } + char *domain = domains[queries[i].domainID].domain; char *client; if(clients[queries[i].clientID].name != NULL && @@ -750,6 +810,9 @@ void getAllQueries(char *client_message, int *sock) if(filterdomainname) free(domainname); + + if(filterforwarddest) + free(forwarddest); } void getRecentBlocked(char *client_message, int *sock) From 79b2865bab032cc9dac4712f031a6b294a62ea01 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Aug 2018 23:45:27 +0200 Subject: [PATCH 28/48] Add new query type QUERY_EXTERNAL_BLOCKED triggered by an NXDOMAIN response with set AD bit (this is how Quad9 signals a blocked domain) Signed-off-by: DL6ER --- FTL.h | 3 ++- database.c | 13 +++++---- dnsmasq/forward.c | 2 ++ dnsmasq_interface.c | 64 +++++++++++++++++++++++++++++++++++++++++++++ dnsmasq_interface.h | 2 ++ gc.c | 1 + 6 files changed, 79 insertions(+), 6 deletions(-) diff --git a/FTL.h b/FTL.h index 6c52d11f..11295255 100644 --- a/FTL.h +++ b/FTL.h @@ -82,7 +82,7 @@ enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER }; enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD }; enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN }; -enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST }; +enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED }; enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX }; enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME }; enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM }; @@ -170,6 +170,7 @@ typedef struct { unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.) unsigned char reply; unsigned char dnssec; + bool AD; } queriesDataStruct; typedef struct { diff --git a/database.c b/database.c index 65c869ac..4d9ba418 100644 --- a/database.c +++ b/database.c @@ -452,9 +452,10 @@ void save_to_DB(void) // Total counter information (delta computation) total++; - if(queries[i].status == 1 || - queries[i].status == 4 || - queries[i].status == 5) + if(queries[i].status == QUERY_GRAVITY || + queries[i].status == QUERY_BLACKLIST || + queries[i].status == QUERY_WILDCARD || + queries[i].status == QUERY_EXTERNAL_BLOCKED) blocked++; // Update lasttimestamp variable with timestamp of the latest stored query @@ -641,9 +642,9 @@ void read_data_from_DB(void) } int status = sqlite3_column_int(stmt, 3); - if(status < QUERY_UNKNOWN || status > QUERY_BLACKLIST) + if(status < QUERY_UNKNOWN || status > QUERY_EXTERNAL_BLOCKED) { - logg("DB warn: STATUS should be within [0,5] but is %i", status); + logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_EXTERNAL_BLOCKED, status); continue; } @@ -704,6 +705,7 @@ void read_data_from_DB(void) queries[queryID].id = 0; // This is dnsmasq's internal ID. We don't store it in the database queries[queryID].complete = true; // Mark as all information is avaiable queries[queryID].response = 0; + queries[queryID].AD = false; lastDBimportedtimestamp = queryTimeStamp; // Handle type counters @@ -733,6 +735,7 @@ void read_data_from_DB(void) case QUERY_GRAVITY: // Blocked by gravity.list case QUERY_WILDCARD: // Blocked by regex filter case QUERY_BLACKLIST: // Blocked by black.list + case QUERY_EXTERNAL_BLOCKED: // Blocked by external provider counters.blocked++; overTime[timeidx].blocked++; domains[domainID].blockedcount++; diff --git a/dnsmasq/forward.c b/dnsmasq/forward.c index 31bd71b4..bff98f37 100644 --- a/dnsmasq/forward.c +++ b/dnsmasq/forward.c @@ -648,6 +648,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server } } + FTL_header_ADbit(header->hb4, daemon->log_display_id); + /* RFC 4035 sect 4.6 para 3 */ if (!is_sign && !option_bool(OPT_DNSSEC_PROXY)) header->hb4 &= ~HB4_AD; diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 4da60aed..0cc8f452 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -165,6 +165,8 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * queries[queryID].reply = REPLY_UNKNOWN; // Store DNSSEC result for this domain queries[queryID].dnssec = DNSSEC_UNSPECIFIED; + // AD has not yet been received for this query + queries[queryID].AD = false; // Increase DNS queries counter counters.queries++; @@ -460,10 +462,31 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); + if(strcmp(domains[domainID].domain, name) == 0) { // Save reply type and update individual reply counters save_reply_type(flags, i, response); + + // If received NXDOMAIN and AD bit is set, Quad9 may have blocked this query + if(flags & F_NXDOMAIN && queries[i].AD) + { + // Correct counters as we won't count this as forwarded ... + counters.forwarded--; + overTime[queries[i].timeidx].forwarded--; + validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); + forwarded[queries[i].forwardID].count--; + + // ... but as blocked + counters.blocked++; + overTime[queries[i].timeidx].blocked++; + validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); + domains[queries[i].domainID].blockedcount++; + validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); + clients[queries[i].clientID].blockedcount++; + + queries[i].status = QUERY_EXTERNAL_BLOCKED; + } } } else if(flags & F_REVERSE) @@ -676,6 +699,47 @@ void FTL_dnssec(int status, int id) disable_thread_lock(); } +void FTL_header_ADbit(unsigned char header4, int id) +{ + enable_thread_lock(); + // Check if AD bit is set in DNS header + if(!(header4 & 0x20)) + { + // AD bit not set + disable_thread_lock(); + return; + } + + // Search for corresponding query identified by ID + bool found = false; + int i; + // Search match in known queries + // See comments in FTL_forwarded() for further details about this loop + validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); + int until = MAX(0, counters.queries-MAXITER); + for(i = counters.queries-1; i >= until; i--) + { + // Check both UUID and generation of this query + if(queries[i].id == id) + { + found = true; + break; + } + } + + if(!found) + { + // This may happen e.g. if the original query was an unhandled query type + disable_thread_lock(); + return; + } + + // Store AD bit in query data + queries[i].AD = true; + + disable_thread_lock(); +} + void print_flags(unsigned int flags) { // Debug function, listing resolver flags in clear text diff --git a/dnsmasq_interface.h b/dnsmasq_interface.h index 7872be16..a2c4c3bf 100644 --- a/dnsmasq_interface.h +++ b/dnsmasq_interface.h @@ -18,5 +18,7 @@ void FTL_dnssec(int status, int id); void FTL_dnsmasq_reload(void); void FTL_fork_and_bind_sockets(void); +void FTL_header_ADbit(unsigned char header4, int id); + void FTL_forwarding_failed(struct server *server); int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz); diff --git a/gc.c b/gc.c index 878ec6f7..658d7096 100644 --- a/gc.c +++ b/gc.c @@ -99,6 +99,7 @@ void *GC_thread(void *val) break; case QUERY_BLACKLIST: // exact blocked case QUERY_WILDCARD: // regex blocked (fall through) + case QUERY_EXTERNAL_BLOCKED: // blocked by upstream provider (fall through) counters.blocked--; overTime[timeidx].blocked--; domains[domainID].blockedcount--; From 8eb10efa0cf5e1b4c7b1d2330a6ec7a13ff636aa Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 16 Aug 2018 11:04:59 -0400 Subject: [PATCH 29/48] Fix missing branch on tag builds Fixes #344 Signed-off-by: Mcat12 --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4dd8127..1a93b535 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,10 @@ version: 2 - run: name: "Build" command: | - make CFLAGS="${CFLAGS}" GIT_BRANCH="${CIRCLE_BRANCH}" GIT_TAG="${CIRCLE_TAG}" + BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master") + echo "export BRANCH=$BRANCH" >> $BASH_ENV + + make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" file pihole-FTL - run: name: "Upload" @@ -21,8 +24,8 @@ version: 2 sha1sum pihole-FTL-* > ${BIN_NAME}.sha1 wget https://ftl.pi-hole.net:8080/FTL-client chmod +x ./FTL-client - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${CIRCLE_BRANCH}" "${BIN_NAME}" "${FTL_SECRET}" - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${CIRCLE_BRANCH}" "${BIN_NAME}.sha1" "${FTL_SECRET}" + [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${BRANCH}" "${BIN_NAME}" "${FTL_SECRET}" + [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${BRANCH}" "${BIN_NAME}.sha1" "${FTL_SECRET}" rm ./FTL-client ls -lah . From 8c14a6fa6bb8b4db283cbf5843dacfb745c509e5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 16 Aug 2018 11:08:40 -0400 Subject: [PATCH 30/48] Put tag builds in a folder named after the tag Signed-off-by: Mcat12 --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a93b535..32d37564 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,19 +13,19 @@ version: 2 name: "Build" command: | BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master") - echo "export BRANCH=$BRANCH" >> $BASH_ENV make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" file pihole-FTL - run: name: "Upload" command: | + FOLDER=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "$CIRCLE_TAG") mv pihole-FTL "${BIN_NAME}" sha1sum pihole-FTL-* > ${BIN_NAME}.sha1 wget https://ftl.pi-hole.net:8080/FTL-client chmod +x ./FTL-client - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${BRANCH}" "${BIN_NAME}" "${FTL_SECRET}" - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${BRANCH}" "${BIN_NAME}.sha1" "${FTL_SECRET}" + [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}" "${FTL_SECRET}" + [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}.sha1" "${FTL_SECRET}" rm ./FTL-client ls -lah . From c0bda36e1657bc5e6fe80b18f3e0d3df6b7d61a2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 16 Aug 2018 18:19:00 +0200 Subject: [PATCH 31/48] Don't return any data if user requested an invalid query type in getallqueries Signed-off-by: DL6ER --- api.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/api.c b/api.c index 2758c7fd..c73643b5 100644 --- a/api.c +++ b/api.c @@ -614,6 +614,17 @@ void getAllQueries(char *client_message, int *sock) sscanf(client_message, ">getallqueries-time %i %i",&from, &until); } + // Query type filtering? + if(command(client_message, ">getallqueries-qtype")) { + // Get query type we want to see only + sscanf(client_message, ">getallqueries-qtype %i", &querytype); + if(querytype < 1 || querytype >= TYPE_MAX) + { + // Invalid query type requested + return; + } + } + // Forward destination filtering? if(command(client_message, ">getallqueries-forward")) { // Get forward destination name we want to see only (limit length to 255 chars) @@ -671,17 +682,6 @@ void getAllQueries(char *client_message, int *sock) filterclientname = true; } - // Query type filtering? - if(command(client_message, ">getallqueries-qtype")) { - // Get query type we want to see only - sscanf(client_message, ">getallqueries-qtype %i", &querytype); - if(querytype < 1 || querytype >= TYPE_MAX) - { - // Invalid query type requested - querytype = 0; - } - } - int ibeg = 0, num; // Test for integer that specifies number of entries to be shown if(sscanf(client_message, "%*[^(](%i)", &num) > 0) From c5490410c4b26efecb334ba322ae1f63bfbdcbe2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 16 Aug 2018 18:19:44 +0200 Subject: [PATCH 32/48] Fix typo in comment Signed-off-by: DL6ER --- api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.c b/api.c index c73643b5..162c782c 100644 --- a/api.c +++ b/api.c @@ -762,7 +762,7 @@ void getAllQueries(char *client_message, int *sock) && queries[i].status != QUERY_WILDCARD && queries[i].status != QUERY_BLACKLIST) continue; - // Does the use want to see queries answered from local cache? + // Does the user want to see queries answered from local cache? else if(forwarddestid == -1 && queries[i].status != QUERY_CACHE) continue; // Does the user want to see queries answered by an upstream server? From 5fa70ec534cc49cf3ba7add49189dfaf7a347146 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 17 Aug 2018 18:44:12 +0200 Subject: [PATCH 33/48] Outsource code into query_externally_blocked() to avoid duplication Signed-off-by: DL6ER --- dnsmasq_interface.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 0cc8f452..fe2908f8 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -17,6 +17,7 @@ void print_flags(unsigned int flags); void save_reply_type(unsigned int flags, int queryID, struct timeval response); unsigned long converttimeval(struct timeval time); static void block_single_domain(char *domain); +static void query_externally_blocked(int i); char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "}; @@ -470,23 +471,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) // If received NXDOMAIN and AD bit is set, Quad9 may have blocked this query if(flags & F_NXDOMAIN && queries[i].AD) - { - // Correct counters as we won't count this as forwarded ... - counters.forwarded--; - overTime[queries[i].timeidx].forwarded--; - validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); - forwarded[queries[i].forwardID].count--; - - // ... but as blocked - counters.blocked++; - overTime[queries[i].timeidx].blocked++; - validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); - domains[queries[i].domainID].blockedcount++; - validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); - clients[queries[i].clientID].blockedcount++; - - queries[i].status = QUERY_EXTERNAL_BLOCKED; - } + query_externally_blocked(i); } } else if(flags & F_REVERSE) @@ -503,6 +488,25 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) disable_thread_lock(); } +static void query_externally_blocked(int i) +{ + // Correct counters as we won't count this as forwarded ... + counters.forwarded--; + overTime[queries[i].timeidx].forwarded--; + validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); + forwarded[queries[i].forwardID].count--; + + // ... but as blocked + counters.blocked++; + overTime[queries[i].timeidx].blocked++; + validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); + domains[queries[i].domainID].blockedcount++; + validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); + clients[queries[i].clientID].blockedcount++; + + queries[i].status = QUERY_EXTERNAL_BLOCKED; +} + void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id) { // Save that this query got answered from cache From a426b8d6cd30fd2a04550cb23b3116024ef7376f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 17 Aug 2018 18:56:21 +0200 Subject: [PATCH 34/48] Detect externally blocked state based on returned IP address (OpenDNS blocking page IPs) Signed-off-by: DL6ER --- dnsmasq_interface.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index fe2908f8..cbf03270 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -471,7 +471,37 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) // If received NXDOMAIN and AD bit is set, Quad9 may have blocked this query if(flags & F_NXDOMAIN && queries[i].AD) + { query_externally_blocked(i); + } + + // If received one of the following IPs as reply, OpenDNS + // (Cisco Umbrella) blocked this query + // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- + // for a full list of these IP addresses + else if(flags & F_IPV4 && + (strcmp("146.112.61.104", answer) == 0 || + strcmp("146.112.61.105", answer) == 0 || + strcmp("146.112.61.106", answer) == 0 || + strcmp("146.112.61.107", answer) == 0 || + strcmp("146.112.61.108", answer) == 0 || + strcmp("146.112.61.109", answer) == 0 || + strcmp("146.112.61.110", answer) == 0)) + { + query_externally_blocked(i); + } + + else if(flags & F_IPV6 && + (strcmp("::ffff:146.112.61.104", answer) == 0 || + strcmp("::ffff:146.112.61.105", answer) == 0 || + strcmp("::ffff:146.112.61.106", answer) == 0 || + strcmp("::ffff:146.112.61.107", answer) == 0 || + strcmp("::ffff:146.112.61.108", answer) == 0 || + strcmp("::ffff:146.112.61.109", answer) == 0 || + strcmp("::ffff:146.112.61.110", answer) == 0)) + { + query_externally_blocked(i); + } } } else if(flags & F_REVERSE) From d3351bf64ac3adea68858b25b985caef4100bf8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 17 Aug 2018 22:36:20 +0200 Subject: [PATCH 35/48] Check string against NULL before passing to strcmp() Signed-off-by: DL6ER --- dnsmasq_interface.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index cbf03270..c42ea55b 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -479,26 +479,26 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) // (Cisco Umbrella) blocked this query // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- // for a full list of these IP addresses - else if(flags & F_IPV4 && + else if(flags & F_IPV4 && answer != NULL && (strcmp("146.112.61.104", answer) == 0 || strcmp("146.112.61.105", answer) == 0 || strcmp("146.112.61.106", answer) == 0 || strcmp("146.112.61.107", answer) == 0 || strcmp("146.112.61.108", answer) == 0 || strcmp("146.112.61.109", answer) == 0 || - strcmp("146.112.61.110", answer) == 0)) + strcmp("146.112.61.110", answer) == 0 )) { query_externally_blocked(i); } - else if(flags & F_IPV6 && + else if(flags & F_IPV6 && answer != NULL && (strcmp("::ffff:146.112.61.104", answer) == 0 || strcmp("::ffff:146.112.61.105", answer) == 0 || strcmp("::ffff:146.112.61.106", answer) == 0 || strcmp("::ffff:146.112.61.107", answer) == 0 || strcmp("::ffff:146.112.61.108", answer) == 0 || strcmp("::ffff:146.112.61.109", answer) == 0 || - strcmp("::ffff:146.112.61.110", answer) == 0)) + strcmp("::ffff:146.112.61.110", answer) == 0 )) { query_externally_blocked(i); } From 13deb5e66248ed41df5ab1b428d052b2febff47e Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 18 Aug 2018 16:56:11 -0400 Subject: [PATCH 36/48] Use the pihole Docker namespace Previous namespace was thepihole Signed-off-by: Mcat12 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32d37564..38b49d6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 .job_template: &job_template docker: - - image: thepihole/ftl-build:$CIRCLE_JOB + - image: pihole/ftl-build:$CIRCLE_JOB steps: - checkout - run: From dfe27e7c3bf315aad72de38e0830bb5fbf3b7fde Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Aug 2018 14:14:06 +0200 Subject: [PATCH 37/48] Add PRIVACY_NOSTATS mode that skips all (!) analysis. We furthermore don't even try to import or save anything to the database. Signed-off-by: DL6ER --- FTL.h | 2 +- config.c | 5 +++-- database.c | 8 ++++++++ dnsmasq_interface.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/FTL.h b/FTL.h index 11295255..63bdc6df 100644 --- a/FTL.h +++ b/FTL.h @@ -85,7 +85,7 @@ enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED }; enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX }; enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME }; -enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM }; +enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS }; enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA }; enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN }; diff --git a/config.c b/config.c index fb20da46..f224eb0a 100644 --- a/config.c +++ b/config.c @@ -167,8 +167,9 @@ void read_FTLconf(void) // PRIVACY_HIDE_DOMAINS (1) = show and store all domains as "hidden", return nothing for Top Domains + Top Ads // PRIVACY_HIDE_DOMAINS_CLIENTS (2) = as above, show all domains as "hidden" and all clients as "127.0.0.1" // (or "::1"), return nothing for any Top Lists - // PRIVACY_MAXIMUM (3) = Disabled basically everything except the anonymous stastics, there will be no entries + // PRIVACY_MAXIMUM (3) = Disabled basically everything except the anonymous statistics, there will be no entries // added to the database, no entries visible in the query log and no Top Item Lists + // PRIVACY_NOSTATS (4) = Disable any analysis on queries. No counters are available in this mode. // defaults to: PRIVACY_SHOW_ALL config.privacylevel = PRIVACY_SHOW_ALL; get_privacy_level(fp); @@ -316,7 +317,7 @@ void get_privacy_level(FILE *fp) // Check for change and validity of privacy level (set in FTL.h) if(value != config.privacylevel && value >= PRIVACY_SHOW_ALL && - value <= PRIVACY_MAXIMUM) + value <= PRIVACY_NOSTATS) { logg("Notice: Changing privacy level from %i to %i", config.privacylevel, value); config.privacylevel = value; diff --git a/database.c b/database.c index 4d9ba418..510ff22a 100644 --- a/database.c +++ b/database.c @@ -337,6 +337,10 @@ int get_number_of_queries_in_DB(void) void save_to_DB(void) { + // Don't save anything to the database if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Start database timer if(debug) timer_start(DATABASE_WRITE_TIMER); @@ -575,6 +579,10 @@ void *DB_thread(void *val) // Get most recent 24 hours data from long-term database void read_data_from_DB(void) { + // Don't try to load anything to the database if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Open database file if(!dbopen()) { diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index c42ea55b..4607ed50 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -23,8 +23,13 @@ char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Create new query in data structure enable_thread_lock(); + // Get timestamp int querytimestamp, overTimetimestamp; gettimestamp(&querytimestamp, &overTimetimestamp); @@ -222,6 +227,10 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Save that this query got forwarded to an upstream server enable_thread_lock(); @@ -344,6 +353,7 @@ void FTL_dnsmasq_reload(void) { // This funtion is called by the dnsmasq code on receive of SIGHUP // *before* clearing the cache and rereading the lists + // This is the only hook that is not skipped in PRIVACY_NOSTATS mode // Called when dnsmasq re-reads its config and hosts files // Reset number of blocked domains @@ -367,6 +377,10 @@ void FTL_dnsmasq_reload(void) void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Interpret hosts files that have been read by dnsmasq enable_thread_lock(); // Determine returned result if available @@ -539,6 +553,10 @@ static void query_externally_blocked(int i) void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Save that this query got answered from cache enable_thread_lock(); char dest[ADDRSTRLEN]; dest[0] = '\0'; @@ -688,6 +706,10 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, void FTL_dnssec(int status, int id) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Process DNSSEC result for a domain enable_thread_lock(); // Search for corresponding query identified by ID @@ -735,6 +757,10 @@ void FTL_dnssec(int status, int id) void FTL_header_ADbit(unsigned char header4, int id) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + enable_thread_lock(); // Check if AD bit is set in DNS header if(!(header4 & 0x20)) @@ -913,6 +939,10 @@ void getCacheInformation(int *sock) void FTL_forwarding_failed(struct server *server) { + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + // Save that this query got forwarded to an upstream server enable_thread_lock(); char dest[ADDRSTRLEN]; From 0e3fcc1737e3b84fe715dc10ba1be7247679d4cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 11:33:22 +0200 Subject: [PATCH 38/48] Add DBIMPORT config boolean that allows skipping the initial import of historic information from the database. This may come in handy on very large networks where the startup of FTL is large delayed when it imports millions of queries. Signed-off-by: DL6ER --- FTL.h | 1 + config.c | 11 +++++++++++ main.c | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/FTL.h b/FTL.h index 63bdc6df..979599ef 100644 --- a/FTL.h +++ b/FTL.h @@ -151,6 +151,7 @@ typedef struct { unsigned char blockingmode; bool regex_debugmode; bool analyze_only_A_AAAA; + bool DBimport; } ConfigStruct; // Dynamic structs diff --git a/config.c b/config.c index f224eb0a..5522732f 100644 --- a/config.c +++ b/config.c @@ -233,6 +233,17 @@ void read_FTLconf(void) else logg(" ANALYZE_ONLY_A_AND_AAAA: Disabled. Analyzing all queries"); + // DBIMPORT + // defaults to: Yes + config.DBimport = true; + buffer = parse_FTLconf(fp, "DBIMPORT"); + if(buffer != NULL && strcasecmp(buffer, "no") == 0) + config.DBimport = false; + if(config.DBimport) + logg(" DBIMPORT: Importing history from database"); + else + logg(" DBIMPORT: Not importing history from database"); + logg("Finished config file parsing"); // Release memory diff --git a/main.c b/main.c index 9c9c7a48..16674f95 100644 --- a/main.c +++ b/main.c @@ -56,7 +56,7 @@ int main (int argc, char* argv[]) db_init(); // Try to import queries from long-term database if available - if(database) + if(database && config.DBimport) read_data_from_DB(); log_counter_info(); From e20cee447c8981a6899348527e6eb99b3593871d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 18:30:37 +0200 Subject: [PATCH 39/48] Optimize ">getallqueries-domain" Signed-off-by: DL6ER --- api.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/api.c b/api.c index 162c782c..777b08d8 100644 --- a/api.c +++ b/api.c @@ -599,6 +599,7 @@ void getAllQueries(char *client_message, int *sock) char *domainname = NULL; bool filterdomainname = false; + int domainid = -1; char *clientname = NULL; bool filterclientname = false; @@ -671,6 +672,24 @@ void getAllQueries(char *client_message, int *sock) if(domainname == NULL) return; sscanf(client_message, ">getallqueries-domain %255s", domainname); filterdomainname = true; + // Iterate through all known domains + int i; + for(i = 0; i < counters.domains; i++) + { + // Try to match the requested string + if(strcmp(domains[i].domain, domainname) == 0) + { + domainid = i; + break; + } + } + if(domainid < 0) + { + // Requested domain has not been found, we directly + // exit here as there is no data to be returned + free(domainname); + return; + } } // Client filtering? @@ -738,8 +757,8 @@ void getAllQueries(char *client_message, int *sock) if(filterdomainname) { - // Skip if domain name is not identical with what the user wants to see - if(strcmp(domains[queries[i].domainID].domain, domainname) != 0) + // Skip if domain is not identical with what the user wants to see + if(domainid >= 0 && queries[i].domainID != domainid) continue; } From a725f03fae8101683d6abcb76d2a7eb935b81b2d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 18:33:27 +0200 Subject: [PATCH 40/48] Optimize ">getallqueries-clients" Signed-off-by: DL6ER --- api.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/api.c b/api.c index 777b08d8..a27e3e18 100644 --- a/api.c +++ b/api.c @@ -603,6 +603,7 @@ void getAllQueries(char *client_message, int *sock) char *clientname = NULL; bool filterclientname = false; + int clientid = -1; int querytype = 0; @@ -699,6 +700,25 @@ void getAllQueries(char *client_message, int *sock) if(clientname == NULL) return; sscanf(client_message, ">getallqueries-client %255s", clientname); filterclientname = true; + int i; + for(i = 0; i < counters.clients; i++) + { + // Try to match the requested string + if(strcmp(clients[i].ip, clientname) == 0 || + (clients[i].name != NULL && + strcmp(clients[i].name, clientname) == 0)) + { + clientid = i; + break; + } + } + if(clientid < 0) + { + // Requested client has not been found, we directly + // exit here as there is no data to be returned + free(clientname); + return; + } } int ibeg = 0, num; @@ -755,22 +775,15 @@ void getAllQueries(char *client_message, int *sock) if((from > queries[i].timestamp && from != 0) || (queries[i].timestamp > until && until != 0)) continue; - if(filterdomainname) - { - // Skip if domain is not identical with what the user wants to see - if(domainid >= 0 && queries[i].domainID != domainid) + // Skip if domain is not identical with what the user wants to see + if(filterdomainname && queries[i].domainID != domainid) continue; - } - if(filterclientname) - { - // Skip if client name and IP are not identical with what the user wants to see - if(strcmp(clients[queries[i].clientID].ip, clientname) != 0 && - (clients[queries[i].clientID].name != NULL && - strcmp(clients[queries[i].clientID].name, clientname) != 0)) + // Skip if client name and IP are not identical with what the user wants to see + if(filterclientname && queries[i].domainID != domainid) continue; - } + // Skip if query type is not identical with what the user wants to see if(querytype != 0 && querytype != queries[i].type) continue; From 697f1407865af014e094a38598b34d7e24a4c5bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 18:35:07 +0200 Subject: [PATCH 41/48] Add validate_access() guards Signed-off-by: DL6ER --- api.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api.c b/api.c index a27e3e18..b100d7d3 100644 --- a/api.c +++ b/api.c @@ -643,6 +643,7 @@ void getAllQueries(char *client_message, int *sock) { // Iterate through all known forward destinations int i; + validate_access("forwards", counters.forwarded, true, __LINE__, __FUNCTION__, __FILE__); forwarddestid = -3; for(i = 0; i < counters.forwarded; i++) { @@ -675,6 +676,7 @@ void getAllQueries(char *client_message, int *sock) filterdomainname = true; // Iterate through all known domains int i; + validate_access("domains", counters.domains, true, __LINE__, __FUNCTION__, __FILE__); for(i = 0; i < counters.domains; i++) { // Try to match the requested string @@ -700,7 +702,9 @@ void getAllQueries(char *client_message, int *sock) if(clientname == NULL) return; sscanf(client_message, ">getallqueries-client %255s", clientname); filterclientname = true; + // Iterate through all known clients int i; + validate_access("clients", counters.clients, true, __LINE__, __FUNCTION__, __FILE__); for(i = 0; i < counters.clients; i++) { // Try to match the requested string From 1dca54c28b1dc2ea9960ec1b55e296697d1b246a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 18:43:12 +0200 Subject: [PATCH 42/48] Ensure validate_access() calls are also valid if no counters.{domains,forwarded,clients} are zero Signed-off-by: DL6ER --- api.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api.c b/api.c index b100d7d3..bc93fdb8 100644 --- a/api.c +++ b/api.c @@ -643,7 +643,7 @@ void getAllQueries(char *client_message, int *sock) { // Iterate through all known forward destinations int i; - validate_access("forwards", counters.forwarded, true, __LINE__, __FUNCTION__, __FILE__); + validate_access("forwards", MAX(0,counters.forwarded-1), true, __LINE__, __FUNCTION__, __FILE__); forwarddestid = -3; for(i = 0; i < counters.forwarded; i++) { @@ -676,7 +676,7 @@ void getAllQueries(char *client_message, int *sock) filterdomainname = true; // Iterate through all known domains int i; - validate_access("domains", counters.domains, true, __LINE__, __FUNCTION__, __FILE__); + validate_access("domains", MAX(0,counters.domains-1), true, __LINE__, __FUNCTION__, __FILE__); for(i = 0; i < counters.domains; i++) { // Try to match the requested string @@ -704,7 +704,7 @@ void getAllQueries(char *client_message, int *sock) filterclientname = true; // Iterate through all known clients int i; - validate_access("clients", counters.clients, true, __LINE__, __FUNCTION__, __FILE__); + validate_access("clients", MAX(0,counters.clients-1), true, __LINE__, __FUNCTION__, __FILE__); for(i = 0; i < counters.clients; i++) { // Try to match the requested string @@ -784,7 +784,7 @@ void getAllQueries(char *client_message, int *sock) continue; // Skip if client name and IP are not identical with what the user wants to see - if(filterclientname && queries[i].domainID != domainid) + if(filterclientname && queries[i].clientID != clientid) continue; // Skip if query type is not identical with what the user wants to see From 6aa7ac5917a49ad408daa7abf1a79e178c148645 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Aug 2018 12:12:14 +0200 Subject: [PATCH 43/48] Fix indentation Signed-off-by: DL6ER --- api.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.c b/api.c index bc93fdb8..43ddc7ce 100644 --- a/api.c +++ b/api.c @@ -781,11 +781,11 @@ void getAllQueries(char *client_message, int *sock) // Skip if domain is not identical with what the user wants to see if(filterdomainname && queries[i].domainID != domainid) - continue; + continue; // Skip if client name and IP are not identical with what the user wants to see if(filterclientname && queries[i].clientID != clientid) - continue; + continue; // Skip if query type is not identical with what the user wants to see if(querytype != 0 && querytype != queries[i].type) From 1936d60279c4a7cedb82c7a268eebec44fa63fd9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Aug 2018 16:22:40 +0200 Subject: [PATCH 44/48] Reduce code duplication by outsourcing queryID finder loop Signed-off-by: DL6ER --- dnsmasq_interface.c | 129 +++++++++++++------------------------------- 1 file changed, 37 insertions(+), 92 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 4607ed50..c7f5b7f5 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -18,6 +18,7 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response); unsigned long converttimeval(struct timeval time); static void block_single_domain(char *domain); static void query_externally_blocked(int i); +static int findQueryID(int id); char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "}; @@ -225,6 +226,29 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * disable_thread_lock(); } +static int findQueryID(int id) +{ + // Loop over all queries - we loop in reverse order (start from the most recent query and + // continuously walk older queries while trying to find a match. Ideally, we should always + // find the correct query with zero iterations, but it may happen that queries are processed + // asynchronously, e.g. for slow upstream relies to a huge amount of requests. + // We iterate from the most recent query down to at most MAXITER queries in the past to avoid + // iterating through the entire array of queries + // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) + + // Validate access only once for the maximum index (all lower will work) + validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); + int until = MAX(0, counters.queries-MAXITER); + int i; + // Check UUIDs of queries + for(i = counters.queries-1; i >= until; i--) + if(queries[i].id == id) + return i; + + // If not found + return -1; +} + void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id) { // Don't analyze anything if in PRIVACY_NOSTATS mode @@ -245,30 +269,8 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id if(debug) logg("**** forwarded %s to %s (ID %i)", name, forward, id); // Save status and forwardID in corresponding query identified by dnsmasq's ID - bool found = false; - int i; - // Loop over all queries - we loop in reverse order (start from the most recent query and - // continuously walk older queries while trying to find a match. Ideally, we should always - // find the correct query with zero iterations, but it may happen that queries are processed - // asynchronously, e.g. for slow upstream relies to a huge amount of requests. - // We iterate from the most recent query down to at most MAXITER queries in the past to avoid - // iterating through the entire array of queries - // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) - - // Validate access only once for the maximum index (all lower will work) - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check UUID of this query - if(queries[i].id == id) - { - queries[i].status = QUERY_FORWARDED; - found = true; - break; - } - } - if(!found) + int i = findQueryID(id); + if(i < 0) { // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether @@ -277,6 +279,9 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id return; } + // Set query status + queries[i].status = QUERY_FORWARDED; + // Proceed only if // - current query has not been marked as replied to so far // (it could be that answers from multiple forward @@ -411,24 +416,8 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) gettimeofday(&response, 0); // Save status in corresponding query identified by dnsmasq's ID - bool found = false; - int i; - - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check UUID of this query - if(queries[i].id == id) - { - found = true; - break; - } - } - - if(!found) + int i = findQueryID(id); + if(i < 0) { // This may happen e.g. if the original query was "pi.hole" if(debug) logg("FTL_reply(): Query %i has not been found", id); @@ -626,22 +615,8 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, print_flags(flags); } - bool found = false; - int i; - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check UUID of this query - if(queries[i].id == id) - { - found = true; - break; - } - } - if(!found) + int i = findQueryID(id); + if(i < 0) { // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether @@ -713,23 +688,8 @@ void FTL_dnssec(int status, int id) // Process DNSSEC result for a domain enable_thread_lock(); // Search for corresponding query identified by ID - bool found = false; - int i; - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check both UUID and generation of this query - if(queries[i].id == id) - { - found = true; - break; - } - } - - if(!found) + int i = findQueryID(id); + if(i < 0) { // This may happen e.g. if the original query was an unhandled query type disable_thread_lock(); @@ -771,23 +731,8 @@ void FTL_header_ADbit(unsigned char header4, int id) } // Search for corresponding query identified by ID - bool found = false; - int i; - // Search match in known queries - // See comments in FTL_forwarded() for further details about this loop - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); - for(i = counters.queries-1; i >= until; i--) - { - // Check both UUID and generation of this query - if(queries[i].id == id) - { - found = true; - break; - } - } - - if(!found) + int i = findQueryID(id); + if(i < 0) { // This may happen e.g. if the original query was an unhandled query type disable_thread_lock(); From 8712dc3d4321cc279a888b7237777d1cf1a3f2d0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Aug 2018 16:48:32 +0200 Subject: [PATCH 45/48] Ensure proper file ownerships if pihole-FTL is started as root. We drop back from root to user pihole:pihole as soon as we did everything we needed to do as root (open privileged ports, etc.). Signed-off-by: DL6ER --- dnsmasq/dnsmasq.c | 2 +- dnsmasq_interface.c | 15 ++++++++++++++- dnsmasq_interface.h | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dnsmasq/dnsmasq.c b/dnsmasq/dnsmasq.c index 54a47da8..390eaad8 100644 --- a/dnsmasq/dnsmasq.c +++ b/dnsmasq/dnsmasq.c @@ -570,7 +570,7 @@ int main_dnsmasq (int argc, char **argv) } } - FTL_fork_and_bind_sockets(); + FTL_fork_and_bind_sockets(ent_pw); log_err = log_start(ent_pw, err_pipe[1]); diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 4607ed50..01df71a6 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -867,7 +867,7 @@ pthread_t socket_listenthread; pthread_t DBthread; pthread_t GCthread; -void FTL_fork_and_bind_sockets(void) +void FTL_fork_and_bind_sockets(struct passwd *ent_pw) { if(!debug && daemonmode) go_daemon(); @@ -919,6 +919,19 @@ void FTL_fork_and_bind_sockets(void) logg("Unable to open GC thread. Exiting..."); exit(EXIT_FAILURE); } + + // Chown files if FTL started as user root but a dnsmasq config option + // states to run as a different user/group (e.g. "nobody") + if(ent_pw != NULL && getuid() == 0) + { + if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, errno, strerror(errno)); + if(database) + { + if(chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, errno, strerror(errno)); + } + } } // int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c diff --git a/dnsmasq_interface.h b/dnsmasq_interface.h index a2c4c3bf..ea08b522 100644 --- a/dnsmasq_interface.h +++ b/dnsmasq_interface.h @@ -16,7 +16,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id); void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id); void FTL_dnssec(int status, int id); void FTL_dnsmasq_reload(void); -void FTL_fork_and_bind_sockets(void); +void FTL_fork_and_bind_sockets(struct passwd *ent_pw); void FTL_header_ADbit(unsigned char header4, int id); From f67de1806d20bfb1deba9310014bc31f47a9a110 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Aug 2018 17:07:06 +0200 Subject: [PATCH 46/48] Fix syntax and use short-circuit evaluation Signed-off-by: DL6ER --- dnsmasq_interface.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 01df71a6..df1e9a8c 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -925,12 +925,9 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw) if(ent_pw != NULL && getuid() == 0) { if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, errno, strerror(errno)); - if(database) - { - if(chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, errno, strerror(errno)); - } + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno); + if(database && chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, strerror(errno), errno); } } From 1958755846dbf1228d9cc1c873627ae209be8a39 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Aug 2018 09:44:34 +0200 Subject: [PATCH 47/48] Fix usage of wrong variable Signed-off-by: DL6ER --- dnsmasq_interface.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 4607ed50..37c864ba 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -351,7 +351,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id void FTL_dnsmasq_reload(void) { - // This funtion is called by the dnsmasq code on receive of SIGHUP + // This function is called by the dnsmasq code on receive of SIGHUP // *before* clearing the cache and rereading the lists // This is the only hook that is not skipped in PRIVACY_NOSTATS mode @@ -365,9 +365,9 @@ void FTL_dnsmasq_reload(void) // Reread pihole-FTL.conf to see which blocking mode the user wants to use // It is possible to change the blocking mode here as we anyhow clear the - // cahce and reread all blocking lists + // cache and reread all blocking lists // Passing NULL to this function means it has to open the config file on - // its own behalf (on initial reading, the confg file is already opened) + // its own behalf (on initial reading, the config file is already opened) get_blocking_mode(NULL); // Reread regex.list @@ -535,7 +535,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) static void query_externally_blocked(int i) { // Correct counters as we won't count this as forwarded ... - counters.forwarded--; + counters.forwardedqueries--; overTime[queries[i].timeidx].forwarded--; validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); forwarded[queries[i].forwardID].count--; From 2d3e91d45ef897322d40e7b08cf6020b337eac55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 Aug 2018 12:30:36 +0200 Subject: [PATCH 48/48] Apply patch: Fix crash parsing a --synth-domain with no prefix. Problem introduced in 2.79/6b2b564ac34cb3c862f168e6b1457f9f0b9ca69c This fixes #367 Signed-off-by: DL6ER --- dnsmasq/option.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dnsmasq/option.c b/dnsmasq/option.c index fd171e70..7779669e 100644 --- a/dnsmasq/option.c +++ b/dnsmasq/option.c @@ -2212,7 +2212,9 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma char *star; new->next = daemon->synth_domains; daemon->synth_domains = new; - if ((star = strrchr(new->prefix, '*')) && *(star+1) == 0) + if (new->prefix && + (star = strrchr(new->prefix, '*')) + && *(star+1) == 0) { *star = 0; new->indexed = 1;