From bffb4431bc5479d4530f0c0f5e48bf83c3e6f799 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Dec 2020 23:33:05 +0100 Subject: [PATCH 01/51] Show BOOTP server and file strings used by TFTP Signed-off-by: DL6ER --- src/dhcp-discover.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/dhcp-discover.c b/src/dhcp-discover.c index d914a4cc..d0f6d30c 100644 --- a/src/dhcp-discover.c +++ b/src/dhcp-discover.c @@ -506,6 +506,18 @@ static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface else logg("N/A"); + logg_sameline(" BOOTP server: "); + if(offer_packet.sname[0] != 0) + logg("%s", offer_packet.sname); + else + logg("(empty)"); + + logg_sameline(" BOOTP file: "); + if(offer_packet.file[0] != 0) + logg("%s", offer_packet.file); + else + logg("(empty)"); + logg(" DHCP options:"); print_dhcp_offer(source.sin_addr, &offer_packet); pthread_mutex_unlock(&lock); From d5c92229b3b174266853a2b38b0addfbe6796914 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Dec 2020 23:55:21 +0100 Subject: [PATCH 02/51] Update dnsmasq version to 2.82 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c1e68b5..06684751 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-2.81) +set(DNSMASQ_VERSION pi-hole-2.82) add_subdirectory(src) From bedc5ab442be934dc165a8fbf3f62fccfb685e51 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 8 Dec 2020 01:10:12 +0100 Subject: [PATCH 03/51] Use fork-private regex substructure because each regex has an opaque structure (once compiled) and cannot be kept globally available through shared memory (at least not with any realistic effort) Signed-off-by: DL6ER --- src/regex.c | 62 +++++++++++++++++++++++++++++++++++++---------------- src/shmem.h | 1 + 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/regex.c b/src/regex.c index 818981f2..9e4ac03f 100644 --- a/src/regex.c +++ b/src/regex.c @@ -32,6 +32,8 @@ const char *regextype[REGEX_MAX] = { "blacklist", "whitelist", "CLI" }; static regex_data *white_regex = NULL; static regex_data *black_regex = NULL; static regex_data *cli_regex = NULL; +static unsigned int num_regex[REGEX_MAX] = { 0 }; +unsigned int regex_change = 0; regex_data *get_regex_from_type(const enum regex_type regexid); inline regex_data *get_regex_from_type(const enum regex_type regexid) @@ -56,7 +58,11 @@ inline regex_data *get_regex_from_type(const enum regex_type regexid) static bool compile_regex(const char *regexin, const enum regex_type regexid) { regex_data *regex = get_regex_from_type(regexid); - int index = counters->num_regex[regexid]++; + int index = num_regex[regexid]++; + + // Update global counter from private counter + // This is safe her because we're (fork-wide) locked + counters->num_regex[regexid] = num_regex[regexid]; // Extract possible Pi-hole extensions char rgxbuf[strlen(regexin) + 1u]; @@ -168,8 +174,20 @@ int match_regex(const char *input, const DNSCacheData* dns_cache, const int clie regmatch_t match = { 0 }; // This also disables any sub-matching #endif + // Check if we need to recompile regex because they were changed in + // another fork. If this is the case, reload everything (regex + // themselves as well as per-client enabled/disabled state) + if(regex_change != counters->regex_change) + { + logg("Reloading externally changed regular expressions"); + read_regex_from_database(); + // Update regex pointer as it will have changed (free_regex has + // been called) + regex = get_regex_from_type(regexid); + } + // Loop over all configured regex filters of this type - for(unsigned int index = 0; index < counters->num_regex[regexid]; index++) + for(unsigned int index = 0; index < num_regex[regexid]; index++) { // Only check regex which have been successfully compiled ... if(!regex[index].available) @@ -185,10 +203,10 @@ int match_regex(const char *input, const DNSCacheData* dns_cache, const int clie // ... and are enabled for this client int regexID = index; if(regexid == REGEX_WHITELIST) - regexID += counters->num_regex[REGEX_BLACKLIST]; + regexID += num_regex[REGEX_BLACKLIST]; else if(regexid == REGEX_CLI) - regexID += counters->num_regex[REGEX_BLACKLIST] + - counters->num_regex[REGEX_WHITELIST]; + regexID += num_regex[REGEX_BLACKLIST] + + num_regex[REGEX_WHITELIST]; // Only use regular expressions enabled for this client // We allow clientID = -1 to get all regex (for testing) @@ -208,6 +226,9 @@ int match_regex(const char *input, const DNSCacheData* dns_cache, const int clie } // Try to match the compiled regular expression against input + if(config.debug & DEBUG_REGEX) + logg("Executing: index = %d, preg = %p, str = \"%s\", pmatch = %p", index, ®ex[index].regex, input, &match); + sync(); #ifdef USE_TRE_REGEX int retval = tre_regexec(®ex[index].regex, input, 0, &match, 0); #else @@ -313,7 +334,7 @@ static void free_regex(void) { regex_data *regex = get_regex_from_type(regexid); // Loop over entries with this regex type - for(unsigned int index = 0; index < counters->num_regex[regexid]; index++) + for(unsigned int index = 0; index < num_regex[regexid]; index++) { if(!regex[index].available) continue; @@ -336,7 +357,7 @@ static void free_regex(void) } // Reset counter for number of regex - counters->num_regex[regexid] = 0; + num_regex[regexid] = 0; } } @@ -353,14 +374,14 @@ void reload_per_client_regex(const int clientID, clientsData *client) reset_per_client_regex(clientID); // Load regex per-group regex blacklist for this client - if(counters->num_regex[REGEX_BLACKLIST] > 0) - gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_BLACKLIST], + if(num_regex[REGEX_BLACKLIST] > 0) + gravityDB_get_regex_client_groups(client, num_regex[REGEX_BLACKLIST], black_regex, REGEX_BLACKLIST, "vw_regex_blacklist", clientID); // Load regex per-group regex whitelist for this client - if(counters->num_regex[REGEX_WHITELIST] > 0) - gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_WHITELIST], + if(num_regex[REGEX_WHITELIST] > 0) + gravityDB_get_regex_client_groups(client, num_regex[REGEX_WHITELIST], white_regex, REGEX_WHITELIST, "vw_regex_whitelist", clientID); } @@ -371,7 +392,7 @@ static void read_regex_table(const enum regex_type regexid) const enum gravity_tables tableID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE; // Get number of lines in the regex table - counters->num_regex[regexid] = 0; + num_regex[regexid] = 0; int count = gravityDB_count(tableID); if(count == 0) @@ -412,10 +433,10 @@ static void read_regex_table(const enum regex_type regexid) { // Avoid buffer overflow if database table changed // since we counted its entries - if(counters->num_regex[regexid] >= (unsigned int)count) + if(num_regex[regexid] >= (unsigned int)count) { logg("INFO: read_regex_table(%s) exiting early to avoid overflow (%d/%d).", - regextype[regexid], counters->num_regex[regexid], count); + regextype[regexid], num_regex[regexid], count); break; } @@ -432,11 +453,14 @@ static void read_regex_table(const enum regex_type regexid) if(config.debug & DEBUG_REGEX) { logg("Compiling %s regex %i (DB ID %i): %s", - regextype[regexid], counters->num_regex[regexid], rowid, domain); + regextype[regexid], num_regex[regexid], rowid, domain); } compile_regex(domain, regexid); - regex[counters->num_regex[regexid]-1].database_id = rowid; + regex[num_regex[regexid]-1].database_id = rowid; + + // Signal other forks that the regex data has changed and should be updated + regex_change = ++counters->regex_change; } // Finalize statement and close gravity database handle @@ -475,7 +499,7 @@ void read_regex_from_database(void) // Print message to FTL's log after reloading regex filters logg("Compiled %i whitelist and %i blacklist regex filters for %i clients in %.1f msec", - counters->num_regex[REGEX_WHITELIST], counters->num_regex[REGEX_BLACKLIST], + num_regex[REGEX_WHITELIST], num_regex[REGEX_BLACKLIST], counters->clients, timer_elapsed_msec(REGEX_TIMER)); } @@ -505,8 +529,8 @@ int regex_test(const bool debug_mode, const bool quiet, const char *domainin, co read_regex_table(REGEX_WHITELIST); log_ctrl(false, !quiet); // Re-apply quiet option after compilation logg(" Compiled %i black- and %i whitelist regex filters in %.3f msec\n", - counters->num_regex[REGEX_BLACKLIST], - counters->num_regex[REGEX_WHITELIST], + num_regex[REGEX_BLACKLIST], + num_regex[REGEX_WHITELIST], timer_elapsed_msec(REGEX_TIMER)); // Check user-provided domain against all loaded regular blacklist expressions diff --git a/src/shmem.h b/src/shmem.h index 615ec782..92c00fb6 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -54,6 +54,7 @@ typedef struct { int dns_cache_size; int dns_cache_MAX; unsigned int num_regex[REGEX_MAX]; + unsigned int regex_change; } countersStruct; extern countersStruct *counters; From 33860bf83e74ecdecd9801b35bf6046740c1839e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 8 Dec 2020 23:14:05 +0100 Subject: [PATCH 04/51] We have to explicitly set conflinebuffersize to zero when freeing the buffer itself to avoid getline() crashing in some special edge-cases Signed-off-by: DL6ER --- src/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.c b/src/config.c index 0bae6280..7d840524 100644 --- a/src/config.c +++ b/src/config.c @@ -548,6 +548,7 @@ void release_config_memory(void) { free(conflinebuffer); conflinebuffer = NULL; + size = 0; } } From 7459d8235b845bbf14f92fd356da04b83dca460d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:10:22 +0100 Subject: [PATCH 05/51] Rename memory.c -> syscalls.c Signed-off-by: DL6ER --- src/CMakeLists.txt | 4 ++-- src/FTL.h | 3 ++- src/api/socket.c | 7 +++---- src/args.c | 1 - src/capabilities.c | 1 - src/config.c | 1 - src/daemon.c | 1 - src/database/aliasclients.c | 2 -- src/database/common.c | 1 - src/database/gravity-db.c | 3 +++ src/database/gravity-db.h | 9 +++------ src/database/network-table.c | 2 -- src/database/query-table.c | 2 -- src/datastructure.c | 1 - src/dnsmasq_interface.c | 1 - src/files.c | 1 - src/gc.c | 2 -- src/log.c | 1 - src/overTime.c | 2 -- src/regex.c | 1 - src/resolve.c | 1 - src/setupVars.c | 1 - src/shmem.c | 1 - src/signals.c | 2 -- src/{memory.c => syscalls.c} | 2 +- src/{memory.h => syscalls.h} | 8 ++++---- src/timers.c | 1 - 27 files changed, 18 insertions(+), 44 deletions(-) rename src/{memory.c => syscalls.c} (99%) rename src/{memory.h => syscalls.h} (90%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b83dc8a7..b89b8fa9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -131,8 +131,8 @@ set(sources log.h main.c main.h - memory.c - memory.h + syscalls.c + syscalls.h overTime.c overTime.h regex.c diff --git a/src/FTL.h b/src/FTL.h index cf645050..d1811cd1 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -114,7 +114,7 @@ // Important: This number has to be smaller than 256 for this mechanism to work #define NUM_RECHECKS 3 -// Use out own memory handling functions that will detect possible errors +// Use out own syscalls handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crashs // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. @@ -124,6 +124,7 @@ #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) +#include "syscalls.h" // Preprocessor help functions #define str(x) # x diff --git a/src/api/socket.c b/src/api/socket.c index a5fb901d..3eb8e0d0 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -10,13 +10,12 @@ #include "FTL.h" #include "api.h" -#include "log.h" +#include "../log.h" #include "socket.h" #include "request.h" -#include "config.h" -#include "memory.h" +#include "../config.h" // global variable killed -#include "signals.h" +#include "../signals.h" // The backlog argument defines the maximum length // to which the queue of pending connections for diff --git a/src/args.c b/src/args.c index b196a83f..0878f43f 100644 --- a/src/args.c +++ b/src/args.c @@ -16,7 +16,6 @@ #include "FTL.h" #include "args.h" #include "version.h" -#include "memory.h" #include "main.h" #include "log.h" // global variable killed diff --git a/src/capabilities.c b/src/capabilities.c index 6478526c..013bba64 100644 --- a/src/capabilities.c +++ b/src/capabilities.c @@ -14,7 +14,6 @@ #undef __USE_XOPEN #include "FTL.h" #include "capabilities.h" -#include "memory.h" #include "config.h" #include "log.h" diff --git a/src/config.c b/src/config.c index 7d840524..64fdffa8 100644 --- a/src/config.c +++ b/src/config.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "config.h" -#include "memory.h" #include "setupVars.h" #include "log.h" // nice() diff --git a/src/daemon.c b/src/daemon.c index e9a29d93..2d72d20b 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "daemon.h" -#include "memory.h" #include "config.h" #include "log.h" // sleepms() diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index 2097d0c7..023c402a 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -17,8 +17,6 @@ #include "../config.h" // logg() #include "../log.h" -// calloc() -#include "../memory.h" // getAliasclientIDfromIP() #include "network-table.h" diff --git a/src/database/common.c b/src/database/common.c index 2f50b1e6..2839255c 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -13,7 +13,6 @@ #include "network-table.h" #include "message-table.h" #include "../shmem.h" -#include "../memory.h" // struct config #include "../config.h" // logg() diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index de8da8b6..13187cea 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -30,6 +30,9 @@ // reset_aliasclient() #include "aliasclients.h" +// Definition of struct regex_data +#include "../regex_r.h" + // Prefix of interface names in the client table #define INTERFACE_SEP ":" diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index e4ce7bf0..c92f671b 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -10,12 +10,9 @@ #ifndef GRAVITY_H #define GRAVITY_H -// global variable counters -#include "memory.h" -// clients data structure -#include "datastructure.h" - -// Definition of struct regex_data +// clientsData +#include "../datastructure.h" +// regex_data #include "../regex_r.h" // Table indices diff --git a/src/database/network-table.c b/src/database/network-table.c index ef12136d..40c9f9ab 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -12,8 +12,6 @@ #include "network-table.h" #include "common.h" #include "../shmem.h" -// strdup() -#include "../memory.h" #include "../log.h" // timer_elapsed_msec() #include "../timers.h" diff --git a/src/database/query-table.c b/src/database/query-table.c index 3bd3ca93..6b04f9ed 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -25,8 +25,6 @@ #include "../config.h" // getstr() #include "../shmem.h" -// free() -#include "../memory.h" static bool saving_failed_before = false; diff --git a/src/datastructure.c b/src/datastructure.c index dfa0bd24..ff8c7510 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "datastructure.h" -#include "memory.h" #include "shmem.h" #include "log.h" // enum REGEX diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c97b68d2..0a0e96f1 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -16,7 +16,6 @@ #include "dnsmasq_interface.h" #include "shmem.h" #include "overTime.h" -#include "memory.h" #include "database/common.h" #include "database/database-thread.h" #include "datastructure.h" diff --git a/src/files.c b/src/files.c index 7f1d3d94..cdcbeb80 100644 --- a/src/files.c +++ b/src/files.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "files.h" -#include "memory.h" #include "config.h" #include "setupVars.h" #include "log.h" diff --git a/src/gc.c b/src/gc.c index 618fa2e1..efe0c9e7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -16,8 +16,6 @@ #include "overTime.h" #include "database/common.h" #include "log.h" -// global variable counters -#include "memory.h" // global variable killed #include "signals.h" // data getter functions diff --git a/src/log.c b/src/log.c index 9cde7d37..28885f41 100644 --- a/src/log.c +++ b/src/log.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "version.h" -#include "memory.h" // is_fork() #include "daemon.h" #include "config.h" diff --git a/src/overTime.c b/src/overTime.c index b0315a29..53e23765 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -13,8 +13,6 @@ #include "shmem.h" #include "config.h" #include "log.h" -// global variable counters -#include "memory.h" // data getter functions #include "datastructure.h" diff --git a/src/regex.c b/src/regex.c index 818981f2..5a0807a3 100644 --- a/src/regex.c +++ b/src/regex.c @@ -11,7 +11,6 @@ #include "FTL.h" #include "regex_r.h" #include "timers.h" -#include "memory.h" #include "log.h" #include "config.h" // data getter functions diff --git a/src/resolve.c b/src/resolve.c index cd71996e..4719f94c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -11,7 +11,6 @@ #include "FTL.h" #include "resolve.h" #include "shmem.h" -#include "memory.h" // struct config #include "config.h" // sleepms() diff --git a/src/setupVars.c b/src/setupVars.c index d018a797..34c00f32 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "log.h" -#include "memory.h" #include "config.h" #include "setupVars.h" diff --git a/src/shmem.c b/src/shmem.c index bacd53fe..bd922480 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -12,7 +12,6 @@ #include "shmem.h" #include "overTime.h" #include "log.h" -#include "memory.h" #include "config.h" // data getter functions #include "datastructure.h" diff --git a/src/signals.c b/src/signals.c index b5ec061d..f39dd6cb 100644 --- a/src/signals.c +++ b/src/signals.c @@ -15,8 +15,6 @@ #include "signals.h" // logg() #include "log.h" -// free() -#include "memory.h" // ls_dir() #include "files.h" // gettid() diff --git a/src/memory.c b/src/syscalls.c similarity index 99% rename from src/memory.c rename to src/syscalls.c index 81dae6be..d3618874 100644 --- a/src/memory.c +++ b/src/syscalls.c @@ -10,7 +10,7 @@ #include "FTL.h" #include "shmem.h" -#include "memory.h" +#include "syscalls.h" #include "log.h" // The special memory handling routines have to be the last ones in this source file diff --git a/src/memory.h b/src/syscalls.h similarity index 90% rename from src/memory.h rename to src/syscalls.h index 72ea92b0..e3fdd4ed 100644 --- a/src/memory.h +++ b/src/syscalls.h @@ -3,12 +3,12 @@ * Network-wide ad blocking via your own hardware. * * FTL Engine -* Memory prototypes +* Syscall prototypes * * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#ifndef MEMORY_H -#define MEMORY_H +#ifndef SYSCALLS_H +#define SYSCALLS_H #include "enums.h" @@ -17,4 +17,4 @@ void *FTLcalloc(size_t n, size_t size, const char *file, const char *function, c void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); void FTLfree(void *ptr, const char* file, const char *function, const int line); -#endif //MEMORY_H +#endif //SYSCALLS_H diff --git a/src/timers.c b/src/timers.c index 56cfc125..691e16ba 100644 --- a/src/timers.c +++ b/src/timers.c @@ -10,7 +10,6 @@ #include "FTL.h" #include "timers.h" -#include "memory.h" #include "log.h" struct timespec t0[NUMTIMERS]; From d4deb01d9fb7e6f8acb2390b07d1e27e92841203 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:12:44 +0100 Subject: [PATCH 06/51] Factor out syscalls for calloc, free, realloc and strdup into dedicated syscalls/{}.c files Signed-off-by: DL6ER --- src/CMakeLists.txt | 4 +- src/FTL.h | 2 +- src/syscalls.c | 95 ----------------------------------- src/syscalls/CMakeLists.txt | 20 ++++++++ src/syscalls/calloc.c | 29 +++++++++++ src/syscalls/free.c | 27 ++++++++++ src/syscalls/realloc.c | 34 +++++++++++++ src/syscalls/strdup.c | 38 ++++++++++++++ src/{ => syscalls}/syscalls.h | 2 - 9 files changed, 151 insertions(+), 100 deletions(-) delete mode 100644 src/syscalls.c create mode 100644 src/syscalls/CMakeLists.txt create mode 100644 src/syscalls/calloc.c create mode 100644 src/syscalls/free.c create mode 100644 src/syscalls/realloc.c create mode 100644 src/syscalls/strdup.c rename src/{ => syscalls}/syscalls.h (97%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b89b8fa9..f4f6fb9c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -131,8 +131,6 @@ set(sources log.h main.c main.h - syscalls.c - syscalls.h overTime.c overTime.h regex.c @@ -174,6 +172,7 @@ add_executable(pihole-FTL $ $ $ + $ ) if(STATIC STREQUAL "true") set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) @@ -228,3 +227,4 @@ add_subdirectory(database) add_subdirectory(dnsmasq) add_subdirectory(lua) add_subdirectory(tre-regex) +add_subdirectory(syscalls) diff --git a/src/FTL.h b/src/FTL.h index d1811cd1..1a724e1a 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -124,7 +124,7 @@ #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) -#include "syscalls.h" +#include "syscalls/syscalls.h" // Preprocessor help functions #define str(x) # x diff --git a/src/syscalls.c b/src/syscalls.c deleted file mode 100644 index d3618874..00000000 --- a/src/syscalls.c +++ /dev/null @@ -1,95 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Global variable definitions and memory reallocation handling -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include "shmem.h" -#include "syscalls.h" -#include "log.h" - -// The special memory handling routines have to be the last ones in this source file -// as we restore the original definition of the strdup, free, calloc, and realloc -// functions in here, i.e. if anything extra would come below these lines, it would -// not be protected by our (error logging) functions! - -#undef strdup -char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, const int line) -{ - // The FTLstrdup() function returns a pointer to a new string which is a - // duplicate of the string s. Memory for the new string is obtained with - // calloc(3), and can be freed with free(3). - if(src == NULL) - { - logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", function, file, line); - return NULL; - } - const size_t len = strlen(src); - char *dest = calloc(len+1, sizeof(char)); - if(dest == NULL) - { - logg("FATAL: Memory allocation failed in %s() (%s:%i)", function, file, line); - return NULL; - } - // Use memcpy as memory areas cannot overlap - memcpy(dest, src, len); - dest[len] = '\0'; - - return dest; -} - -#undef calloc -void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char * file, const char * function, const int line) -{ - // The FTLcalloc() function allocates memory for an array of nmemb elements - // of size bytes each and returns a pointer to the allocated memory. The - // memory is set to zero. If nmemb or size is 0, then calloc() returns - // either NULL, or a unique pointer value that can later be successfully - // passed to free(). - void *ptr = calloc(nmemb, size); - if(ptr == NULL) - logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", - nmemb, size, function, file, line); - - return ptr; -} - -#undef realloc -void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * function, const int line) -{ - // The FTLrealloc() function changes the size of the memory block pointed to - // by ptr to size bytes. The contents will be unchanged in the range from - // the start of the region up to the minimum of the old and new sizes. If - // the new size is larger than the old size, the added memory will not be - // initialized. If ptr is NULL, then the call is equivalent to malloc(size), - // for all values of size; if size is equal to zero, and ptr is - // not NULL, then the call is equivalent to free(ptr). Unless ptr is - // NULL, it must have been returned by an earlier call to malloc(), cal‐ - // loc() or realloc(). If the area pointed to was moved, a free(ptr) is - // done. - void *ptr_out = realloc(ptr_in, size); - if(ptr_out == NULL) - logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", - ptr_in, size, function, file, line); - - return ptr_out; -} - -#undef free -void FTLfree(void *ptr, const char * file, const char * function, const int line) -{ - // The free() function frees the memory space pointed to by ptr, which - // must have been returned by a previous call to malloc(), calloc(), or - // realloc(). Otherwise, or if free(ptr) has already been called before, - // undefined behavior occurs. If ptr is NULL, no operation is performed. - if(ptr == NULL) - logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); - - // We intentionally run free() nevertheless to see the crash in the debugger - free(ptr); -} diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt new file mode 100644 index 00000000..cde8b6de --- /dev/null +++ b/src/syscalls/CMakeLists.txt @@ -0,0 +1,20 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2020 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/syscalls/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +set(sources + calloc.c + free.c + realloc.c + strdup.c + syscalls.h + ) + +add_library(syscalls OBJECT ${sources}) +target_compile_options(syscalls PRIVATE ${EXTRAWARN}) diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c new file mode 100644 index 00000000..facaf13c --- /dev/null +++ b/src/syscalls/calloc.c @@ -0,0 +1,29 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for calloc +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef calloc +void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char * file, const char * function, const int line) +{ + // The FTLcalloc() function allocates memory for an array of nmemb elements + // of size bytes each and returns a pointer to the allocated memory. The + // memory is set to zero. If nmemb or size is 0, then calloc() returns + // either NULL, or a unique pointer value that can later be successfully + // passed to free(). + void *ptr = calloc(nmemb, size); + if(ptr == NULL) + logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", + nmemb, size, function, file, line); + + return ptr; +} diff --git a/src/syscalls/free.c b/src/syscalls/free.c new file mode 100644 index 00000000..1c3fb66d --- /dev/null +++ b/src/syscalls/free.c @@ -0,0 +1,27 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for free +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef free +void FTLfree(void *ptr, const char * file, const char * function, const int line) +{ + // The free() function frees the memory space pointed to by ptr, which + // must have been returned by a previous call to malloc(), calloc(), or + // realloc(). Otherwise, or if free(ptr) has already been called before, + // undefined behavior occurs. If ptr is NULL, no operation is performed. + if(ptr == NULL) + logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); + + // We intentionally run free() nevertheless to see the crash in the debugger + free(ptr); +} diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c new file mode 100644 index 00000000..2c5786cc --- /dev/null +++ b/src/syscalls/realloc.c @@ -0,0 +1,34 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for realloc +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef realloc +void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * function, const int line) +{ + // The FTLrealloc() function changes the size of the memory block pointed to + // by ptr to size bytes. The contents will be unchanged in the range from + // the start of the region up to the minimum of the old and new sizes. If + // the new size is larger than the old size, the added memory will not be + // initialized. If ptr is NULL, then the call is equivalent to malloc(size), + // for all values of size; if size is equal to zero, and ptr is + // not NULL, then the call is equivalent to free(ptr). Unless ptr is + // NULL, it must have been returned by an earlier call to malloc(), cal‐ + // loc() or realloc(). If the area pointed to was moved, a free(ptr) is + // done. + void *ptr_out = realloc(ptr_in, size); + if(ptr_out == NULL) + logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", + ptr_in, size, function, file, line); + + return ptr_out; +} \ No newline at end of file diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c new file mode 100644 index 00000000..8730af0f --- /dev/null +++ b/src/syscalls/strdup.c @@ -0,0 +1,38 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for strdup +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef strdup +char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, const int line) +{ + // The FTLstrdup() function returns a pointer to a new string which is a + // duplicate of the string s. Memory for the new string is obtained with + // calloc(3), and can be freed with free(3). + if(src == NULL) + { + logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", function, file, line); + return NULL; + } + const size_t len = strlen(src); + char *dest = calloc(len+1, sizeof(char)); + if(dest == NULL) + { + logg("FATAL: Memory allocation failed in %s() (%s:%i)", function, file, line); + return NULL; + } + // Use memcpy as memory areas cannot overlap + memcpy(dest, src, len); + dest[len] = '\0'; + + return dest; +} \ No newline at end of file diff --git a/src/syscalls.h b/src/syscalls/syscalls.h similarity index 97% rename from src/syscalls.h rename to src/syscalls/syscalls.h index e3fdd4ed..887b1fa7 100644 --- a/src/syscalls.h +++ b/src/syscalls/syscalls.h @@ -10,8 +10,6 @@ #ifndef SYSCALLS_H #define SYSCALLS_H -#include "enums.h" - char *FTLstrdup(const char *src, const char *file, const char *function, const int line) __attribute__((malloc)); void *FTLcalloc(size_t n, size_t size, const char *file, const char *function, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); From 9537d7de3da04e78c2a4ad28dd74c45c123dff0c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:28:02 +0100 Subject: [PATCH 07/51] Add interrupt-safe fprintf() and printf() routines Signed-off-by: DL6ER --- src/FTL.h | 2 + src/syscalls/CMakeLists.txt | 2 + src/syscalls/fprintf.c | 80 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 3 +- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 src/syscalls/fprintf.c diff --git a/src/FTL.h b/src/FTL.h index 1a724e1a..7985b228 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -124,6 +124,8 @@ #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) +#define printf(format, ...) FTLfprintf(stdout, format, ##__VA_ARGS__) +#define fprintf(stream, format, ...) FTLfprintf(stream, format, ##__VA_ARGS__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index cde8b6de..4dde8b66 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -10,7 +10,9 @@ set(sources calloc.c + fprintf.c free.c + printf.c realloc.c strdup.c syscalls.h diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c new file mode 100644 index 00000000..82bf8e7e --- /dev/null +++ b/src/syscalls/fprintf.c @@ -0,0 +1,80 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for fprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLfprintf(FILE *stream, const char *format, ...) +{ + // Print into dynamically allocated memory + va_list arg; + char *buffer = NULL; + int length = 0; + do + { + va_start(arg, format); + // Reset errno before trying to get the string + errno = 0; + length = vasprintf(&buffer, format, arg); + va_end(arg); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(length < 0 && errno == EINTR); + + // Error handling + if(length < 0 || buffer == NULL) + { + fputs("WARN: fprintf() failed to allocate memory: ", stream); + fputs(strerror(errno), stream); + fputs("\n", stream); + fputs("Not processing string: ", stream); + fputs(format, stream); + fputs("\n", stream); + + // Return early, there isn't anything we can do here + return length; + } + + // Actually write into file now + char *_buffer = buffer; + int bytes_written = 0; + do + { + // Reset errno before trying to write + errno = 0; + // Print buffer into stream + bytes_written = fputs(_buffer, stream); + // Add number of written bytes + _buffer += bytes_written; + } + // Try to write the remaining content into the stream if this failed due + // to an interruption by an incoming signal + while(_buffer < buffer && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(_buffer < buffer) + { + fputs("WARN: fprintf() did not print all characters: ", stream); + fputs(strerror(errno), stream); + fputs("\n", stream); + fputs("Not processing string: ", stream); + fputs(format, stream); + fputs("\n", stream); + } + + // Free allocated memory + free(buffer); + + // Return number of written bytes + return length; +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 887b1fa7..2b458273 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -1,5 +1,5 @@ /* Pi-hole: A black hole for Internet advertisements -* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) * Network-wide ad blocking via your own hardware. * * FTL Engine @@ -14,5 +14,6 @@ char *FTLstrdup(const char *src, const char *file, const char *function, const i void *FTLcalloc(size_t n, size_t size, const char *file, const char *function, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); void FTLfree(void *ptr, const char* file, const char *function, const int line); +int FTLfprintf(FILE *stream, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); #endif //SYSCALLS_H From 217f957af359f72e1775c25eaf251e649d8dbef3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:39:32 +0100 Subject: [PATCH 08/51] Add interrupt-safe vfprintf() and vprintf() routines Signed-off-by: DL6ER --- src/FTL.h | 2 + src/syscalls/CMakeLists.txt | 2 +- src/syscalls/fprintf.c | 65 ++------------------------ src/syscalls/syscalls.h | 1 + src/syscalls/vfprintf.c | 92 +++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 62 deletions(-) create mode 100644 src/syscalls/vfprintf.c diff --git a/src/FTL.h b/src/FTL.h index 7985b228..d508523d 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -126,6 +126,8 @@ #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) #define printf(format, ...) FTLfprintf(stdout, format, ##__VA_ARGS__) #define fprintf(stream, format, ...) FTLfprintf(stream, format, ##__VA_ARGS__) +#define vprintf(format, args) FTLvfprintf(stdout, format, args) +#define vfprintf(stream, format, args) FTLvfprintf(stream, format, args) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 4dde8b66..c38f86cb 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -12,10 +12,10 @@ set(sources calloc.c fprintf.c free.c - printf.c realloc.c strdup.c syscalls.h + vfprintf.c ) add_library(syscalls OBJECT ${sources}) diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c index 82bf8e7e..e8016c88 100644 --- a/src/syscalls/fprintf.c +++ b/src/syscalls/fprintf.c @@ -14,67 +14,10 @@ int FTLfprintf(FILE *stream, const char *format, ...) { - // Print into dynamically allocated memory - va_list arg; - char *buffer = NULL; - int length = 0; - do - { - va_start(arg, format); - // Reset errno before trying to get the string - errno = 0; - length = vasprintf(&buffer, format, arg); - va_end(arg); - } - // Try again to allocate memory if this failed due to an interruption by - // an incoming signal - while(length < 0 && errno == EINTR); + va_list args; + va_start(args, format); + const int length = FTLvfprintf(stream, format, args); + va_end(args); - // Error handling - if(length < 0 || buffer == NULL) - { - fputs("WARN: fprintf() failed to allocate memory: ", stream); - fputs(strerror(errno), stream); - fputs("\n", stream); - fputs("Not processing string: ", stream); - fputs(format, stream); - fputs("\n", stream); - - // Return early, there isn't anything we can do here - return length; - } - - // Actually write into file now - char *_buffer = buffer; - int bytes_written = 0; - do - { - // Reset errno before trying to write - errno = 0; - // Print buffer into stream - bytes_written = fputs(_buffer, stream); - // Add number of written bytes - _buffer += bytes_written; - } - // Try to write the remaining content into the stream if this failed due - // to an interruption by an incoming signal - while(_buffer < buffer && errno == EINTR); - - // Final error checking (may have faild for some other reason then an - // EINTR = interrupted system call) - if(_buffer < buffer) - { - fputs("WARN: fprintf() did not print all characters: ", stream); - fputs(strerror(errno), stream); - fputs("\n", stream); - fputs("Not processing string: ", stream); - fputs(format, stream); - fputs("\n", stream); - } - - // Free allocated memory - free(buffer); - - // Return number of written bytes return length; } diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 2b458273..a94ca940 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -15,5 +15,6 @@ void *FTLcalloc(size_t n, size_t size, const char *file, const char *function, c void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); void FTLfree(void *ptr, const char* file, const char *function, const int line); int FTLfprintf(FILE *stream, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); +int FTLvfprintf(FILE *stream, const char *format, va_list args) __attribute__ ((format (gnu_printf, 2, 0))); #endif //SYSCALLS_H diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c new file mode 100644 index 00000000..96de1cf2 --- /dev/null +++ b/src/syscalls/vfprintf.c @@ -0,0 +1,92 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for vfprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLvfprintf(FILE *stream, const char *format, va_list args) +{ + // Print into dynamically allocated memory + char *buffer = NULL; + int _errno, length = 0; + do + { + // The va_copy() macro copies the (previously initialized) variable + // argument list args to the local _args. The behavior is as if + // va_start() were applied to _args with the same last argument, + // followed by the same number of va_arg() invocations that was used to + // reach the current state of args. We do this to be able to reuse the + // arguments in args when we need to redo the string preparation + // procedure + va_list _args; + va_copy(_args, args); + // Reset errno before trying to get the string + errno = 0; + // Do the actual string transformation + length = vasprintf(&buffer, format, _args); + // Copy errno into buffer before calling va_end() + _errno = errno; + va_end(_args); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(length < 0 && _errno == EINTR); + + // Handle other errors than EINTR + if(length < 0 || buffer == NULL) + { + fputs("WARN: fprintf() failed to allocate memory: ", stream); + fputs(strerror(errno), stream); + fputs("\n", stream); + fputs("Not processing string: ", stream); + fputs(format, stream); + fputs("\n", stream); + + // Free the buffer in case anything got allocated + if(buffer != NULL) + free(buffer); + + // Return early, there isn't anything we can do here + return length; + } + + // Actually write into the requested stream now + char *_buffer = buffer; + do + { + // Reset errno before trying to write + errno = 0; + // Print buffer into stream and advance working pointer by number of + // written bytes + _buffer += fputs(_buffer, stream); + } + // Try to write the remaining content into the stream if this failed due + // to an interruption by an incoming signal + while(_buffer < buffer && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(_buffer < buffer) + { + fputs("WARN: fprintf() did not print all characters: ", stream); + fputs(strerror(errno), stream); + fputs("\n", stream); + fputs("Not processing string: ", stream); + fputs(format, stream); + fputs("\n", stream); + } + + // Free allocated memory + free(buffer); + + // Return number of written bytes + return length; +} From 44f88e8870562751f9d91fb1880f395d5fd5cfb9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:43:38 +0100 Subject: [PATCH 09/51] Make calloc(), realloc() and strdup() interrupt-safe Signed-off-by: DL6ER --- src/syscalls/calloc.c | 12 +++++++++++- src/syscalls/free.c | 1 - src/syscalls/realloc.c | 12 +++++++++++- src/syscalls/strdup.c | 10 +++++----- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index facaf13c..4eab1b3c 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -20,7 +20,17 @@ void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const s // memory is set to zero. If nmemb or size is 0, then calloc() returns // either NULL, or a unique pointer value that can later be successfully // passed to free(). - void *ptr = calloc(nmemb, size); + void *ptr = NULL; + do + { + errno = 0; + ptr = calloc(nmemb, size); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(ptr == NULL && errno == EINTR); + + // Handle other errors than EINTR if(ptr == NULL) logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", nmemb, size, function, file, line); diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 1c3fb66d..56ccc4ae 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -22,6 +22,5 @@ void FTLfree(void *ptr, const char * file, const char * function, const int line if(ptr == NULL) logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); - // We intentionally run free() nevertheless to see the crash in the debugger free(ptr); } diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index 2c5786cc..ff73da45 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -25,7 +25,17 @@ void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, // NULL, it must have been returned by an earlier call to malloc(), cal‐ // loc() or realloc(). If the area pointed to was moved, a free(ptr) is // done. - void *ptr_out = realloc(ptr_in, size); + void *ptr_out = NULL; + do + { + errno = 0; + ptr_out = realloc(ptr_in, size); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(ptr_out == NULL && errno == EINTR); + + // Handle other errors than EINTR if(ptr_out == NULL) logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", ptr_in, size, function, file, line); diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c index 8730af0f..e71d4a7a 100644 --- a/src/syscalls/strdup.c +++ b/src/syscalls/strdup.c @@ -12,7 +12,6 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -#undef strdup char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, const int line) { // The FTLstrdup() function returns a pointer to a new string which is a @@ -24,12 +23,13 @@ char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, cons return NULL; } const size_t len = strlen(src); - char *dest = calloc(len+1, sizeof(char)); + char *dest = FTLcalloc(len+1, sizeof(char), file, function, line); + + // Return early in case of an unrecoverable error, error reporting has + // already been done in FTLcalloc() if(dest == NULL) - { - logg("FATAL: Memory allocation failed in %s() (%s:%i)", function, file, line); return NULL; - } + // Use memcpy as memory areas cannot overlap memcpy(dest, src, len); dest[len] = '\0'; From 1a0df27d28af6d38f3f3ccef981a3860216b8636 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:52:40 +0100 Subject: [PATCH 10/51] Add interrupt-safe write() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/syscalls.h | 1 + src/syscalls/write.c | 45 +++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 src/syscalls/write.c diff --git a/src/FTL.h b/src/FTL.h index d508523d..a8672382 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -128,6 +128,7 @@ #define fprintf(stream, format, ...) FTLfprintf(stream, format, ##__VA_ARGS__) #define vprintf(format, args) FTLvfprintf(stdout, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, format, args) +#define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index c38f86cb..aaa2b9e4 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -16,6 +16,7 @@ set(sources strdup.c syscalls.h vfprintf.c + write.c ) add_library(syscalls OBJECT ${sources}) diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index a94ca940..591a0b4f 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -16,5 +16,6 @@ void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *functi void FTLfree(void *ptr, const char* file, const char *function, const int line); int FTLfprintf(FILE *stream, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); int FTLvfprintf(FILE *stream, const char *format, va_list args) __attribute__ ((format (gnu_printf, 2, 0))); +ssize_t FTLwrite(int fd, const void *buf, size_t total, const char * file, const char * function, const int line); #endif //SYSCALLS_H diff --git a/src/syscalls/write.c b/src/syscalls/write.c new file mode 100644 index 00000000..76b606b9 --- /dev/null +++ b/src/syscalls/write.c @@ -0,0 +1,45 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for write +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +ssize_t FTLwrite(int fd, const void *buf, size_t total, const char * file, const char * function, const int line) +{ + if(buf == NULL) + { + logg("ERROR: Trying to write a NULL string in %s() (%s:%i)", function, file, line); + return 0; + } + + ssize_t ret = 0; + size_t written = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = write(fd, buf, total); + if(ret > 0) + written += ret; + } + // Try to write the remaining content into the stream if + // (a) we haven't written all the data, however, there was no other error + // (b) the last write() call failed due to an interruption by an incoming signal + while((written < total && errno == 0) || (ret < 0 && errno == EINTR)); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(written < total) + logg("WARN: Could not write() everything in %s() (%s:%i): %s", + function, file, line, strerror(errno)); + + return written; +} \ No newline at end of file From e23eb128ec592528184d5a9e116ceb0b26e150c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 18:57:16 +0100 Subject: [PATCH 11/51] Add interrupt-safe accept() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/accept.c | 35 +++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 1 + 4 files changed, 38 insertions(+) create mode 100644 src/syscalls/accept.c diff --git a/src/FTL.h b/src/FTL.h index a8672382..709f1bf1 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -129,6 +129,7 @@ #define vprintf(format, args) FTLvfprintf(stdout, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) +#define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index aaa2b9e4..67b49dc9 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -9,6 +9,7 @@ # Please see LICENSE file for your rights under this license. set(sources + accept.c calloc.c fprintf.c free.c diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c new file mode 100644 index 00000000..0fc7e0a0 --- /dev/null +++ b/src/syscalls/accept.c @@ -0,0 +1,35 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for accept +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char * file, const char * function, const int line) +{ + int ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = accept(sockfd, addr, addrlen); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not accept() in %s() (%s:%i): %s", + function, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 591a0b4f..fa16a58e 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -17,5 +17,6 @@ void FTLfree(void *ptr, const char* file, const char *function, const int line); int FTLfprintf(FILE *stream, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); int FTLvfprintf(FILE *stream, const char *format, va_list args) __attribute__ ((format (gnu_printf, 2, 0))); ssize_t FTLwrite(int fd, const void *buf, size_t total, const char * file, const char * function, const int line); +int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char * file, const char * function, const int line); #endif //SYSCALLS_H From 06396bab5c1f885069dcd965074826794bb46981 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:01:18 +0100 Subject: [PATCH 12/51] Avoid redundant error reporting Signed-off-by: DL6ER --- src/FTL.h | 2 -- src/api/msgpack.c | 22 +++++++++++----------- src/api/socket.c | 12 +++--------- src/api/socket.h | 1 - 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 709f1bf1..e3323876 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -119,8 +119,6 @@ // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. #define free(ptr) FTLfree(ptr, __FILE__, __FUNCTION__, __LINE__) -#define lib_strdup() strdup() -#undef strdup #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/api/msgpack.c b/src/api/msgpack.c index eae1ff53..ede6065f 100644 --- a/src/api/msgpack.c +++ b/src/api/msgpack.c @@ -16,12 +16,12 @@ void pack_eom(const int sock) { // This byte is explicitly never used in the MessagePack spec, so it is perfect to use as an EOM for this API. uint8_t eom = 0xc1; - swrite(sock, &eom, sizeof(eom)); + write(sock, &eom, sizeof(eom)); } static void pack_basic(const int sock, const uint8_t format, const void *value, const size_t size) { - swrite(sock, &format, sizeof(format)); - swrite(sock, value, size); + write(sock, &format, sizeof(format)); + write(sock, value, size); } static uint64_t __attribute__((const)) leToBe64(const uint64_t value) { @@ -42,7 +42,7 @@ static uint64_t __attribute__((const)) leToBe64(const uint64_t value) { void pack_bool(const int sock, const bool value) { uint8_t packed = (uint8_t) (value ? 0xc3 : 0xc2); - swrite(sock, &packed, sizeof(packed)); + write(sock, &packed, sizeof(packed)); } void pack_uint8(const int sock, const uint8_t value) { @@ -87,8 +87,8 @@ bool pack_fixstr(const int sock, const char *string) { } const uint8_t format = (uint8_t) (0xA0 | length); - swrite(sock, &format, sizeof(format)); - swrite(sock, string, length); + write(sock, &format, sizeof(format)); + write(sock, string, length); return true; } @@ -104,17 +104,17 @@ bool pack_str32(const int sock, const char *string) { } const uint8_t format = 0xdb; - swrite(sock, &format, sizeof(format)); + write(sock, &format, sizeof(format)); const uint32_t bigELength = htonl((uint32_t) length); - swrite(sock, &bigELength, sizeof(bigELength)); - swrite(sock, string, length); + write(sock, &bigELength, sizeof(bigELength)); + write(sock, string, length); return true; } void pack_map16_start(const int sock, const uint16_t length) { const uint8_t format = 0xde; - swrite(sock, &format, sizeof(format)); + write(sock, &format, sizeof(format)); const uint16_t bigELength = htons(length); - swrite(sock, &bigELength, sizeof(bigELength)); + write(sock, &bigELength, sizeof(bigELength)); } diff --git a/src/api/socket.c b/src/api/socket.c index 3eb8e0d0..2201dbbf 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -222,21 +222,15 @@ void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const cha char *buffer; va_list args; va_start(args, format); - int ret = vasprintf(&buffer, format, args); + int bytes = vasprintf(&buffer, format, args); va_end(args); - if(ret > 0) + if(bytes > 0 && buffer != NULL) { - if(!write(sock, buffer, strlen(buffer))) - logg("WARNING: Socket write returned error %s (%i)", strerror(errno), errno); + write(sock, buffer, bytes); free(buffer); } } -void swrite(const int sock, const void *value, size_t size) { - if(write(sock, value, size) == -1) - logg("WARNING: Socket write returned error code %i", errno); -} - static inline int checkClientLimit(const int socket) { if(socket < MAXCONNS) { diff --git a/src/api/socket.h b/src/api/socket.h index 630ba960..bfb7150f 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -15,7 +15,6 @@ void close_telnet_socket(void); void close_unix_socket(bool unlink_file); void seom(const int sock); void ssend(const int sock, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); -void swrite(const int sock, const void* value, const size_t size); void *telnet_listening_thread_IPv4(void *args); void *telnet_listening_thread_IPv6(void *args); void *socket_listening_thread(void *args); From b00548edb9db309759dd3cc4d14ae2ca08337e65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:21:46 +0100 Subject: [PATCH 13/51] Improve printf(), fprintf(), vprintf(), and vfprintf() error reporting Signed-off-by: DL6ER --- src/FTL.h | 8 ++-- src/syscalls/accept.c | 4 +- src/syscalls/calloc.c | 6 +-- src/syscalls/fprintf.c | 4 +- src/syscalls/free.c | 4 +- src/syscalls/realloc.c | 13 +++--- src/syscalls/strdup.c | 6 +-- src/syscalls/syscalls.h | 21 +++++---- src/syscalls/vfprintf.c | 101 ++++++++++++++++++++++++++++++++++------ src/syscalls/write.c | 8 ++-- 10 files changed, 127 insertions(+), 48 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index e3323876..aec2c3ce 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -122,10 +122,10 @@ #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) -#define printf(format, ...) FTLfprintf(stdout, format, ##__VA_ARGS__) -#define fprintf(stream, format, ...) FTLfprintf(stream, format, ##__VA_ARGS__) -#define vprintf(format, args) FTLvfprintf(stdout, format, args) -#define vfprintf(stream, format, args) FTLvfprintf(stream, format, args) +#define printf(format, ...) FTLfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) +#define fprintf(stream, format, ...) FTLfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) +#define vprintf(format, args) FTLvfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, args) +#define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 0fc7e0a0..8dc12d21 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -12,7 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char * file, const char * function, const int line) +int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) { int ret = 0; do @@ -29,7 +29,7 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not accept() in %s() (%s:%i): %s", - function, file, line, strerror(errno)); + func, file, line, strerror(errno)); return ret; } \ No newline at end of file diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index 4eab1b3c..b79ad5fd 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -13,9 +13,9 @@ #include "../log.h" #undef calloc -void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char * file, const char * function, const int line) +void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char *file, const char *func, const int line) { - // The FTLcalloc() function allocates memory for an array of nmemb elements + // The FTLcalloc() func allocates memory for an array of nmemb elements // of size bytes each and returns a pointer to the allocated memory. The // memory is set to zero. If nmemb or size is 0, then calloc() returns // either NULL, or a unique pointer value that can later be successfully @@ -33,7 +33,7 @@ void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const s // Handle other errors than EINTR if(ptr == NULL) logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", - nmemb, size, function, file, line); + nmemb, size, func, file, line); return ptr; } diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c index e8016c88..90b88652 100644 --- a/src/syscalls/fprintf.c +++ b/src/syscalls/fprintf.c @@ -12,11 +12,11 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -int FTLfprintf(FILE *stream, const char *format, ...) +int FTLfprintf(FILE *stream, const char* file, const char *func, const int line, const char *format, ...) { va_list args; va_start(args, format); - const int length = FTLvfprintf(stream, format, args); + const int length = FTLvfprintf(stream, file, func, line, format, args); va_end(args); return length; diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 56ccc4ae..da08daf1 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -13,14 +13,14 @@ #include "../log.h" #undef free -void FTLfree(void *ptr, const char * file, const char * function, const int line) +void FTLfree(void *ptr, const char *file, const char *func, const int line) { // The free() function frees the memory space pointed to by ptr, which // must have been returned by a previous call to malloc(), calloc(), or // realloc(). Otherwise, or if free(ptr) has already been called before, // undefined behavior occurs. If ptr is NULL, no operation is performed. if(ptr == NULL) - logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); + logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", func, file, line); free(ptr); } diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index ff73da45..53df93b1 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -13,18 +13,17 @@ #include "../log.h" #undef realloc -void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * function, const int line) +void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * func, const int line) { // The FTLrealloc() function changes the size of the memory block pointed to // by ptr to size bytes. The contents will be unchanged in the range from // the start of the region up to the minimum of the old and new sizes. If // the new size is larger than the old size, the added memory will not be // initialized. If ptr is NULL, then the call is equivalent to malloc(size), - // for all values of size; if size is equal to zero, and ptr is - // not NULL, then the call is equivalent to free(ptr). Unless ptr is - // NULL, it must have been returned by an earlier call to malloc(), cal‐ - // loc() or realloc(). If the area pointed to was moved, a free(ptr) is - // done. + // for all values of size; if size is equal to zero, and ptr is not NULL, + // then the call is equivalent to free(ptr). Unless ptr is NULL, it must + // have been returned by an earlier call to malloc(), calloc() or realloc(). + // If the area pointed to was moved, a free(ptr) is done implicitly. void *ptr_out = NULL; do { @@ -38,7 +37,7 @@ void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, // Handle other errors than EINTR if(ptr_out == NULL) logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", - ptr_in, size, function, file, line); + ptr_in, size, func, file, line); return ptr_out; } \ No newline at end of file diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c index e71d4a7a..cb3543de 100644 --- a/src/syscalls/strdup.c +++ b/src/syscalls/strdup.c @@ -12,18 +12,18 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, const int line) +char* __attribute__((malloc)) FTLstrdup(const char *src, const char *file, const char *func, const int line) { // The FTLstrdup() function returns a pointer to a new string which is a // duplicate of the string s. Memory for the new string is obtained with // calloc(3), and can be freed with free(3). if(src == NULL) { - logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", function, file, line); + logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", func, file, line); return NULL; } const size_t len = strlen(src); - char *dest = FTLcalloc(len+1, sizeof(char), file, function, line); + char *dest = FTLcalloc(len+1, sizeof(char), file, func, line); // Return early in case of an unrecoverable error, error reporting has // already been done in FTLcalloc() diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index fa16a58e..7351c348 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -10,13 +10,18 @@ #ifndef SYSCALLS_H #define SYSCALLS_H -char *FTLstrdup(const char *src, const char *file, const char *function, const int line) __attribute__((malloc)); -void *FTLcalloc(size_t n, size_t size, const char *file, const char *function, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); -void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); -void FTLfree(void *ptr, const char* file, const char *function, const int line); -int FTLfprintf(FILE *stream, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); -int FTLvfprintf(FILE *stream, const char *format, va_list args) __attribute__ ((format (gnu_printf, 2, 0))); -ssize_t FTLwrite(int fd, const void *buf, size_t total, const char * file, const char * function, const int line); -int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char * file, const char * function, const int line); +// Interrupt-safe memory routines +char *FTLstrdup(const char *src, const char *file, const char *func, const int line) __attribute__((malloc)); +void *FTLcalloc(size_t n, size_t size, const char *file, const char *func, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); +void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, const int line) __attribute__((alloc_size(2))); +void FTLfree(void *ptr, const char*file, const char *func, const int line); + +// Interrupt-safe printing routines +int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); +int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); + +// Interrupt-safe socket routines +ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); +int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line); #endif //SYSCALLS_H diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index 96de1cf2..501791ed 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -12,7 +12,90 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -int FTLvfprintf(FILE *stream, const char *format, va_list args) +// itoa implementation using only static memory +// taken from Kernighan and Ritchie's "The C Programming Language" +// see https://clc-wiki.net/wiki/K&R2_solutions:Chapter_3:Exercise_4 +// This implementation has its drawbacks, however, we only use it for +// automated conversion of code line numbers to strings so we're not +// interested in its performance outside the range of [1, 10'000] +static void itoa(int n, char s[]) +{ + int i = 0, sign = n; + + // Make n positive if negative + if (sign < 0) + n = -n; + + // Generate digits in reverse order + do + { + s[i++] = n % 10 + '0'; /* get next digit */ + } while ((n /= 10) > 0); /* delete it */ + + // Add sign (if needed) + if (sign < 0) + s[i++] = '-'; + + // Rero-terminate string + s[i] = '\0'; + + // Reverse string s in place + int j; + char c; + int len = strlen(s); + for (i = 0, j = len-1; i Date: Wed, 9 Dec 2020 19:30:28 +0100 Subject: [PATCH 14/51] Add interrupt-safe recv() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/recv.c | 37 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 1 + 4 files changed, 40 insertions(+) create mode 100644 src/syscalls/recv.c diff --git a/src/FTL.h b/src/FTL.h index aec2c3ce..41685e76 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -128,6 +128,7 @@ #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) +#define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 67b49dc9..ba7f763c 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -14,6 +14,7 @@ set(sources fprintf.c free.c realloc.c + recv.c strdup.c syscalls.h vfprintf.c diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c new file mode 100644 index 00000000..7c274a06 --- /dev/null +++ b/src/syscalls/recv.c @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for recv +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#include + +int FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) +{ + int ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = recv(sockfd, buf, len, flags); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not recv() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 7351c348..15a1f844 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -23,5 +23,6 @@ int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line); +int FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); #endif //SYSCALLS_H From 3dfc0698dda9b4d97d5951dac12119b32a100434 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:35:33 +0100 Subject: [PATCH 15/51] Add interrupt-safe recvfrom() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/recv.c | 4 ++-- src/syscalls/recvfrom.c | 38 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 3 ++- 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 src/syscalls/recvfrom.c diff --git a/src/FTL.h b/src/FTL.h index 41685e76..9b936930 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -129,6 +129,7 @@ #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) +#define recvfrom(sockfd, buf, len, flags, src_addr, addrlen) FTLrecvfrom(sockfd, buf, len, flags, src_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index ba7f763c..178c7467 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -15,6 +15,7 @@ set(sources free.c realloc.c recv.c + recvfrom.c strdup.c syscalls.h vfprintf.c diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index 7c274a06..404da899 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -14,9 +14,9 @@ #include -int FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) +ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) { - int ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c new file mode 100644 index 00000000..347194a8 --- /dev/null +++ b/src/syscalls/recvfrom.c @@ -0,0 +1,38 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for recvfrom +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#include +#include + +ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line) +{ + ssize_t ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = recvfrom(sockfd, buf, len, flags, src_addr, addrlen); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not recvfrom() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 15a1f844..caa116e1 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -23,6 +23,7 @@ int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line); -int FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); +ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); +ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line); #endif //SYSCALLS_H From 2c35038d578a7185a5faca27193397f4ac1c343f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:44:18 +0100 Subject: [PATCH 16/51] Add interrupt-safe pthread_mutex_lock() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/pthread_mutex_lock.c | 37 +++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 3 +++ 4 files changed, 42 insertions(+) create mode 100644 src/syscalls/pthread_mutex_lock.c diff --git a/src/FTL.h b/src/FTL.h index 9b936930..8c9c67f0 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -130,6 +130,7 @@ #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) #define recvfrom(sockfd, buf, len, flags, src_addr, addrlen) FTLrecvfrom(sockfd, buf, len, flags, src_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) +#define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 178c7467..2039c251 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -13,6 +13,7 @@ set(sources calloc.c fprintf.c free.c + pthread_mutex_lock.c realloc.c recv.c recvfrom.c diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c new file mode 100644 index 00000000..01d024de --- /dev/null +++ b/src/syscalls/pthread_mutex_lock.c @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for pthread_mutex_lock +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#include + +int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line) +{ + ssize_t ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = pthread_mutex_lock(__mutex); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not pthread_mutex_lock() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index caa116e1..4243265b 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -26,4 +26,7 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line); +// Interrupt-safe thread routines +int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line); + #endif //SYSCALLS_H From ffdc52a1d1e4f8f82e83e49701d62fd33e77aa52 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:52:19 +0100 Subject: [PATCH 17/51] Add interrupt-safe select() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/select.c | 37 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 3 +++ 4 files changed, 42 insertions(+) create mode 100644 src/syscalls/select.c diff --git a/src/FTL.h b/src/FTL.h index 8c9c67f0..ef2398d9 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -130,6 +130,7 @@ #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) #define recvfrom(sockfd, buf, len, flags, src_addr, addrlen) FTLrecvfrom(sockfd, buf, len, flags, src_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) +#define select(nfds, readfds, writefds, exceptfds, timeout) FTLselect(nfds, readfds, writefds, exceptfds, timeout, __FILE__, __FUNCTION__, __LINE__) #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 2039c251..46a8689a 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -17,6 +17,7 @@ set(sources realloc.c recv.c recvfrom.c + select.c strdup.c syscalls.h vfprintf.c diff --git a/src/syscalls/select.c b/src/syscalls/select.c new file mode 100644 index 00000000..83f676ec --- /dev/null +++ b/src/syscalls/select.c @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for select +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#include + +int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line) +{ + int ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = select(nfds, readfds, writefds, exceptfds, timeout); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not select() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 4243265b..53ec360c 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -17,7 +17,9 @@ void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, void FTLfree(void *ptr, const char*file, const char *func, const int line); // Interrupt-safe printing routines +// printf() is derived from fprintf(stdout, ...) int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); +// vprintf() is derived from vfprintf(stdout, ...) int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); // Interrupt-safe socket routines @@ -25,6 +27,7 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line); ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line); +int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line); // Interrupt-safe thread routines int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line); From 67b1a65464f883e23a66af7176106e977396623c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 19:58:27 +0100 Subject: [PATCH 18/51] Add interrupt-safe fopen() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/fopen.c | 35 +++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 3 +++ 4 files changed, 40 insertions(+) create mode 100644 src/syscalls/fopen.c diff --git a/src/FTL.h b/src/FTL.h index ef2398d9..3e211e3a 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -132,6 +132,7 @@ #define recvfrom(sockfd, buf, len, flags, src_addr, addrlen) FTLrecvfrom(sockfd, buf, len, flags, src_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define select(nfds, readfds, writefds, exceptfds, timeout) FTLselect(nfds, readfds, writefds, exceptfds, timeout, __FILE__, __FUNCTION__, __LINE__) #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) +#define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 46a8689a..0c105497 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -11,6 +11,7 @@ set(sources accept.c calloc.c + fopen.c fprintf.c free.c pthread_mutex_lock.c diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c new file mode 100644 index 00000000..81c47520 --- /dev/null +++ b/src/syscalls/fopen.c @@ -0,0 +1,35 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for fopen +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) +{ + FILE *file_ptr = 0; + do + { + // Reset errno before trying to write + errno = 0; + file_ptr = fopen(pathname, mode); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(file_ptr == NULL && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(file_ptr == NULL) + logg("WARN: Could not fopen(%s, %s) in %s() (%s:%i): %s", + pathname, mode, func, file, line, strerror(errno)); + + return file_ptr; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 53ec360c..f14d04fa 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -32,4 +32,7 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st // Interrupt-safe thread routines int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line); +// Interrupt-safe file routines +FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line); + #endif //SYSCALLS_H From 543ea8558725faa094f0587588dcd7f3db7d0100 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 20:03:55 +0100 Subject: [PATCH 19/51] Add interrupt-safe sendto() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/sendto.c | 38 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 1 + 4 files changed, 41 insertions(+) create mode 100644 src/syscalls/sendto.c diff --git a/src/FTL.h b/src/FTL.h index 3e211e3a..052f0ff6 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -130,6 +130,7 @@ #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) #define recvfrom(sockfd, buf, len, flags, src_addr, addrlen) FTLrecvfrom(sockfd, buf, len, flags, src_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) +#define sendto(sockfd, buf, len, flags, dest_addr, addrlen) FTLsendto(sockfd, buf, len, flags, dest_addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define select(nfds, readfds, writefds, exceptfds, timeout) FTLselect(nfds, readfds, writefds, exceptfds, timeout, __FILE__, __FUNCTION__, __LINE__) #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) #define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 0c105497..3b9d60b3 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -19,6 +19,7 @@ set(sources recv.c recvfrom.c select.c + sendto.c strdup.c syscalls.h vfprintf.c diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c new file mode 100644 index 00000000..1f63ce77 --- /dev/null +++ b/src/syscalls/sendto.c @@ -0,0 +1,38 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for sendto +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#include +#include + +ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen, const char *file, const char *func, const int line) +{ + ssize_t ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = sendto(sockfd, buf, len, flags, dest_addr, addrlen); + } + // Try again if the last accept() call failed due to an interruption by an + // incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not sendto() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index f14d04fa..3fcc303e 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -28,6 +28,7 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line); ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line); int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line); +ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen, const char *file, const char *func, const int line); // Interrupt-safe thread routines int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line); From d234dad71c45beace8e06c740a83ace4dfc4806c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 20:06:17 +0100 Subject: [PATCH 20/51] Backup and restore errno in real-time signal handler. Signed-off-by: DL6ER --- src/signals.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/signals.c b/src/signals.c index f39dd6cb..ebab301c 100644 --- a/src/signals.c +++ b/src/signals.c @@ -245,10 +245,17 @@ static void __attribute__((noreturn)) signal_handler(int sig, siginfo_t *si, voi } static void SIGRT_handler(int signum, siginfo_t *si, void *unused) -{ +{ + // Backup errno + const int _errno = errno; + // Ignore real-time signals outside of the main process (TCP forks) if(mpid != getpid()) + { + // Restore errno before returning + errno = _errno; return; + } int rtsig = signum - SIGRTMIN; logg("Received: %s (%d -> %d)", strsignal(signum), signum, rtsig); @@ -288,6 +295,9 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // Parse neighbor cache set_event(PARSE_NEIGHBOR_CACHE); } + + // Restore errno before returning back to previous context + errno = _errno; } // Register SIGSEGV handler From d4d4a0b1d6b19659f819e3f1b9a3bc24d854e610 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:02:00 +0100 Subject: [PATCH 21/51] Add interrupt-safe vsnprintf() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/accept.c | 1 + src/syscalls/fopen.c | 1 + src/syscalls/pthread_mutex_lock.c | 1 + src/syscalls/recv.c | 1 + src/syscalls/recvfrom.c | 1 + src/syscalls/select.c | 1 + src/syscalls/sendto.c | 1 + src/syscalls/syscalls.h | 4 +++ src/syscalls/vfprintf.c | 49 ++++++++++++------------- src/syscalls/vsnprintf.c | 59 +++++++++++++++++++++++++++++++ src/syscalls/write.c | 1 + 13 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 src/syscalls/vsnprintf.c diff --git a/src/FTL.h b/src/FTL.h index 052f0ff6..2a1f48be 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -126,6 +126,7 @@ #define fprintf(stream, format, ...) FTLfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) #define vprintf(format, args) FTLvfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) +#define vsnprintf(buffer, maxlen, format, args) FTLvsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) #define recv(sockfd, buf, len, flags) FTLrecv(sockfd, buf, len, flags, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 3b9d60b3..1597b2c3 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -23,6 +23,7 @@ set(sources strdup.c syscalls.h vfprintf.c + vsnprintf.c write.c ) diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 8dc12d21..5b9cdd94 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -12,6 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" +#undef accept int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) { int ret = 0; diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index 81c47520..ac06bac4 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -12,6 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" +#undef fopen FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) { FILE *file_ptr = 0; diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c index 01d024de..da741764 100644 --- a/src/syscalls/pthread_mutex_lock.c +++ b/src/syscalls/pthread_mutex_lock.c @@ -14,6 +14,7 @@ #include +#undef pthread_mutex_lock int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line) { ssize_t ret = 0; diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index 404da899..84f2d4bd 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -14,6 +14,7 @@ #include +#undef recv ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) { ssize_t ret = 0; diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index 347194a8..f28c9066 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -15,6 +15,7 @@ #include #include +#undef recvfrom ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line) { ssize_t ret = 0; diff --git a/src/syscalls/select.c b/src/syscalls/select.c index 83f676ec..5c26352d 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -14,6 +14,7 @@ #include +#undef select int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line) { int ret = 0; diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index 1f63ce77..9c0e165e 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -15,6 +15,7 @@ #include #include +#undef sendto ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen, const char *file, const char *func, const int line) { ssize_t ret = 0; diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 3fcc303e..583e8916 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -21,6 +21,7 @@ void FTLfree(void *ptr, const char*file, const char *func, const int line); int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); // vprintf() is derived from vfprintf(stdout, ...) int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); @@ -36,4 +37,7 @@ int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char // Interrupt-safe file routines FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line); +// Syscall helpers +void syscalls_report_error(const char *error, FILE *stream, const int _errno, const char *format, const char *func, const char *file, const int line); + #endif //SYSCALLS_H diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index 501791ed..bad620b9 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -75,12 +75,15 @@ static int fputs_convert_newline(const char *string, FILE *stream) // Special error reporting for our own vfprintf() // Since we cannot rely on (heap) being available (allocation may have failed // earlier), we do the reporting entirely manually, writing one string at a time -static void report_error(const char *error, FILE *stream, const char *format, const char *func, const char *file, const int line) +void syscalls_report_error(const char *error, FILE *stream, const int _errno, const char *format, const char *func, const char *file, const int line) { + char linestr[16] = { 0 }; + itoa(line, linestr); + fputs("WARN: ", stream); fputs(error, stream); fputs(": ", stream); - fputs(strerror(errno), stream); + fputs(strerror(_errno), stream); fputs("\n Not processing string \"", stream); fputs_convert_newline(format, stream); fputs("\" in ", stream); @@ -88,8 +91,6 @@ static void report_error(const char *error, FILE *stream, const char *format, co fputs("() [", stream); fputs(file, stream); fputs(":", stream); - char linestr[16] = { 0 }; - itoa(line, linestr); fputs(linestr, stream); fputs("]\n", stream); } @@ -102,22 +103,22 @@ int FTLvfprintf(FILE *stream, const char* file, const char *func, const int line int _errno, length = 0; do { - // The va_copy() macro copies the (previously initialized) variable - // argument list args to the local _args. The behavior is as if - // va_start() were applied to _args with the same last argument, - // followed by the same number of va_arg() invocations that was used to - // reach the current state of args. We do this to be able to reuse the - // arguments in args when we need to redo the string preparation - // procedure - va_list _args; - va_copy(_args, args); + // The va_copy() macro copies the (previously initialized) variable + // argument list args to the local _args. The behavior is as if + // va_start() were applied to _args with the same last argument, + // followed by the same number of va_arg() invocations that was used to + // reach the current state of args. We do this to be able to reuse the + // arguments in args when we need to redo the string preparation + // procedure + va_list _args; + va_copy(_args, args); // Reset errno before trying to get the string errno = 0; - // Do the actual string transformation + // Do the actual string transformation length = vasprintf(&buffer, format, _args); - // Copy errno into buffer before calling va_end() - _errno = errno; - va_end(_args); + // Copy errno into buffer before calling va_end() + _errno = errno; + va_end(_args); } // Try again to allocate memory if this failed due to an interruption by // an incoming signal @@ -126,12 +127,12 @@ int FTLvfprintf(FILE *stream, const char* file, const char *func, const int line // Handle other errors than EINTR if(length < 0 || buffer == NULL) { - report_error("vfprintf() failed to allocate memory", - stream, format, func, file, line); + syscalls_report_error("vfprintf() failed to allocate memory", + stream, _errno, format, func, file, line); - // Free the buffer in case anything got allocated - if(buffer != NULL) - free(buffer); + // Free the buffer in case anything got allocated + if(buffer != NULL) + free(buffer); // Return early, there isn't anything we can do here return length; @@ -155,8 +156,8 @@ int FTLvfprintf(FILE *stream, const char* file, const char *func, const int line // EINTR = interrupted system call) if(_buffer < buffer) { - report_error("vfprintf() did not print all characters", - stream, format, func, file, line); + syscalls_report_error("vfprintf() did not print all characters", + stream, errno, format, func, file, line); } // Free allocated memory diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c new file mode 100644 index 00000000..bd1b04ae --- /dev/null +++ b/src/syscalls/vsnprintf.c @@ -0,0 +1,59 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for vsnprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef vsnprintf +int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, size_t maxlen, const char *format, va_list args) +{ + // Sanity check + if(buffer == NULL) + { + syscalls_report_error("vsnprintf() called with NULL buffer", + stdout, 0, format, func, file, line); + return 0; + } + // Print into dynamically allocated memory + int _errno, length = 0; + do + { + // The va_copy() macro copies the (previously initialized) variable + // argument list args to the local _args. The behavior is as if + // va_start() were applied to _args with the same last argument, + // followed by the same number of va_arg() invocations that was used to + // reach the current state of args. We do this to be able to reuse the + // arguments in args when we need to redo the string preparation + // procedure + va_list _args; + va_copy(_args, args); + // Reset errno before trying to get the string + errno = 0; + // Do the actual string transformation + length = vsnprintf(buffer, maxlen, format, _args); + // Copy errno into buffer before calling va_end() + _errno = errno; + va_end(_args); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(length < 0 && _errno == EINTR); + + // Handle other errors than EINTR + if(length < 0) + { + syscalls_report_error("vsnprintf() failed to print into buffer", + stdout, _errno, format, func, file, line); + } + + // Return number of written bytes + return length; +} diff --git a/src/syscalls/write.c b/src/syscalls/write.c index 3ea90641..b5787e89 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -12,6 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" +#undef write ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line) { if(buf == NULL) From 70a4ad13e2ac53be33777904eb57dfd19e8f122a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:06:21 +0100 Subject: [PATCH 22/51] Add interrupt-safe snprintf() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/CMakeLists.txt | 1 + src/syscalls/snprintf.c | 23 +++++++++++++++++++++++ src/syscalls/syscalls.h | 3 ++- src/syscalls/vsnprintf.c | 2 +- 5 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/syscalls/snprintf.c diff --git a/src/FTL.h b/src/FTL.h index 2a1f48be..35edfbac 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -126,6 +126,7 @@ #define fprintf(stream, format, ...) FTLfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) #define vprintf(format, args) FTLvfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) +#define snprintf(buffer, maxlen, format, ...) FTLsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, ##__VA_ARGS__) #define vsnprintf(buffer, maxlen, format, args) FTLvsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) #define accept(sockfd, addr, addrlen) FTLaccept(sockfd, addr, addrlen, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 1597b2c3..847c3ff5 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -20,6 +20,7 @@ set(sources recvfrom.c select.c sendto.c + snprintf.c strdup.c syscalls.h vfprintf.c diff --git a/src/syscalls/snprintf.c b/src/syscalls/snprintf.c new file mode 100644 index 00000000..0460ac15 --- /dev/null +++ b/src/syscalls/snprintf.c @@ -0,0 +1,23 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for snprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) +{ + va_list args; + va_start(args, format); + const int length = FTLvsnprintf(file, func, line, buffer, maxlen, format, args); + va_end(args); + + return length; +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 583e8916..33ef2598 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -21,7 +21,8 @@ void FTLfree(void *ptr, const char*file, const char *func, const int line); int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); // vprintf() is derived from vfprintf(stdout, ...) int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); -int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); +int FTLsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); +int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index bd1b04ae..c195fefe 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -13,7 +13,7 @@ #include "../log.h" #undef vsnprintf -int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, size_t maxlen, const char *format, va_list args) +int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) { // Sanity check if(buffer == NULL) From 20fa5ef6270ec644de3bf7da1c8ce8c22baaf53a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:12:29 +0100 Subject: [PATCH 23/51] Add interrupt-safe vsprintf() routine Signed-off-by: DL6ER --- src/FTL.h | 1 + src/syscalls/fprintf.c | 2 +- src/syscalls/snprintf.c | 2 +- src/syscalls/syscalls.h | 5 ++-- src/syscalls/vfprintf.c | 2 +- src/syscalls/vsnprintf.c | 2 +- src/syscalls/vsprintf.c | 59 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 src/syscalls/vsprintf.c diff --git a/src/FTL.h b/src/FTL.h index 35edfbac..7ca0e1b2 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -126,6 +126,7 @@ #define fprintf(stream, format, ...) FTLfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) #define vprintf(format, args) FTLvfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) +#define vsprintf(buffer, format, args) FTLvsprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, args) #define snprintf(buffer, maxlen, format, ...) FTLsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, ##__VA_ARGS__) #define vsnprintf(buffer, maxlen, format, args) FTLvsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c index 90b88652..be3bee68 100644 --- a/src/syscalls/fprintf.c +++ b/src/syscalls/fprintf.c @@ -12,7 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -int FTLfprintf(FILE *stream, const char* file, const char *func, const int line, const char *format, ...) +int FTLfprintf(FILE *stream, const char *file, const char *func, const int line, const char *format, ...) { va_list args; va_start(args, format); diff --git a/src/syscalls/snprintf.c b/src/syscalls/snprintf.c index 0460ac15..699d942c 100644 --- a/src/syscalls/snprintf.c +++ b/src/syscalls/snprintf.c @@ -12,7 +12,7 @@ //#include "syscalls.h" is implicitly done in FTL.h #include "../log.h" -int FTLsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) +int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) { va_list args; va_start(args, format); diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 33ef2598..15dcca12 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -21,8 +21,9 @@ void FTLfree(void *ptr, const char*file, const char *func, const int line); int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); // vprintf() is derived from vfprintf(stdout, ...) int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); -int FTLsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); -int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); +int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); +int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index bad620b9..f2e65353 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -96,7 +96,7 @@ void syscalls_report_error(const char *error, FILE *stream, const int _errno, co } // The actual vfprintf() routine -int FTLvfprintf(FILE *stream, const char* file, const char *func, const int line, const char *format, va_list args) +int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line, const char *format, va_list args) { // Print into dynamically allocated memory char *buffer = NULL; diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index c195fefe..10f7b6c2 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -13,7 +13,7 @@ #include "../log.h" #undef vsnprintf -int FTLvsnprintf(const char* file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) +int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) { // Sanity check if(buffer == NULL) diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c new file mode 100644 index 00000000..b31ba414 --- /dev/null +++ b/src/syscalls/vsprintf.c @@ -0,0 +1,59 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for vsprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef vsprintf +int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) +{ + // Sanity check + if(buffer == NULL) + { + syscalls_report_error("vsprintf() called with NULL buffer", + stdout, 0, format, func, file, line); + return 0; + } + // Print into dynamically allocated memory + int _errno, length = 0; + do + { + // The va_copy() macro copies the (previously initialized) variable + // argument list args to the local _args. The behavior is as if + // va_start() were applied to _args with the same last argument, + // followed by the same number of va_arg() invocations that was used to + // reach the current state of args. We do this to be able to reuse the + // arguments in args when we need to redo the string preparation + // procedure + va_list _args; + va_copy(_args, args); + // Reset errno before trying to get the string + errno = 0; + // Do the actual string transformation + length = vsprintf(buffer, format, _args); + // Copy errno into buffer before calling va_end() + _errno = errno; + va_end(_args); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(length < 0 && _errno == EINTR); + + // Handle other errors than EINTR + if(length < 0) + { + syscalls_report_error("vsprintf() failed to print into buffer", + stdout, _errno, format, func, file, line); + } + + // Return number of written bytes + return length; +} From ab0daeb7bb17b05aaf6bff2ba503a826fb7c6802 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:16:06 +0100 Subject: [PATCH 24/51] Add interrupt-safe sprintf() routine Signed-off-by: DL6ER --- src/FTL.h | 2 ++ src/syscalls/CMakeLists.txt | 2 ++ src/syscalls/sprintf.c | 23 +++++++++++++++++++++++ src/syscalls/syscalls.h | 5 ++++- 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/syscalls/sprintf.c diff --git a/src/FTL.h b/src/FTL.h index 7ca0e1b2..b97ca183 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -118,6 +118,7 @@ // and report accordingly in the log. This will make debugging FTL crashs // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. +#undef strdup // strdup() is a macro in itself, it needs special handling #define free(ptr) FTLfree(ptr, __FILE__, __FUNCTION__, __LINE__) #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) @@ -126,6 +127,7 @@ #define fprintf(stream, format, ...) FTLfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) #define vprintf(format, args) FTLvfprintf(stdout, __FILE__, __FUNCTION__, __LINE__, format, args) #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) +#define sprintf(buffer, format, ...) FTLsprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, ##__VA_ARGS__) #define vsprintf(buffer, format, args) FTLvsprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, args) #define snprintf(buffer, maxlen, format, ...) FTLsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, ##__VA_ARGS__) #define vsnprintf(buffer, maxlen, format, args) FTLvsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, args) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 847c3ff5..fe0940a4 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -21,10 +21,12 @@ set(sources select.c sendto.c snprintf.c + sprintf.c strdup.c syscalls.h vfprintf.c vsnprintf.c + vsprintf.c write.c ) diff --git a/src/syscalls/sprintf.c b/src/syscalls/sprintf.c new file mode 100644 index 00000000..a6cc4094 --- /dev/null +++ b/src/syscalls/sprintf.c @@ -0,0 +1,23 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for sprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) +{ + va_list args; + va_start(args, format); + const int length = FTLvsprintf(file, func, line, buffer, format, args); + va_end(args); + + return length; +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 15dcca12..07f2f7dc 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -18,10 +18,13 @@ void FTLfree(void *ptr, const char*file, const char *func, const int line); // Interrupt-safe printing routines // printf() is derived from fprintf(stdout, ...) -int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); // vprintf() is derived from vfprintf(stdout, ...) +int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); + +int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); + int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); From cd7fc84c82761057ecc912f90b8e2a4473a37d24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:43:30 +0100 Subject: [PATCH 25/51] Show complete list of args when complaining about unsupported argument Signed-off-by: DL6ER --- src/args.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index b196a83f..259cf078 100644 --- a/src/args.c +++ b/src/args.c @@ -350,7 +350,15 @@ void parse_args(int argc, char* argv[]) // Complain if invalid options have been found if(!ok) { - printf("pihole-FTL: invalid option -- '%s'\nTry '%s --help' for more information\n", argv[i], argv[0]); + printf("pihole-FTL: invalid option -- '%s'\n", argv[i]); + printf("Command: \""); + for(int j = 0; j < argc; j++) + { + printf("%s", argv[j]); + if(j < argc - 1) + printf(" "); + } + printf("\"\n\nTry '%s --help' for more information\n", argv[0]); exit(EXIT_FAILURE); } } From 7580a6f7b3bdf564ca43f0828587cb4077402af2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 21:52:30 +0100 Subject: [PATCH 26/51] Adjust test for unknown argument to support the new format Signed-off-by: DL6ER --- src/args.c | 4 ++-- test/test_suite.bats | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/args.c b/src/args.c index 259cf078..b488ce07 100644 --- a/src/args.c +++ b/src/args.c @@ -351,14 +351,14 @@ void parse_args(int argc, char* argv[]) if(!ok) { printf("pihole-FTL: invalid option -- '%s'\n", argv[i]); - printf("Command: \""); + printf("Command: '"); for(int j = 0; j < argc; j++) { printf("%s", argv[j]); if(j < argc - 1) printf(" "); } - printf("\"\n\nTry '%s --help' for more information\n", argv[0]); + printf("'\nTry '%s --help' for more information\n", argv[0]); exit(EXIT_FAILURE); } } diff --git a/test/test_suite.bats b/test/test_suite.bats index 54980509..84ec9051 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -473,7 +473,8 @@ run bash -c '/home/pihole/pihole-FTL abc' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "pihole-FTL: invalid option -- 'abc'" ]] - [[ ${lines[1]} == "Try '/home/pihole/pihole-FTL --help' for more information" ]] + [[ ${lines[1]} == "Command: '/home/pihole/pihole-FTL abc'" ]] + [[ ${lines[2]} == "Try '/home/pihole/pihole-FTL --help' for more information" ]] } @test "Help CLI argument return help text" { From 4610a0574926de98c7eb09f34c60fb691d526ff9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 22:05:50 +0100 Subject: [PATCH 27/51] Expose lua-interpreter as virtual pihole-lua binary Signed-off-by: DL6ER --- src/args.c | 89 ++++++++++------------------------------------- src/lua/ftl_lua.c | 59 +++++++++++++++++++++++++++++++ src/lua/ftl_lua.h | 3 ++ 3 files changed, 81 insertions(+), 70 deletions(-) diff --git a/src/args.c b/src/args.c index b488ce07..297f8024 100644 --- a/src/args.c +++ b/src/args.c @@ -27,8 +27,6 @@ #include "shmem.h" // LUA dependencies #include "lua/ftl_lua.h" -#include -#include // run_dhcp_discover() #include "dhcp-discover.h" // defined in dnsmasq.c @@ -58,11 +56,30 @@ void parse_args(int argc, char* argv[]) if(strEndsWith(argv[0], "dnsmasq")) consume_for_dnsmasq = true; + if(strEndsWith(argv[0], "pihole-lua")) + exit(run_lua_interpreter(argc, argv, false)); + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { bool ok = false; + // Expose internal lua interpreter + if(strcmp(argv[i], "lua") == 0 || + strcmp(argv[i], "--lua") == 0) + { + exit(run_lua_interpreter(argc - i, &argv[i], dnsmasq_debug)); + } + + // Expose internal lua compiler + if(strcmp(argv[i], "luac") == 0 || + strcmp(argv[i], "--luac") == 0) + { + if(argc == i + 1) // No arguments after this one + printf("Pi-hole FTL %s\n", get_FTL_version()); + exit(luac_main(argc - i, &argv[i])); + } + // Implement dnsmasq's test function, no need to prepare the entire FTL // environment (initialize shared memory, lead queries from long-term // database, ...) when the task is a simple (dnsmasq) syntax check @@ -279,74 +296,6 @@ void parse_args(int argc, char* argv[]) exit(EXIT_SUCCESS); } - // Expose internal lua interpreter - if(strcmp(argv[i], "lua") == 0 || - strcmp(argv[i], "--lua") == 0) - { - if(argc == i + 1) // No arguments after this one - printf("Pi-hole FTL %s\n", get_FTL_version()); -#if defined(LUA_USE_READLINE) - wordexp_t word; - wordexp(LUA_HISTORY_FILE, &word, WRDE_NOCMD); - const char *history_file = NULL; - if(word.we_wordc == 1) - { - history_file = word.we_wordv[0]; - const int ret_r = read_history(history_file); - if(dnsmasq_debug) - { - printf("Reading history ... "); - if(ret_r == 0) - printf("success\n"); - else - printf("error - %s: %s\n", history_file, strerror(ret_r)); - } - - // The history file may not exist, try to create an empty one in this case - if(ret_r == ENOENT) - { - if(dnsmasq_debug) - { - printf("Creating new history file: %s\n", history_file); - } - FILE *history = fopen(history_file, "w"); - if(history != NULL) - fclose(history); - } - } -#else - if(dnsmasq_debug) - printf("No readline available!\n"); -#endif - const int ret = lua_main(argc - i, &argv[i]); -#if defined(LUA_USE_READLINE) - if(history_file != NULL) - { - const int ret_w = write_history(history_file); - if(dnsmasq_debug) - { - printf("Writing history ... "); - if(ret_w == 0) - printf("success\n"); - else - printf("error - %s: %s\n", history_file, strerror(ret_w)); - } - - wordfree(&word); - } -#endif - exit(ret); - } - - // Expose internal lua compiler - if(strcmp(argv[i], "luac") == 0 || - strcmp(argv[i], "--luac") == 0) - { - if(argc == i + 1) // No arguments after this one - printf("Pi-hole FTL %s\n", get_FTL_version()); - exit(luac_main(argc - i, &argv[i])); - } - // Complain if invalid options have been found if(!ok) { diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 649d0984..e7edd12a 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -14,6 +14,65 @@ #include "lauxlib.h" // get_FTL_version() #include "../log.h" +#include +#include + +int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug) +{ + if(argc == 1) // No arguments after this one + printf("Pi-hole FTL %s\n", get_FTL_version()); +#if defined(LUA_USE_READLINE) + wordexp_t word; + wordexp(LUA_HISTORY_FILE, &word, WRDE_NOCMD); + const char *history_file = NULL; + if(word.we_wordc == 1) + { + history_file = word.we_wordv[0]; + const int ret_r = read_history(history_file); + if(dnsmasq_debug) + { + printf("Reading history ... "); + if(ret_r == 0) + printf("success\n"); + else + printf("error - %s: %s\n", history_file, strerror(ret_r)); + } + + // The history file may not exist, try to create an empty one in this case + if(ret_r == ENOENT) + { + if(dnsmasq_debug) + { + printf("Creating new history file: %s\n", history_file); + } + FILE *history = fopen(history_file, "w"); + if(history != NULL) + fclose(history); + } + } +#else + if(dnsmasq_debug) + printf("No readline available!\n"); +#endif + const int ret = lua_main(argc, argv); +#if defined(LUA_USE_READLINE) + if(history_file != NULL) + { + const int ret_w = write_history(history_file); + if(dnsmasq_debug) + { + printf("Writing history ... "); + if(ret_w == 0) + printf("success\n"); + else + printf("error - %s: %s\n", history_file, strerror(ret_w)); + } + + wordfree(&word); + } +#endif + return ret; +} // pihole.ftl_version() static int pihole_ftl_version(lua_State *L) { diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index b123178f..a8ce39b0 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -11,9 +11,12 @@ #define FTL_LUA_H #include "lua.h" +#include #define LUA_HISTORY_FILE "~/.pihole_lua_history" +int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug); + int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); From 794e8fae95c308642db4f4dacf52aca39140465d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 22:46:51 +0100 Subject: [PATCH 28/51] Add drop-in support for lua binary Signed-off-by: DL6ER --- src/args.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index 297f8024..571ea1ac 100644 --- a/src/args.c +++ b/src/args.c @@ -56,7 +56,7 @@ void parse_args(int argc, char* argv[]) if(strEndsWith(argv[0], "dnsmasq")) consume_for_dnsmasq = true; - if(strEndsWith(argv[0], "pihole-lua")) + if(strEndsWith(argv[0], "lua")) exit(run_lua_interpreter(argc, argv, false)); // start from 1, as argv[0] is the executable name From 112beb42144ef1c344a98e6c4383e0cf981cec30 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Dec 2020 22:50:52 +0100 Subject: [PATCH 29/51] Add drop-in replacement support for luac as well Signed-off-by: DL6ER --- src/args.c | 5 +++-- src/lua/ftl_lua.c | 7 +++++++ src/lua/ftl_lua.h | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/args.c b/src/args.c index 571ea1ac..9d144311 100644 --- a/src/args.c +++ b/src/args.c @@ -59,6 +59,9 @@ void parse_args(int argc, char* argv[]) if(strEndsWith(argv[0], "lua")) exit(run_lua_interpreter(argc, argv, false)); + if(strEndsWith(argv[0], "luac")) + exit(run_luac(argc, argv)); + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { @@ -75,8 +78,6 @@ void parse_args(int argc, char* argv[]) if(strcmp(argv[i], "luac") == 0 || strcmp(argv[i], "--luac") == 0) { - if(argc == i + 1) // No arguments after this one - printf("Pi-hole FTL %s\n", get_FTL_version()); exit(luac_main(argc - i, &argv[i])); } diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index e7edd12a..2da6cb22 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -74,6 +74,13 @@ int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug) return ret; } +int run_luac(const int argc, char **argv) +{ + if(argc == 1) // No arguments after this one + printf("Pi-hole FTL %s\n", get_FTL_version()); + return luac_main(argc, argv); +} + // pihole.ftl_version() static int pihole_ftl_version(lua_State *L) { lua_pushstring(L, get_FTL_version()); diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index a8ce39b0..c2ef12ab 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -16,6 +16,7 @@ #define LUA_HISTORY_FILE "~/.pihole_lua_history" int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug); +int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); From 1070469b762e3fde03ae68c1cf6e9aab873dc3d9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 10 Dec 2020 04:09:05 +0100 Subject: [PATCH 30/51] Fix freeing regex pointers to set the global not the local object to NULL after free(). Signed-off-by: DL6ER --- src/regex.c | 58 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/regex.c b/src/regex.c index 818981f2..1af82733 100644 --- a/src/regex.c +++ b/src/regex.c @@ -33,8 +33,7 @@ static regex_data *white_regex = NULL; static regex_data *black_regex = NULL; static regex_data *cli_regex = NULL; -regex_data *get_regex_from_type(const enum regex_type regexid); -inline regex_data *get_regex_from_type(const enum regex_type regexid) +static inline regex_data *get_regex_ptr(const enum regex_type regexid) { switch (regexid) { @@ -50,12 +49,39 @@ inline regex_data *get_regex_from_type(const enum regex_type regexid) } } +static inline void free_regex_ptr(const enum regex_type regexid) +{ + regex_data **regex; + switch (regexid) + { + case REGEX_BLACKLIST: + regex = &black_regex; + break; + case REGEX_WHITELIST: + regex = &white_regex; + break; + case REGEX_CLI: + regex = &cli_regex; + break; + case REGEX_MAX: // Fall through + default: // This is not possible + return; + } + + // Free pointer (if not already NULL) + if(*regex != NULL) + { + free(*regex); + *regex = NULL; + } +} + #define FTL_REGEX_SEP ";" /* Compile regular expressions into data structures that can be used with regexec() to match against a string */ static bool compile_regex(const char *regexin, const enum regex_type regexid) { - regex_data *regex = get_regex_from_type(regexid); + regex_data *regex = get_regex_ptr(regexid); int index = counters->num_regex[regexid]++; // Extract possible Pi-hole extensions @@ -163,7 +189,7 @@ int match_regex(const char *input, const DNSCacheData* dns_cache, const int clie const enum regex_type regexid, const bool regextest) { int match_idx = -1; - regex_data *regex = get_regex_from_type(regexid); + regex_data *regex = get_regex_ptr(regexid); #ifdef USE_TRE_REGEX regmatch_t match = { 0 }; // This also disables any sub-matching #endif @@ -309,11 +335,20 @@ static void free_regex(void) // Free regex datastructure // Loop over regex types - for(unsigned char regexid = 0; regexid < REGEX_MAX; regexid++) + for(enum regex_type regexid = REGEX_BLACKLIST; regexid < REGEX_MAX; regexid++) { - regex_data *regex = get_regex_from_type(regexid); + regex_data *regex = get_regex_ptr(regexid); + + // Reset counter for number of regex + const unsigned int oldcount = counters->num_regex[regexid]; + counters->num_regex[regexid] = 0; + + // Exit early if the regex has already been freed (or has never been used) + if(regex == NULL) + continue; + // Loop over entries with this regex type - for(unsigned int index = 0; index < counters->num_regex[regexid]; index++) + for(unsigned int index = 0; index < oldcount; index++) { if(!regex[index].available) continue; @@ -329,14 +364,7 @@ static void free_regex(void) } // Free array with regex datastructure - if(regex != NULL) - { - free(regex); - regex = NULL; - } - - // Reset counter for number of regex - counters->num_regex[regexid] = 0; + free_regex_ptr(regexid); } } From 85c8af44de3479d0074c1239836b710c66f332ef Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 10 Dec 2020 22:01:47 +0100 Subject: [PATCH 31/51] Add interrupt-safe asprintf() and vasprintf() routines Signed-off-by: DL6ER --- src/FTL.h | 2 ++ src/syscalls/CMakeLists.txt | 2 ++ src/syscalls/asprintf.c | 23 +++++++++++++++ src/syscalls/syscalls.h | 3 ++ src/syscalls/vasprintf.c | 59 +++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+) create mode 100644 src/syscalls/asprintf.c create mode 100644 src/syscalls/vasprintf.c diff --git a/src/FTL.h b/src/FTL.h index b97ca183..05f4f34d 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -129,6 +129,8 @@ #define vfprintf(stream, format, args) FTLvfprintf(stream, __FILE__, __FUNCTION__, __LINE__, format, args) #define sprintf(buffer, format, ...) FTLsprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, ##__VA_ARGS__) #define vsprintf(buffer, format, args) FTLvsprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, args) +#define asprintf(buffer, format, ...) FTLasprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, ##__VA_ARGS__) +#define vasprintf(buffer, format, args) FTLvasprintf(__FILE__, __FUNCTION__, __LINE__, buffer, format, args) #define snprintf(buffer, maxlen, format, ...) FTLsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, ##__VA_ARGS__) #define vsnprintf(buffer, maxlen, format, args) FTLvsnprintf(__FILE__, __FUNCTION__, __LINE__, buffer, maxlen, format, args) #define write(fd, buf, n) FTLwrite(fd, buf, n, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index fe0940a4..e1b971e7 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -10,6 +10,7 @@ set(sources accept.c + asprintf.c calloc.c fopen.c fprintf.c @@ -24,6 +25,7 @@ set(sources sprintf.c strdup.c syscalls.h + vasprintf.c vfprintf.c vsnprintf.c vsprintf.c diff --git a/src/syscalls/asprintf.c b/src/syscalls/asprintf.c new file mode 100644 index 00000000..4da1ea82 --- /dev/null +++ b/src/syscalls/asprintf.c @@ -0,0 +1,23 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for asprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) +{ + va_list args; + va_start(args, format); + const int length = FTLvasprintf(file, func, line, buffer, format, args); + va_end(args); + + return length; +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 07f2f7dc..230068b6 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -25,6 +25,9 @@ int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); +int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); + int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c new file mode 100644 index 00000000..8300a8c5 --- /dev/null +++ b/src/syscalls/vasprintf.c @@ -0,0 +1,59 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for vasprintf +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef vasprintf +int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) +{ + // Sanity check + if(buffer == NULL) + { + syscalls_report_error("vasprintf() called with NULL buffer", + stdout, 0, format, func, file, line); + return 0; + } + // Print into dynamically allocated memory + int _errno, length = 0; + do + { + // The va_copy() macro copies the (previously initialized) variable + // argument list args to the local _args. The behavior is as if + // va_start() were applied to _args with the same last argument, + // followed by the same number of va_arg() invocations that was used to + // reach the current state of args. We do this to be able to reuse the + // arguments in args when we need to redo the string preparation + // procedure + va_list _args; + va_copy(_args, args); + // Reset errno before trying to get the string + errno = 0; + // Do the actual string transformation + length = vasprintf(buffer, format, _args); + // Copy errno into buffer before calling va_end() + _errno = errno; + va_end(_args); + } + // Try again to allocate memory if this failed due to an interruption by + // an incoming signal + while(length < 0 && _errno == EINTR); + + // Handle other errors than EINTR + if(length < 0) + { + syscalls_report_error("vasprintf() failed to print into buffer", + stdout, _errno, format, func, file, line); + } + + // Return number of written bytes + return length; +} From 0e0899691f1e0328b4ddf2bbcbc01503ec7c544f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 10 Dec 2020 04:19:44 +0100 Subject: [PATCH 32/51] Add more debugging output for domain reloading (on receipt of SIGHUP) Signed-off-by: DL6ER --- src/database/gravity-db.c | 9 +++++++++ src/datastructure.c | 8 +++++++- src/regex.c | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index de8da8b6..6f24128f 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1087,6 +1087,12 @@ int gravityDB_count(const enum gravity_tables list) // Finalize statement gravityDB_finalizeTable(); + if(config.debug & DEBUG_DATABASE) + { + logg("gravityDB_count(%d): %i entries in %s", + list, result, tablename[list]); + } + // Return result return result; } @@ -1294,6 +1300,9 @@ bool in_auditlist(const char *domain) bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regex_data *regex, const unsigned char type, const char* table, const int clientID) { + if(config.debug & DEBUG_REGEX) + logg("Getting regex client groups for client with ID %i", clientID); + char *querystr = NULL; if(!client->found_group && !get_client_groupids(client)) return false; diff --git a/src/datastructure.c b/src/datastructure.c index dfa0bd24..317cfea0 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -25,6 +25,8 @@ #include "database/aliasclients.h" // piholeFTLDB_reopen() #include "database/common.h" +// config struct +#include "config.h" const char *querytypes[TYPE_MAX] = {"UNKNOWN", "A", "AAAA", "ANY", "SRV", "SOA", "PTR", "TXT", "NAPTR", "MX", "DS", "RRSIG", "DNSKEY", "NS", "OTHER"}; @@ -453,13 +455,17 @@ const char *getClientNameString(const queriesData* query) void FTL_reset_per_client_domain_data(void) { + if(config.debug & DEBUG_DATABASE) + logg("Resetting per-client DNS cache, size is %i", counters->dns_cache_size); + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) { // Reset all blocking yes/no fields for all domains and clients // This forces a reprocessing of all available filters for any // given domain and client the next time they are seen DNSCacheData *dns_cache = getDNSCache(cacheID, true); - dns_cache->blocking_status = UNKNOWN_BLOCKED; + if(dns_cache != NULL) + dns_cache->blocking_status = UNKNOWN_BLOCKED; } } diff --git a/src/regex.c b/src/regex.c index 1af82733..7c0a1896 100644 --- a/src/regex.c +++ b/src/regex.c @@ -318,16 +318,19 @@ int match_regex(const char *input, const DNSCacheData* dns_cache, const int clie static void free_regex(void) { - // Reset FTL's DNS cache - FTL_reset_per_client_domain_data(); - // Return early if we don't use any regex filters if(white_regex == NULL && black_regex == NULL && cli_regex == NULL) + { + if(config.debug & DEBUG_DATABASE) + logg("Not using any regex filters, nothing to free or reset"); return; + } // Reset client configuration + if(config.debug & DEBUG_DATABASE) + logg("Resetting per-client regex settings"); for(int clientID = 0; clientID < counters->clients; clientID++) { reset_per_client_regex(clientID); @@ -347,6 +350,12 @@ static void free_regex(void) if(regex == NULL) continue; + if(config.debug & DEBUG_DATABASE) + { + logg("Going to free %i entries in %s regex struct", + oldcount, regextype[regexid]); + } + // Loop over entries with this regex type for(unsigned int index = 0; index < oldcount; index++) { @@ -363,6 +372,11 @@ static void free_regex(void) } } + if(config.debug & DEBUG_DATABASE) + { + logg("Loop done, freeing regex pointer (%p)", regex); + } + // Free array with regex datastructure free_regex_ptr(regexid); } @@ -377,7 +391,7 @@ void reload_per_client_regex(const int clientID, clientsData *client) // Ensure there is enough memory in the shared memory object add_per_client_regex(clientID); - // Zero-initialize(or wipe previous) regex + // Zero-initialize (or wipe previous) regex reset_per_client_regex(clientID); // Load regex per-group regex blacklist for this client @@ -398,6 +412,9 @@ static void read_regex_table(const enum regex_type regexid) // Get table ID const enum gravity_tables tableID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE; + if(config.debug & DEBUG_DATABASE) + logg("Reading regex %s from database", regextype[regexid]); + // Get number of lines in the regex table counters->num_regex[regexid] = 0; int count = gravityDB_count(tableID); @@ -469,6 +486,13 @@ static void read_regex_table(const enum regex_type regexid) // Finalize statement and close gravity database handle gravityDB_finalizeTable(); + + if(config.debug & DEBUG_DATABASE) + { + logg("Read %i %s regex entries", + counters->num_regex[regexid], + regextype[regexid]); + } } void read_regex_from_database(void) @@ -490,6 +514,8 @@ void read_regex_from_database(void) // Loop over all clients and ensure we have enough space and load // per-client regex data, not all of the regex read and compiled above // will also be used by all clients + if(config.debug & DEBUG_DATABASE) + logg("Loading per-client regex data"); for(int clientID = 0; clientID < counters->clients; clientID++) { // Get client pointer From ef02fa5ede6a33d7066863d0c2357c18cf4796a1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 17 Dec 2020 09:39:20 +0100 Subject: [PATCH 33/51] Add REFRESH_HOSTNAMES=UNKNWON to support only refreshing recently active clients with unknown hostnames Signed-off-by: DL6ER --- src/config.c | 5 +++++ src/enums.h | 1 + src/resolve.c | 10 +++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 7d840524..85c6d58c 100644 --- a/src/config.c +++ b/src/config.c @@ -438,6 +438,11 @@ void read_FTLconf(void) config.refresh_hostnames = REFRESH_NONE; logg(" REFRESH_HOSTNAMES: Not periodically refreshing names"); } + else if(buffer != NULL && strcasecmp(buffer, "UNKNOWN") == 0) + { + config.refresh_hostnames = REFRESH_NONE; + logg(" REFRESH_HOSTNAMES: Only refreshing recently active clients with unknown hostnames"); + } else { config.refresh_hostnames = REFRESH_IPV4_ONLY; diff --git a/src/enums.h b/src/enums.h index 35caf92c..fbccba59 100644 --- a/src/enums.h +++ b/src/enums.h @@ -153,6 +153,7 @@ enum events { enum refresh_hostnames { REFRESH_ALL, REFRESH_IPV4_ONLY, + REFRESH_UNKNOWN, REFRESH_NONE } __attribute__ ((packed)); diff --git a/src/resolve.c b/src/resolve.c index cd71996e..bee3dd9a 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -419,10 +419,14 @@ static void resolveClients(const bool onlynew) IPv6 = true; // If we're in refreshing mode (onlynew == false), we skip clients if - // either IPv4-only or none is selected + // 1. We should not refresh any hostnames + // 2. We should only refresh IPv4 client, but this client is IPv6 + // 3. We should only refresh unknown hostnames, but leave + // existing ones as they are if(onlynew == false && - (config.refresh_hostnames == REFRESH_NONE || - (config.refresh_hostnames == REFRESH_IPV4_ONLY && IPv6))) + (config.refresh_hostnames == REFRESH_NONE || + (config.refresh_hostnames == REFRESH_IPV4_ONLY && IPv6) || + (config.refresh_hostnames == REFRESH_UNKNOWN && oldnamepos != 0))) { if(config.debug & DEBUG_RESOLVER) { From 65dd6f93f698ff12ac89de4b46caf69d6b8553e9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 17 Dec 2020 09:46:43 +0100 Subject: [PATCH 34/51] Force refreshing of hostnames (according to REFRESH_HOSTNAMES config) on receipt of SIGRT4 Signed-off-by: DL6ER --- src/enums.h | 1 + src/events.c | 2 ++ src/resolve.c | 18 +++++++++++++----- src/signals.c | 4 +++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/enums.h b/src/enums.h index fbccba59..447bf50a 100644 --- a/src/enums.h +++ b/src/enums.h @@ -145,6 +145,7 @@ enum events { RELOAD_GRAVITY, RELOAD_PRIVACY_LEVEL, RERESOLVE_HOSTNAMES, + RERESOLVE_HOSTNAMES_FORCE, REIMPORT_ALIASCLIENTS, PARSE_NEIGHBOR_CACHE, EVENTS_MAX diff --git a/src/events.c b/src/events.c index 84583702..06280707 100644 --- a/src/events.c +++ b/src/events.c @@ -92,6 +92,8 @@ static const char *eventtext(const enum events event) return "RELOAD_PRIVACY_LEVEL"; case RERESOLVE_HOSTNAMES: return "RERESOLVE_HOSTNAMES"; + case RERESOLVE_HOSTNAMES_FORCE: + return "RERESOLVE_HOSTNAMES_FORCE"; case REIMPORT_ALIASCLIENTS: return "REIMPORT_ALIASCLIENTS"; case PARSE_NEIGHBOR_CACHE: diff --git a/src/resolve.c b/src/resolve.c index bee3dd9a..24576530 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -350,7 +350,7 @@ static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) } // Resolve client host names -static void resolveClients(const bool onlynew) +static void resolveClients(const bool onlynew, const bool force_refreshing) { const time_t now = time(NULL); // Lock counter access here, we use a copy in the following loop @@ -386,7 +386,7 @@ static void resolveClients(const bool onlynew) // Only try to resolve host names of clients which were recently active if we are re-resolving // Limit for a "recently active" client is two hours ago - if(onlynew == false && client->lastQuery < now - 2*60*60) + if(!force_refreshing && onlynew == false && client->lastQuery < now - 2*60*60) { if(config.debug & DEBUG_RESOLVER) { @@ -401,7 +401,7 @@ static void resolveClients(const bool onlynew) // If onlynew flag is set, we will only resolve new clients // If not, we will try to re-resolve all known clients - if(onlynew && !newflag) + if(!force_refreshing && onlynew && !newflag) { if(config.debug & DEBUG_RESOLVER) { @@ -563,7 +563,8 @@ void *DNSclient_thread(void *val) if(resolver_ready && (time(NULL) % RESOLVE_INTERVAL == 0)) { // Try to resolve new client host names (onlynew=true) - resolveClients(true); + // We're not forcing refreshing here + resolveClients(true, false); // Try to resolve new upstream destination host names (onlynew=true) resolveUpstreams(true); // Prevent immediate re-run of this routine @@ -576,11 +577,18 @@ void *DNSclient_thread(void *val) set_event(RERESOLVE_HOSTNAMES); // done below } + bool force_refreshing = false; + if(get_and_clear_event(RERESOLVE_HOSTNAMES_FORCE)) + { + set_event(RERESOLVE_HOSTNAMES); // done below + force_refreshing = true; + } + // Process resolver related event queue elements if(get_and_clear_event(RERESOLVE_HOSTNAMES)) { // Try to resolve all client host names (onlynew=false) - resolveClients(false); + resolveClients(false, force_refreshing); // Try to resolve all upstream destination host names (onlynew=false) resolveUpstreams(false); // Prevent immediate re-run of this routine diff --git a/src/signals.c b/src/signals.c index b5ec061d..ec370507 100644 --- a/src/signals.c +++ b/src/signals.c @@ -283,7 +283,9 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) else if(rtsig == 4) { // Re-resolve all clients and forward destinations - set_event(RERESOLVE_HOSTNAMES); + // Force refreshing hostnames according to + // REFRESH_HOSTNAMES config option + set_event(RERESOLVE_HOSTNAMES_FORCE); } else if(rtsig == 5) { From d46acb547d952114d3794372ea754ea3c63b763e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 17 Dec 2020 11:32:26 +0100 Subject: [PATCH 35/51] Give explicit reason for skipping in debug message Signed-off-by: DL6ER --- src/config.c | 2 +- src/resolve.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 85c6d58c..e118d74d 100644 --- a/src/config.c +++ b/src/config.c @@ -440,7 +440,7 @@ void read_FTLconf(void) } else if(buffer != NULL && strcasecmp(buffer, "UNKNOWN") == 0) { - config.refresh_hostnames = REFRESH_NONE; + config.refresh_hostnames = REFRESH_UNKNOWN; logg(" REFRESH_HOSTNAMES: Only refreshing recently active clients with unknown hostnames"); } else diff --git a/src/resolve.c b/src/resolve.c index 24576530..7b49048b 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -430,8 +430,16 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) { if(config.debug & DEBUG_RESOLVER) { - logg("Skipping client %s (%s) because it should not be refreshed", - getstr(ippos), getstr(oldnamepos)); + const char *reason = "N/A"; + if(config.refresh_hostnames == REFRESH_NONE) + reason = "Not refreshing any hostnames"; + else if(config.refresh_hostnames == REFRESH_IPV4_ONLY) + reason = "Only refreshing IPv4 names"; + else if(config.refresh_hostnames == REFRESH_UNKNOWN) + reason = "Looking only for unknown hostnames"; + + logg("Skipping client %s (%s) because it should not be refreshed: %s", + getstr(ippos), getstr(oldnamepos), reason); } skipped++; continue; From 10124fe6b0b81a781d030007befcd1d56fb20a6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 17 Dec 2020 12:16:11 +0100 Subject: [PATCH 36/51] Fall back to using ftruncate() when fallocate() return with "Operation not supported". This may happen if the kernel is older than 2.6.23 or glibc older than 2.10. ftruncate() has its own disadvantages, however, it is POSIX compliant (POSIX.1-2001) so should be supported even by ancient kernels. Signed-off-by: DL6ER --- src/shmem.c | 64 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/src/shmem.c b/src/shmem.c index bacd53fe..f69a8a91 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -511,12 +511,35 @@ SharedMemory create_shm(const char *name, const size_t size, bool create_new) // Using fallocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = fallocate(fd, 0, 0U, size); + int ret = fallocate(fd, 0, 0U, size); + const int fallocate_errno = errno; if(ret != 0) { - logg("FATAL: create_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory.name, fd, size, strerror(errno), ret); - exit(EXIT_FAILURE); + // Try again with ftruncate() if fallocate() failed with + // "Operation not supported" + if(errno == ENOTSUP) + { + ret = ftruncate(fd, size); + + // Report info that space has only been reserved, not + // really allocated + if(ret == 0) + { + logg("INFO: create_shm(): Failed to allocate %zu bytes for \"%s\" (%s, %i), "\ + "space has only been reserved.", + size, sharedMemory.name, strerror(errno), errno); + } + } + if(ret != 0) + { + logg("FATAL: create_shm(): Failed to resize \"%s\" (%i) to %zu:", + sharedMemory.name, fd, size); + logg(" fallocate returned: %s (%i)", + strerror(fallocate_errno), fallocate_errno); + logg(" ftruncate returned: %s (%i)", + strerror(errno), errno); + exit(EXIT_FAILURE); + } } // Create shared memory mapping @@ -635,12 +658,35 @@ bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const size_t si // Using fallocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = fallocate(fd, 0, 0U, size); - if(ret != 0) + int ret = fallocate(fd, 0, 0U, size); + const int fallocate_errno = errno; + if(ret == -1) { - logg("FATAL: realloc_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory->name, fd, size, strerror(errno), ret); - exit(EXIT_FAILURE); + // Try again with ftruncate() if fallocate() failed with + // "Operation not supported" + if(errno == ENOTSUP) + { + ret = ftruncate(fd, size); + + // Report info that space has only been reserved, not + // really allocated + if(ret == 0) + { + logg("INFO: realloc_shm(): Failed to allocate %zu bytes for \"%s\" (%s), "\ + "space has only been reserved.", + size, sharedMemory->name, strerror(errno)); + } + } + if(ret == -1) + { + logg("FATAL: realloc_shm(): Failed to resize \"%s\" (%i) to %zu:", + sharedMemory->name, fd, size); + logg(" fallocate returned: %s (%i)", + strerror(fallocate_errno), fallocate_errno); + logg(" ftruncate returned: %s (%i)", + strerror(errno), errno); + exit(EXIT_FAILURE); + } } // Close shared memory object file descriptor as it is no longer From 302abf80bf16e1ab153e3a0f9ea532576f3e0c74 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 18 Dec 2020 07:02:40 +0100 Subject: [PATCH 37/51] Add new DEBUG_EXTRA flag used for special (temporary) debugging Signed-off-by: DL6ER --- src/config.c | 16 ++++++++++++++-- src/enums.h | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index 64fdffa8..ef41b6fa 100644 --- a/src/config.c +++ b/src/config.c @@ -495,7 +495,7 @@ static char *parse_FTLconf(FILE *fp, const char * key) if(fp == NULL) return NULL; - char * keystr = calloc(strlen(key)+2,sizeof(char)); + char *keystr = calloc(strlen(key)+2, sizeof(char)); if(keystr == NULL) { logg("WARN: parse_FTLconf failed: could not allocate memory for keystr"); @@ -505,10 +505,17 @@ static char *parse_FTLconf(FILE *fp, const char * key) // Go to beginning of file fseek(fp, 0L, SEEK_SET); + + logg("initial: conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); errno = 0; while(getline(&conflinebuffer, &size, fp) != -1) { + if(config.debug & DEBUG_EXTRA) + { + logg("conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); + logg(" while reading line \"%s\" looking for \"%s\"", conflinebuffer, keystr); + } // Check if memory allocation failed if(conflinebuffer == NULL) break; @@ -525,7 +532,7 @@ static char *parse_FTLconf(FILE *fp, const char * key) free(keystr); // Note: value is still a pointer into the conflinebuffer // its memory will get released in release_config_memory() - char* value = find_equals(conflinebuffer) + 1; + char *value = find_equals(conflinebuffer) + 1; // Trim whitespace at beginning and end, this function // modifies the string inplace trim_whitespace(value); @@ -751,6 +758,10 @@ void read_debuging_settings(FILE *fp) // defaults to: false setDebugOption(fp, "DEBUG_HELPER", DEBUG_HELPER); + // DEBUG_EXTRA + // defaults to: false + setDebugOption(fp, "DEBUG_EXTRA", DEBUG_EXTRA); + if(config.debug) { logg("*****************************"); @@ -776,6 +787,7 @@ void read_debuging_settings(FILE *fp) logg("* DEBUG_ALIASCLIENTS %s *", (config.debug & DEBUG_ALIASCLIENTS)? "YES":"NO "); logg("* DEBUG_EVENTS %s *", (config.debug & DEBUG_EVENTS)? "YES":"NO "); logg("* DEBUG_HELPER %s *", (config.debug & DEBUG_HELPER)? "YES":"NO "); + logg("* DEBUG_EXTRA %s *", (config.debug & DEBUG_EXTRA)? "YES":"NO "); logg("*****************************"); } diff --git a/src/enums.h b/src/enums.h index 35caf92c..ade2d55f 100644 --- a/src/enums.h +++ b/src/enums.h @@ -139,6 +139,7 @@ enum debug_flags { DEBUG_ALIASCLIENTS = (1 << 18), /* 00000000 00000100 00000000 00000000 */ DEBUG_EVENTS = (1 << 19), /* 00000000 00001000 00000000 00000000 */ DEBUG_HELPER = (1 << 20), /* 00000000 00010000 00000000 00000000 */ + DEBUG_EXTRA = (1 << 21), /* 00000000 00100000 00000000 00000000 */ } __attribute__ ((packed)); enum events { From 0a07385f88426365ee86243c5a011618ee2b8916 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 18 Dec 2020 08:46:40 +0100 Subject: [PATCH 38/51] Update src/resolve.c Co-authored-by: Adam Warner --- src/resolve.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resolve.c b/src/resolve.c index 7b49048b..7262f677 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -386,7 +386,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) // Only try to resolve host names of clients which were recently active if we are re-resolving // Limit for a "recently active" client is two hours ago - if(!force_refreshing && onlynew == false && client->lastQuery < now - 2*60*60) + if(!force_refreshing && !onlynew && client->lastQuery < now - 2*60*60) { if(config.debug & DEBUG_RESOLVER) { From 23038dad375917422185e47d77efc082ef2fe2cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 18 Dec 2020 17:35:57 +0100 Subject: [PATCH 39/51] num_regex is not in counters any more Signed-off-by: DL6ER --- src/regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/regex.c b/src/regex.c index 28932367..b5b22fee 100644 --- a/src/regex.c +++ b/src/regex.c @@ -529,7 +529,7 @@ static void read_regex_table(const enum regex_type regexid) if(config.debug & DEBUG_DATABASE) { logg("Read %i %s regex entries", - counters->num_regex[regexid], + num_regex[regexid], regextype[regexid]); } } From a90c52a9678d3e6a9c9dcb0a97e0dabb5db1d40c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 18 Dec 2020 20:16:04 +0100 Subject: [PATCH 40/51] Enable extra logging only when DEBUG_EXTRA is set Signed-off-by: DL6ER --- src/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index ef41b6fa..899de723 100644 --- a/src/config.c +++ b/src/config.c @@ -506,7 +506,8 @@ static char *parse_FTLconf(FILE *fp, const char * key) // Go to beginning of file fseek(fp, 0L, SEEK_SET); - logg("initial: conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); + if(config.debug & DEBUG_EXTRA) + logg("initial: conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); errno = 0; while(getline(&conflinebuffer, &size, fp) != -1) From 51cfef35abbebe8c7a005949d9cb17a4e24a1a24 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 15 Dec 2020 22:10:14 +1100 Subject: [PATCH 41/51] docs: fix simple typo, timestemp -> timestamp There is a small typo in src/overTime.c. Should read `timestamp` rather than `timestemp`. Signed-off-by: tgates --- src/overTime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/overTime.c b/src/overTime.c index b0315a29..fab07b8e 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -125,7 +125,7 @@ unsigned int getOverTimeID(time_t timestamp) void moveOverTimeMemory(const time_t mintime) { const time_t oldestOverTimeIS = overTime[0].timestamp; - // Shift SHOULD timestemp into the future by the amount GC is running earlier + // Shift SHOULD timestamp into the future by the amount GC is running earlier time_t oldestOverTimeSHOULD = mintime; // Center in interval From fd788585a1e3a51cc8c2834954a0d1f6e5ea6fe7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 10:12:50 +0100 Subject: [PATCH 42/51] Add interrupt-safe fallocate() routine, due to the special nature of the fallocate() macro, we hav to use a modified name fTLallocate() to implement this function Signed-off-by: DL6ER --- src/FTL.h | 1 + src/shmem.c | 10 +++++----- src/syscalls/CMakeLists.txt | 1 + src/syscalls/ftlallocate.c | 37 +++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 2 ++ 5 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 src/syscalls/ftlallocate.c diff --git a/src/FTL.h b/src/FTL.h index 05f4f34d..4b659f85 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -141,6 +141,7 @@ #define select(nfds, readfds, writefds, exceptfds, timeout) FTLselect(nfds, readfds, writefds, exceptfds, timeout, __FILE__, __FUNCTION__, __LINE__) #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) #define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__, __FUNCTION__, __LINE__) +#define ftlallocate(fd, offset, len) FTLfallocate(fd, offset, len, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/shmem.c b/src/shmem.c index bd922480..312ccac9 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -507,10 +507,10 @@ SharedMemory create_shm(const char *name, const size_t size, bool create_new) } // Allocate shared memory object to specified size - // Using fallocate() will ensure that there's actually space for + // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = fallocate(fd, 0, 0U, size); + const int ret = ftlallocate(fd, 0U, size); if(ret != 0) { logg("FATAL: create_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", @@ -631,10 +631,10 @@ bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const size_t si } // Allocate shared memory object to specified size - // Using fallocate() will ensure that there's actually space for + // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = fallocate(fd, 0, 0U, size); + const int ret = ftlallocate(fd, 0U, size); if(ret != 0) { logg("FATAL: realloc_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", @@ -643,7 +643,7 @@ bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const size_t si } // Close shared memory object file descriptor as it is no longer - // needed after having called fallocate() + // needed after having called f[tl]allocate() close(fd); // Update shm counters to indicate that at least one shared memory object changed diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index e1b971e7..8582ac51 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -12,6 +12,7 @@ set(sources accept.c asprintf.c calloc.c + ftlallocate.c fopen.c fprintf.c free.c diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c new file mode 100644 index 00000000..5095d296 --- /dev/null +++ b/src/syscalls/ftlallocate.c @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for fallocate +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" +#include + +// off_t is automatically set as off64_t when this is a 64bit system +int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *file, const char *func, const int line) +{ + int ret = 0; + do + { + // Reset errno before trying to write + errno = 0; + ret = posix_fallocate(fd, offset, len); + } + // Try again if the last posix_fallocate() call failed due to an + // interruption by an incoming signal + while(ret < 0 && errno == EINTR); + + // Final error checking (may have faild for some other reason then an + // EINTR = interrupted system call) + if(ret < 0) + logg("WARN: Could not fallocate() in %s() (%s:%i): %s", + func, file, line, strerror(errno)); + + return ret; +} \ No newline at end of file diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 230068b6..1b2787cd 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -15,6 +15,8 @@ char *FTLstrdup(const char *src, const char *file, const char *func, const int l void *FTLcalloc(size_t n, size_t size, const char *file, const char *func, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, const int line) __attribute__((alloc_size(2))); void FTLfree(void *ptr, const char*file, const char *func, const int line); +int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *file, const char *func, const int line); + // Interrupt-safe printing routines // printf() is derived from fprintf(stdout, ...) From 9627ca6909785b93edfd3703ffe284c5a4efb90f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 10:37:39 +0100 Subject: [PATCH 43/51] Prevent possible deadlock if log is not writable (e.g., permission denied) Signed-off-by: DL6ER --- src/log.c | 30 ++++++++++++++++-------------- src/log.h | 2 +- src/main.c | 6 ------ 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/log.c b/src/log.c index 28885f41..51c171d3 100644 --- a/src/log.c +++ b/src/log.c @@ -27,6 +27,7 @@ static pthread_mutex_t lock; static FILE *logfile = NULL; +static bool FTL_log_ready = false; static bool print_log = true, print_stdout = true; void log_ctrl(bool plog, bool pstdout) @@ -41,27 +42,25 @@ static void close_FTL_log(void) fclose(logfile); } -void init_FTL_log(void) +void open_FTL_log(const bool init) { - if (pthread_mutex_init(&lock, NULL) != 0) + if(init) { - printf("FATAL: Log mutex init failed\n"); - // Return failure - exit(EXIT_FAILURE); - } -} + // Initialize logging mutex + if (pthread_mutex_init(&lock, NULL) != 0) + { + printf("FATAL: Log mutex init failed\n"); + // Return failure + exit(EXIT_FAILURE); + } -void open_FTL_log(const bool test) -{ - if(test) - { // Obtain log file location getLogFilePath(); } // Open the log file in append/create mode logfile = fopen(FTLfiles.log, "a+"); - if((logfile == NULL) && test){ + if((logfile == NULL) && init){ syslog(LOG_ERR, "Opening of FTL\'s log file failed!"); printf("FATAL: Opening of FTL log (%s) failed!\n",FTLfiles.log); printf(" Make sure it exists and is writeable by user %s\n", username); @@ -69,7 +68,10 @@ void open_FTL_log(const bool test) exit(EXIT_FAILURE); } - if(test) + // Set log as ready (we were able to open it) + FTL_log_ready = true; + + if(init) { close_FTL_log(); } @@ -140,7 +142,7 @@ void _FTL_log(const bool newline, const char *format, ...) printf("\n"); } - if(print_log) + if(print_log && FTL_log_ready) { // Open log file open_FTL_log(false); diff --git a/src/log.h b/src/log.h index c6cd0516..eb99cd52 100644 --- a/src/log.h +++ b/src/log.h @@ -14,7 +14,7 @@ #include void init_FTL_log(void); -void open_FTL_log(const bool test); +void open_FTL_log(const bool init); void log_counter_info(void); void format_memory_size(char * const prefix, unsigned long long int bytes, double * const formated); diff --git a/src/main.c b/src/main.c index 8b6864a1..aa68c320 100644 --- a/src/main.c +++ b/src/main.c @@ -39,12 +39,6 @@ int main (int argc, char* argv[]) // it if needed username = getUserName(); - // This only prepares the log file lock, we - // do not want to log already here (parsing - // args may bring up something we want to do - // separated from the log in foreground) - init_FTL_log(); - // Parse arguments // We run this also for no direct arguments // to have arg{c,v}_dnsmasq initialized From f88b14985e3a9ffeef32b5e1c503315946a506f1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 10:40:23 +0100 Subject: [PATCH 44/51] Don't fail when trying to free(NULL) Signed-off-by: DL6ER --- src/syscalls/fopen.c | 2 +- src/syscalls/free.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index ac06bac4..96f4bb07 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -29,7 +29,7 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // Final error checking (may have faild for some other reason then an // EINTR = interrupted system call) if(file_ptr == NULL) - logg("WARN: Could not fopen(%s, %s) in %s() (%s:%i): %s", + logg("WARN: Could not fopen(\"%s\", \"%s\") in %s() (%s:%i): %s", pathname, mode, func, file, line, strerror(errno)); return file_ptr; diff --git a/src/syscalls/free.c b/src/syscalls/free.c index da08daf1..d3a48ad9 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -20,7 +20,10 @@ void FTLfree(void *ptr, const char *file, const char *func, const int line) // realloc(). Otherwise, or if free(ptr) has already been called before, // undefined behavior occurs. If ptr is NULL, no operation is performed. if(ptr == NULL) - logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", func, file, line); + { + logg("WARN: Trying to free NULL pointer in %s() (%s:%i)", func, file, line); + return; + } free(ptr); } From b52eb1f2a5bd27e7f9589ee511560d26e35c7b9e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 10:44:34 +0100 Subject: [PATCH 45/51] Fix Unix socket error handling Signed-off-by: DL6ER --- src/api/socket.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/api/socket.c b/src/api/socket.c index 2201dbbf..eba21f26 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -512,8 +512,11 @@ void *socket_listening_thread(void *args) // Return early to avoid CPU spinning if Unix socket is not available sock_avail = bind_to_unix_socket(&socketfd); - if(sock_avail) + if(!sock_avail) + { + logg("INFO: Unix socket will not be available"); return NULL; + } // Listen as long as FTL is not killed while(!killed) From 0837b8d2df3e2a4e82a50c9350a8a7c16198f2fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 10:48:26 +0100 Subject: [PATCH 46/51] Do not print user change information if there is no user change Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0a0e96f1..0801bd32 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1774,7 +1774,9 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw) // option states to run as a different user/group (e.g. "nobody") if(getuid() == 0) { - if(ent_pw != NULL) + // Only print this and change ownership of shmem objects when + // we're actually dropping root (user/group my be set to root) + if(ent_pw != NULL && ent_pw->pw_uid != 0) { logg("INFO: FTL is going to drop from root to user %s (UID %d)", ent_pw->pw_name, (int)ent_pw->pw_uid); From 02196cb716a01508031b2c944eadec0038f2e94f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 12:12:28 +0100 Subject: [PATCH 47/51] Reply with configured BLOCKINGMODE to blocked CNAME requests Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 62afda22..4c2c5414 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -692,7 +692,6 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server a.log.rcode = rcode; FTL_upstream_error(rcode, daemon->log_display_id); log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL); - FTL_upstream_error(rcode, daemon->log_display_id); return resize_packet(header, n, pheader, plen); } @@ -735,8 +734,12 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server int ret = extract_addresses(header, n, daemon->namebuff, now, sets, is_sign, check_rebind, no_cache, cache_secure, &doctored); if (ret == 2) { - munged = 1; cache_secure = 0; + union all_addr *addrp = NULL; + // Pretend this is an A type reply + unsigned int flags = F_IPV4; + FTL_get_blocking_metadata(&addrp, &flags); + n = setup_reply(header, n, addrp, flags, daemon->local_ttl); } else if(ret) { @@ -1627,16 +1630,13 @@ void receive_query(struct listener *listen, time_t now) /************ Pi-hole modification ************/ if(piholeblocked) { - size_t plen = n; union all_addr *addrp = NULL; // DNS resource record type for AAAA is 28 (decimal) following RFC 3596, section 2.1 unsigned int flags = (type == 28u) ? F_IPV6 : F_IPV4; FTL_get_blocking_metadata(&addrp, &flags); log_query(flags, daemon->namebuff, addrp, (char*)blockingreason); - plen = setup_reply(header, n, addrp, flags, daemon->local_ttl); - if (find_pseudoheader(header, plen, NULL, NULL, NULL, NULL)) - plen = add_pseudoheader(header, plen, ((unsigned char *) header) + PACKETSZ, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); - send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, plen, (union mysockaddr*)&source_addr, &dst_addr, if_index); + n = setup_reply(header, n, addrp, flags, daemon->local_ttl); + send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, n, (union mysockaddr*)&source_addr, &dst_addr, if_index); return; } /**********************************************/ @@ -2027,8 +2027,6 @@ unsigned char *tcp_request(int confd, time_t now, FTL_get_blocking_metadata(&addrp, &flags); log_query(flags, daemon->namebuff, addrp, (char*)blockingreason); m = setup_reply(header, size, addrp, flags, daemon->local_ttl); - if (have_pseudoheader) - m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); } else { From a66315d292ab9aa7550cd6e39bfd26232b7c3efe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 12:28:59 +0100 Subject: [PATCH 48/51] Revert "Improve compatibility with old (ancient) kernels" --- src/shmem.c | 64 ++++++++--------------------------------------------- 1 file changed, 9 insertions(+), 55 deletions(-) diff --git a/src/shmem.c b/src/shmem.c index dfd6a1ff..4d0b8362 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -513,35 +513,12 @@ SharedMemory create_shm(const char *name, const size_t size, bool create_new) // Using fallocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - int ret = fallocate(fd, 0, 0U, size); - const int fallocate_errno = errno; + const int ret = fallocate(fd, 0, 0U, size); if(ret != 0) { - // Try again with ftruncate() if fallocate() failed with - // "Operation not supported" - if(errno == ENOTSUP) - { - ret = ftruncate(fd, size); - - // Report info that space has only been reserved, not - // really allocated - if(ret == 0) - { - logg("INFO: create_shm(): Failed to allocate %zu bytes for \"%s\" (%s, %i), "\ - "space has only been reserved.", - size, sharedMemory.name, strerror(errno), errno); - } - } - if(ret != 0) - { - logg("FATAL: create_shm(): Failed to resize \"%s\" (%i) to %zu:", - sharedMemory.name, fd, size); - logg(" fallocate returned: %s (%i)", - strerror(fallocate_errno), fallocate_errno); - logg(" ftruncate returned: %s (%i)", - strerror(errno), errno); - exit(EXIT_FAILURE); - } + logg("FATAL: create_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", + sharedMemory.name, fd, size, strerror(errno), ret); + exit(EXIT_FAILURE); } // Create shared memory mapping @@ -660,35 +637,12 @@ bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const size_t si // Using fallocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - int ret = fallocate(fd, 0, 0U, size); - const int fallocate_errno = errno; - if(ret == -1) + const int ret = fallocate(fd, 0, 0U, size); + if(ret != 0) { - // Try again with ftruncate() if fallocate() failed with - // "Operation not supported" - if(errno == ENOTSUP) - { - ret = ftruncate(fd, size); - - // Report info that space has only been reserved, not - // really allocated - if(ret == 0) - { - logg("INFO: realloc_shm(): Failed to allocate %zu bytes for \"%s\" (%s), "\ - "space has only been reserved.", - size, sharedMemory->name, strerror(errno)); - } - } - if(ret == -1) - { - logg("FATAL: realloc_shm(): Failed to resize \"%s\" (%i) to %zu:", - sharedMemory->name, fd, size); - logg(" fallocate returned: %s (%i)", - strerror(fallocate_errno), fallocate_errno); - logg(" ftruncate returned: %s (%i)", - strerror(errno), errno); - exit(EXIT_FAILURE); - } + logg("FATAL: realloc_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", + sharedMemory->name, fd, size, strerror(errno), ret); + exit(EXIT_FAILURE); } // Close shared memory object file descriptor as it is no longer From 616bebde88ec80cdc5196e74869ed3fe2bb4f33d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Dec 2020 22:10:06 +0100 Subject: [PATCH 49/51] Analyze original question and use it to decide whether we mock an A or AAAA reply when blocking Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 5 +++-- src/dnsmasq_interface.c | 50 +++++++++++++++++++++++++++++++++++++++++ src/dnsmasq_interface.h | 2 ++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 4c2c5414..782963af 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -736,8 +736,9 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server { cache_secure = 0; union all_addr *addrp = NULL; - // Pretend this is an A type reply - unsigned int flags = F_IPV4; + // Extract IPv4/IPv6 information from the original question in the DNS + // header + unsigned int flags = FTL_extract_question_flags(header, n); FTL_get_blocking_metadata(&addrp, &flags); n = setup_reply(header, n, addrp, flags, daemon->local_ttl); } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c97b68d2..3e9e36f3 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1956,6 +1956,56 @@ static void prepare_blocking_metadata(void) clearSetupVarsArray(); } +unsigned int FTL_extract_question_flags(struct dns_header *header, const size_t qlen) +{ + // Create working pointer + unsigned char *p = (unsigned char *)(header+1); + uint16_t qtype, qclass; + + // Go through the questions + for (uint16_t i = ntohs(header->qdcount); i != 0; i--) + { + // Prime dnsmasq flags + int flags = RCODE(header) == NXDOMAIN ? F_NXDOMAIN : 0; + + // Extract name from this question + char name[MAXDNAME]; + if (!extract_name(header, qlen, &p, name, 1, 4)) + break; // bad packet, go to fallback solution + + // Extract query type + GETSHORT(qtype, p); + GETSHORT(qclass, p); + + // Only further analyze IN questions here (not CHAOS, etc.) + if (qclass != C_IN) + continue; + + // Very simple decision: If the question is AAAA, the reply + // should be IPv6. We use IPv4 in all other cases + if(qtype == T_AAAA) + flags |= F_IPV6; + else + flags |= F_IPV4; + + // Debug logging if enabled + if(config.debug & DEBUG_QUERIES) + { + char *qtype_str = querystr(NULL, qtype); + logg("CNAME header: Question was %s %s", qtype_str, name); + } + + return flags; + } + + // Fall back to IPv4 (type A) when for the unlikely event that we cannot + // find any questions in this header + if(config.debug & DEBUG_QUERIES) + logg("CNAME header: No valid IN question found in header"); + + return F_IPV4; +} + // Called when a (forked) TCP worker is terminated by receiving SIGALRM // We close the dedicated database connection this client had opened // to avoid dangling database locks diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 572395fc..e80de13c 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -52,6 +52,8 @@ void _FTL_get_blocking_metadata(union all_addr **addrp, unsigned int *flags, con #define FTL_CNAME(domain, cpp, id) _FTL_CNAME(domain, cpp, id, __FILE__, __LINE__) bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line); +unsigned int FTL_extract_question_flags(struct dns_header *header, const size_t qlen); + void FTL_dnsmasq_reload(void); void FTL_fork_and_bind_sockets(struct passwd *ent_pw); void FTL_TCP_worker_created(const int confd, const char *iface_name); From 5cca201338304407d2713c0611bd661ff7448c03 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 23 Sep 2020 12:00:12 +0200 Subject: [PATCH 50/51] Don't iterate over all clients every minute trying to find new ones but only do this when the RESOLVE_NEW_HOSTNAMES event is set Signed-off-by: DL6ER --- src/FTL.h | 4 ---- src/datastructure.c | 4 ++++ src/enums.h | 1 + src/events.c | 2 ++ src/resolve.c | 45 +++++++++++++++++++++++++++++++++------------ 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 4b659f85..879e1e9b 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -79,10 +79,6 @@ // can be 24 hours + 59 minutes #define OVERTIME_SLOTS ((MAXLOGAGE+1)*3600/OVERTIME_INTERVAL) -// Interval for resolving NEW client and upstream server host names [seconds] -// Default: 60 (once every minute) -#define RESOLVE_INTERVAL 60 - // Interval for re-resolving ALL known host names [seconds] // Default: 3600 (once every hour) #define RERESOLVE_INTERVAL 3600 diff --git a/src/datastructure.c b/src/datastructure.c index 03ff9efd..6c92d864 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -26,6 +26,8 @@ #include "database/common.h" // config struct #include "config.h" +// set_event(RESOLVE_NEW_HOSTNAMES) +#include "events.h" const char *querytypes[TYPE_MAX] = {"UNKNOWN", "A", "AAAA", "ANY", "SRV", "SOA", "PTR", "TXT", "NAPTR", "MX", "DS", "RRSIG", "DNSKEY", "NS", "OTHER"}; @@ -120,6 +122,7 @@ int findUpstreamID(const char * upstreamString, const in_port_t port, const bool // to be done separately to be non-blocking upstream->new = true; upstream->namepos = 0; // 0 -> string with length zero + set_event(RESOLVE_NEW_HOSTNAMES); // This is a new upstream server upstream->lastQuery = time(NULL); // Store port @@ -243,6 +246,7 @@ int findClientID(const char *clientIP, const bool count, const bool aliasclient) // to be done separately to be non-blocking client->new = true; client->namepos = 0; + set_event(RESOLVE_NEW_HOSTNAMES); // No query seen so far client->lastQuery = 0; client->numQueriesARP = client->count; diff --git a/src/enums.h b/src/enums.h index 447bf50a..15092cb5 100644 --- a/src/enums.h +++ b/src/enums.h @@ -144,6 +144,7 @@ enum debug_flags { enum events { RELOAD_GRAVITY, RELOAD_PRIVACY_LEVEL, + RESOLVE_NEW_HOSTNAMES, RERESOLVE_HOSTNAMES, RERESOLVE_HOSTNAMES_FORCE, REIMPORT_ALIASCLIENTS, diff --git a/src/events.c b/src/events.c index 06280707..bd599c64 100644 --- a/src/events.c +++ b/src/events.c @@ -98,6 +98,8 @@ static const char *eventtext(const enum events event) return "REIMPORT_ALIASCLIENTS"; case PARSE_NEIGHBOR_CACHE: return "PARSE_NEIGHBOR_CACHE"; + case RESOLVE_NEW_HOSTNAMES: + return "RESOLVE_NEW_HOSTNAMES"; case EVENTS_MAX: // fall through default: return "UNKNOWN"; diff --git a/src/resolve.c b/src/resolve.c index e6c9f4c3..d06b0108 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -441,6 +441,12 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) getstr(ippos), getstr(oldnamepos), reason); } skipped++; + if(config.debug & DEBUG_RESOLVER) + { + lock_shm(); + logg("Client %s -> \"%s\" already known", getstr(ippos), getstr(oldnamepos)); + unlock_shm(); + } continue; } @@ -465,6 +471,10 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) client->namepos = newnamepos; // Mark entry as not new client->new = false; + + if(config.debug & DEBUG_RESOLVER) + logg("Client %s -> \"%s\" is new", getstr(ippos), getstr(newnamepos)); + unlock_shm(); } @@ -522,6 +532,12 @@ static void resolveUpstreams(const bool onlynew) if(onlynew && !newflag) { skipped++; + if(config.debug & DEBUG_RESOLVER) + { + lock_shm(); + logg("Upstream %s -> \"%s\" already known", getstr(ippos), getstr(oldnamepos)); + unlock_shm(); + } continue; } @@ -546,6 +562,10 @@ static void resolveUpstreams(const bool onlynew) upstream->namepos = newnamepos; // Mark entry as not new upstream->new = false; + + if(config.debug & DEBUG_RESOLVER) + logg("Upstream %s -> \"%s\" is new", getstr(ippos), getstr(newnamepos)); + unlock_shm(); } @@ -566,16 +586,17 @@ void *DNSclient_thread(void *val) while(!killed) { - // Run every minute to resolve only new clients and upstream servers - if(resolver_ready && (time(NULL) % RESOLVE_INTERVAL == 0)) + // Run whenever necessary to resolve only new clients and + // upstream servers + if(resolver_ready && get_and_clear_event(RESOLVE_NEW_HOSTNAMES)) { - // Try to resolve new client host names (onlynew=true) + // Try to resolve new client host names + // (onlynew=true) // We're not forcing refreshing here resolveClients(true, false); - // Try to resolve new upstream destination host names (onlynew=true) + // Try to resolve new upstream destination host names + // (onlynew=true) resolveUpstreams(true); - // Prevent immediate re-run of this routine - sleepms(500); } // Run every hour to update possibly changed client host names @@ -594,16 +615,16 @@ void *DNSclient_thread(void *val) // Process resolver related event queue elements if(get_and_clear_event(RERESOLVE_HOSTNAMES)) { - // Try to resolve all client host names (onlynew=false) + // Try to resolve all client host names + // (onlynew=false) resolveClients(false, force_refreshing); - // Try to resolve all upstream destination host names (onlynew=false) + // Try to resolve all upstream destination host names + // (onlynew=false) resolveUpstreams(false); - // Prevent immediate re-run of this routine - sleepms(500); } - // Idle for 0.1 sec before checking again the time criteria - sleepms(100); + // Idle for 1 sec before checking again the time criteria + sleepms(1000); } return NULL; From e4483bd87bacc7d5761b546047433da635516d1c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 23 Dec 2020 18:28:15 +0100 Subject: [PATCH 51/51] Escape spaces by ~ Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- src/shmem.c | 79 ++++++++++++++++++++++++++++++++++++++--- src/shmem.h | 10 ++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c4bdc7ab..89368368 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1022,7 +1022,7 @@ void _FTL_reply(const unsigned int flags, const char *name, const union all_addr } // Check if this domain matches exactly - const bool isExactMatch = (name != NULL && strcasecmp(getstr(domain->domainpos), name) == 0); + const bool isExactMatch = strcmp_escaped(name, getstr(domain->domainpos)); if((flags & F_CONFIG) && isExactMatch && !query->complete) { diff --git a/src/shmem.c b/src/shmem.c index 23266a65..5d1b83e7 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -151,22 +151,81 @@ void chown_all_shmem(struct passwd *ent_pw) chown_shmem(&shm_per_client_regex, ent_pw); } -size_t addstr(const char *str) +// A function that duplicates a string and replaces all characters "s" by "r" +static char *__attribute__ ((malloc)) str_replace(const char *input, + const char s, + const char r, + unsigned int *N) { - if(str == NULL) + // Duplicate string + char *copy = strdup(input); + if(copy == NULL) + return NULL; + + // Woring pointer + char *ix = copy; + // Loop over string until there are no further "s" chars in the string + while((ix = strchr(ix, s)) != NULL) + { + *ix++ = r; + (*N)++; + } + + return copy; +} + +char *str_escape(const char *input, unsigned int *N) +{ + // If no escaping is done, this routine returns the original pointer + // and N stays 0 + *N = 0; + char *out = (char *)input; + if(strchr(input, ' ') != NULL) + { + // Replace any spaces by ~ if we find them in the domain name + // This is necessary as our telnet API uses space delimiters + out = str_replace(out, ' ', '~', N); + } + return out; +} + +bool strcmp_escaped(const char *a, const char *b) +{ + if(a == NULL || b == NULL) + return false; + + unsigned int Na, Nb; + char *aa = str_escape(a, &Na); + char *bb = str_escape(b, &Nb); + + const char result = strcasecmp(aa, bb) == 0; + + if(Na > 0) + free(aa); + if(Nb > 0) + free(bb); + + return result; +} + + +size_t addstr(const char *input) +{ + if(input == NULL) { logg("WARN: Called addstr() with NULL pointer"); return 0; } // Get string length, add terminating character - size_t len = strlen(str) + 1; + size_t len = strlen(input) + 1; // If this is an empty string (only the terminating character is present), // use the shared memory string at position zero instead of creating a new // entry here. We also ensure that the given string is not too long to // prevent possible memory corruption caused by strncpy() further down - if(len == 1) { + if(len == 1) + { return 0; } else if(len > (size_t)(pagesize-1)) @@ -175,6 +234,12 @@ size_t addstr(const char *str) len = pagesize; } + unsigned int N = 0; + char *str = str_escape(input, &N); + + if(N > 0) + logg("INFO: FTL escaped %ui characters in \"%s\"", N, str); + // Debugging output if(config.debug & DEBUG_SHMEM) logg("Adding \"%s\" (len %zu) to buffer. next_str_pos is %u", str, len, shmSettings->next_str_pos); @@ -182,7 +247,11 @@ size_t addstr(const char *str) // Reserve additional memory if necessary if(shmSettings->next_str_pos + len > shm_strings.size && !realloc_shm(&shm_strings, shm_strings.size + pagesize, sizeof(char), true)) + { + if(N > 0) + free(str); return 0; + } // Store new string buffer size in corresponding counters entry // for re-using when we need to re-map shared memory objects @@ -190,6 +259,8 @@ size_t addstr(const char *str) // Copy the C string pointed by str into the shared string buffer strncpy(&((char*)shm_strings.ptr)[shmSettings->next_str_pos], str, len); + if(N > 0) + free(str); // Increment string length counter shmSettings->next_str_pos += len; diff --git a/src/shmem.h b/src/shmem.h index 79cd0dcb..7d7eb9fe 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -95,6 +95,16 @@ size_t addstr(const char *str); const char *getstr(const size_t pos); void *enlarge_shmem_struct(const char type); +/** + * Escapes a string by replacing special characters, such as spaces + */ +char *str_escape(const char *input, unsigned int *N); + +/** + * Compare two strings. Escape them if needed + */ +bool strcmp_escaped(const char *a, const char *b); + /** * Create a new overTime client shared memory block. * This also updates `overTimeClientData`.