From 36d14998043f7c27eabaa0d6a0aec4e1dda6cfc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 23 Dec 2018 12:40:39 +0100 Subject: [PATCH 01/47] Add network table. This will update the database to version 3. Furthermore, we make some database routines globally (add prototypes to routines.h) and mark some internal database variables as static. This commit also improves on the speed of the database routines as the main loop is changed to run from the last saved query to the most recent one instead of looping over all queries in memory. This ID is corrected when queries are removed in gc.c Signed-off-by: DL6ER --- FTL.h | 6 +++++- Makefile | 2 +- database.c | 47 +++++++++++++++++++++++++++++++---------------- gc.c | 2 ++ networktable.c | 31 +++++++++++++++++++++++++++++++ routines.h | 7 +++++++ 6 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 networktable.c diff --git a/FTL.h b/FTL.h index 402f48ef..876910f7 100644 --- a/FTL.h +++ b/FTL.h @@ -80,6 +80,11 @@ enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN }; +// Database table "ftl" +enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP }; +// Database table "counters" +enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES }; + // Privacy mode constants #define HIDDEN_DOMAIN "hidden" #define HIDDEN_CLIENT "0.0.0.0" @@ -250,7 +255,6 @@ extern long int lastdbindex; extern bool travis; extern bool DBdeleteoldqueries; extern bool rereadgravity; -extern long int lastDBimportedtimestamp; extern bool ipv4telnet, ipv6telnet; extern bool istelnet[MAXCONNS]; diff --git a/Makefile b/Makefile index 6e27a0e7..c2f4b1dc 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h -FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o +FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o networktable.o DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o diff --git a/database.c b/database.c index 7dcb6cba..bc930619 100644 --- a/database.c +++ b/database.c @@ -11,21 +11,14 @@ #include "FTL.h" #include "shmem.h" -sqlite3 *db; +static sqlite3 *db; bool database = false; bool DBdeleteoldqueries = false; long int lastdbindex = 0; -long int lastDBimportedtimestamp = 0; -pthread_mutex_t dblock; - -// TABLE ftl -enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP }; -// TABLE counters -enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES }; +static pthread_mutex_t dblock; bool db_set_counter(unsigned int ID, int value); -bool db_set_FTL_property(unsigned int ID, int value); int db_get_FTL_property(unsigned int ID); void check_database(int rc) @@ -155,8 +148,8 @@ bool db_create(void) ret = dbquery("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"); if(!ret){ dbclose(); return false; } - // DB version 2 - ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,2);", DB_VERSION); + // Set DB version 1 + ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION); if(!ret){ dbclose(); return false; } // Most recent timestamp initialized to 00:00 1 Jan 1970 @@ -164,9 +157,15 @@ bool db_create(void) if(!ret){ dbclose(); return false; } // Create counter table + // Will update DB version to 2 if(!create_counter_table()) return false; + // Create network table + // Will update DB version to 3 + if(!create_network_table()) + return false; + return true; } @@ -203,16 +202,33 @@ void db_init(void) database = false; return; } - else if(dbversion < 2) + // Update to version 2 if still version 1 + if(dbversion < 2) { - // Database is still in version 1 - // Update to version 2 and create counters table + // Update to version 2: Create counters table + logg("Updating long-term database to version 2"); if (!create_counter_table()) { logg("Counter table not initialized, database not available"); database = false; return; } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); + } + // Update to version 2 if still version 1 + if(dbversion < 3) + { + // Update to version 3: Create network table + logg("Updating long-term database to version 3"); + if (!create_network_table()) + { + logg("Network table not initialized, database not available"); + database = false; + return; + } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); } // Close database to prevent having it opened all time @@ -406,7 +422,7 @@ void save_to_DB(void) int total = 0, blocked = 0; time_t currenttimestamp = time(NULL); time_t newlasttimestamp = 0; - for(i = 0; i < counters->queries; i++) + for(i = lastdbindex; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); if(queries[i].db != 0) @@ -741,7 +757,6 @@ void read_data_from_DB(void) queries[queryIndex].complete = true; // Mark as all information is avaiable queries[queryIndex].response = 0; queries[queryIndex].AD = false; - lastDBimportedtimestamp = queryTimeStamp; // Handle type counters if(type >= TYPE_A && type < TYPE_MAX) diff --git a/gc.c b/gc.c index bc93aa58..813b96aa 100644 --- a/gc.c +++ b/gc.c @@ -154,6 +154,8 @@ void *GC_thread(void *val) // Update queries counter counters->queries -= removed; + // Update DB index as total number of queries reduced + lastdbindex -= removed; // Zero out remaining memory (marked as "F" in the above example) memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries)); diff --git a/networktable.c b/networktable.c new file mode 100644 index 00000000..79cb7853 --- /dev/null +++ b/networktable.c @@ -0,0 +1,31 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Network table routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" + +bool create_network_table(void) +{ + bool ret; + // Create FTL table in the database (holds properties like database version, etc.) + ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, \ + ip TEXT NOT NULL, \ + mac TEXT NOT NULL, \ + name TEXT, \ + firstSeen INTEGER NOT NULL, \ + lastSeen INTEGER NOT NULL, \ + PiholeDNS BOOLEAN NOT NULL );"); + if(!ret){ dbclose(); return false; } + + // Update database version to 3 + ret = db_set_FTL_property(DB_VERSION, 3); + if(!ret){ dbclose(); return false; } + + return true; +} diff --git a/routines.h b/routines.h index b4ad082b..c4a3a22d 100644 --- a/routines.h +++ b/routines.h @@ -84,6 +84,10 @@ int get_number_of_queries_in_DB(void); void save_to_DB(void); void read_data_from_DB(void); +bool db_set_FTL_property(unsigned int ID, int value); +bool dbquery(const char *format, ...); +void dbclose(void); + // memory.c void memory_check(int which); char *FTLstrdup(const char *src, const char *file, const char *function, int line); @@ -126,3 +130,6 @@ void newOverTimeClient(); * This also updates `overTimeClientData`. */ void addOverTimeClientSlot(); + +// networktable.c +bool create_network_table(void); From 40fd4d317e8d46ae92ceaaada2b64ee0d55be257 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 24 Dec 2018 13:18:51 +0100 Subject: [PATCH 02/47] Add / update rows in table network depending on what we see in the ARP cache Signed-off-by: DL6ER --- database.c | 46 +++++++++++++++++++++- networktable.c | 103 +++++++++++++++++++++++++++++++++++++++++++++---- request.c | 5 +++ routines.h | 4 +- 4 files changed, 148 insertions(+), 10 deletions(-) diff --git a/database.c b/database.c index bc930619..d0593993 100644 --- a/database.c +++ b/database.c @@ -32,6 +32,7 @@ void check_database(int rc) rc != SQLITE_ROW && rc != SQLITE_BUSY) { + logg("check_database(%i): Disabling database connection due to error", rc); database = false; } } @@ -196,13 +197,14 @@ void db_init(void) // Test DB version and see if we need to upgrade the database file int dbversion = db_get_FTL_property(DB_VERSION); + logg("Database version is %i", dbversion); if(dbversion < 1) { logg("Database version incorrect, database not available"); database = false; return; } - // Update to version 2 if still version 1 + // Update to version 2 if lower if(dbversion < 2) { // Update to version 2: Create counters table @@ -216,7 +218,7 @@ void db_init(void) // Get updated version dbversion = db_get_FTL_property(DB_VERSION); } - // Update to version 2 if still version 1 + // Update to version 3 if lower if(dbversion < 3) { // Update to version 3: Create network table @@ -306,6 +308,46 @@ bool db_update_counters(int total, int blocked) return true; } +int db_query_int(const char* querystr) +{ + // Check if database is enabled + if(!database) + return -2; + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc ){ + logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db)); + dbclose(); + check_database(rc); + return -2; + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = -1; + } + else + { + logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db)); + dbclose(); + check_database(rc); + return -2; + } + + sqlite3_finalize(stmt); + + return result; +} + int number_of_queries_in_DB(void) { sqlite3_stmt* stmt; diff --git a/networktable.c b/networktable.c index 79cb7853..7f94c436 100644 --- a/networktable.c +++ b/networktable.c @@ -9,18 +9,20 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#define ARPCACHE "/proc/net/arp" bool create_network_table(void) { bool ret; // Create FTL table in the database (holds properties like database version, etc.) - ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, \ - ip TEXT NOT NULL, \ - mac TEXT NOT NULL, \ - name TEXT, \ - firstSeen INTEGER NOT NULL, \ - lastSeen INTEGER NOT NULL, \ - PiholeDNS BOOLEAN NOT NULL );"); + ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \ + "ip TEXT NOT NULL, " \ + "hwaddr TEXT NOT NULL, " \ + "interface TEXT NOT NULL, " \ + "name TEXT, " \ + "firstSeen INTEGER NOT NULL, " \ + "lastSeen INTEGER NOT NULL, " \ + "usesPihole BOOLEAN NOT NULL );"); if(!ret){ dbclose(); return false; } // Update database version to 3 @@ -29,3 +31,90 @@ bool create_network_table(void) return true; } + +// Read kernel's ARP cache using procfs +void read_arp_cache(void) +{ + FILE* arpfp = NULL; + // Try to access the kernel's ARP cache + if((arpfp = fopen(ARPCACHE, "r")) == NULL) + { + logg("WARN: Opening of %s failed!", ARPCACHE); + logg(" Message: %s", strerror(errno)); + return; + } + + // Open database file + if(!dbopen()) + { + logg("read_arp_cache() - Failed to open DB"); + return; + } + + // Prepare buffers + char * linebuffer = NULL; + size_t linebuffersize = 0; + char ip[100], mask[100], hwaddr[100], iface[100]; + int type, flags, entries = 0; + time_t now = time(NULL); + + // Read ARP cache line by line + while(getline(&linebuffer, &linebuffersize, arpfp) != -1) + { + int num = sscanf(linebuffer, "%99s 0x%x 0x%x %99s %99s %99s\n", + ip, &type, &flags, hwaddr, mask, iface); + + // Skip header and empty lines + if (num < 4) + continue; + + if (num == 5) + { + /* + * This happens for incomplete ARP entries for which there is + * no hardware address in the line. We don't use these + */ + //num = sscanf(linebuffer, "%s 0x%x 0x%x %99s %99s\n", + // ip, &type, &flags, mask, iface); + //hwaddr[0] = '\0'; + } + + entries++; + if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip); + + // Get ID of this device in our network database. If it cannot be found, then this is a new device + char querystr[256]; + sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr); + int dbID = db_query_int(querystr); + + if(dbID == -2) + { + // SQLite error + break; + } + else if(dbID == -1) + { + // Device not in database, add new entry + dbquery("INSERT INTO network "\ + "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\ + "VALUES "\ + "(\"%s\",\"%s\",\"%s\",%lu,%lu,false);",\ + ip, hwaddr, iface, now, now); + } + else + { + // Device already known, update lastSeen + dbquery("UPDATE network "\ + "SET lastSeen = %lu "\ + "WHERE id = %i;",\ + now, dbID); + } + + } + + // Close file handle + fclose(arpfp); + + // Close database connection + dbclose(); +} diff --git a/request.c b/request.c index 82a791a2..1549e0e5 100644 --- a/request.c +++ b/request.c @@ -132,6 +132,11 @@ void process_request(char *client_message, int *sock) free_regex(); read_regex_from_file(); } + else if(command(client_message, ">arp")) + { + processed = true; + read_arp_cache(); + } // Test only at the end if we want to quit or kill // so things can be processed before diff --git a/routines.h b/routines.h index c4a3a22d..c3485379 100644 --- a/routines.h +++ b/routines.h @@ -83,10 +83,11 @@ void *DB_thread(void *val); int get_number_of_queries_in_DB(void); void save_to_DB(void); void read_data_from_DB(void); - bool db_set_FTL_property(unsigned int ID, int value); bool dbquery(const char *format, ...); +bool dbopen(void); void dbclose(void); +int db_query_int(const char*); // memory.c void memory_check(int which); @@ -133,3 +134,4 @@ void addOverTimeClientSlot(); // networktable.c bool create_network_table(void); +void read_arp_cache(void); From c7bdf9bd60ef6e2484cae3678b908d92a1bb4134 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Dec 2018 13:44:39 +0100 Subject: [PATCH 03/47] Update if device uses Pi-hole and store host name if available Print executed SQL statements when in debug mode Signed-off-by: DL6ER --- database.c | 4 +++- datastructure.c | 6 +++++- dnsmasq_interface.c | 2 +- networktable.c | 50 +++++++++++++++++++++++++++++++-------------- routines.h | 2 +- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/database.c b/database.c index d0593993..fe1dc64a 100644 --- a/database.c +++ b/database.c @@ -88,6 +88,8 @@ bool dbquery(const char *format, ...) return false; } + if(debug) logg("dbquery: %s", query); + int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg); if( rc != SQLITE_OK ){ @@ -775,7 +777,7 @@ void read_data_from_DB(void) int overTimeTimeStamp = queryTimeStamp - (queryTimeStamp % 600) + 300; int timeidx = findOverTimeID(overTimeTimeStamp); int domainID = findDomainID(domain); - int clientID = findClientID(client); + int clientID = findClientID(client, true); // Ensure we have enough space in the queries struct memory_check(QUERIES); diff --git a/datastructure.c b/datastructure.c index 18cb871a..610d4cd9 100644 --- a/datastructure.c +++ b/datastructure.c @@ -176,7 +176,7 @@ int findDomainID(const char *domain) return domainID; } -int findClientID(const char *client) +int findClientID(const char *client, bool addNew) { int i; // Compare content of client against known client IP addresses @@ -196,6 +196,10 @@ int findClientID(const char *client) } } + // Return -1 (= not found) if addNew is false + if(!addNew) + return -1; + // If we did not return until here, then this client is definitely new // Store ID int clientID = counters->clients; diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 9aa4869d..45d8bac9 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -137,7 +137,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * int domainID = findDomainID(domain); // Go through already knows clients and see if it is one of them - int clientID = findClientID(client); + int clientID = findClientID(client, true); // Save everything validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__); diff --git a/networktable.c b/networktable.c index 7f94c436..1ef16916 100644 --- a/networktable.c +++ b/networktable.c @@ -68,17 +68,6 @@ void read_arp_cache(void) if (num < 4) continue; - if (num == 5) - { - /* - * This happens for incomplete ARP entries for which there is - * no hardware address in the line. We don't use these - */ - //num = sscanf(linebuffer, "%s 0x%x 0x%x %99s %99s\n", - // ip, &type, &flags, mask, iface); - //hwaddr[0] = '\0'; - } - entries++; if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip); @@ -92,22 +81,53 @@ void read_arp_cache(void) // SQLite error break; } - else if(dbID == -1) + + // If we reach this point, we can check if this client + // is known to pihole-FTL + // false = do not create a new record if the client is + // unknown (only DNS requesting clients do this) + int clientID = findClientID(ip, false); + bool clientKnown = clientID >= 0; + + if(dbID == -1) { // Device not in database, add new entry dbquery("INSERT INTO network "\ "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\ "VALUES "\ - "(\"%s\",\"%s\",\"%s\",%lu,%lu,false);",\ - ip, hwaddr, iface, now, now); + "(\"%s\",\"%s\",\"%s\",%lu,%lu,%s);",\ + ip, hwaddr, iface, now, now, + clientKnown ? "true" : "false"); } else { - // Device already known, update lastSeen + // Device already in database, update lastSeen dbquery("UPDATE network "\ "SET lastSeen = %lu "\ "WHERE id = %i;",\ now, dbID); + + // Store if device uses Pi-hole + if(clientKnown) + { + // Device uses Pi-hole, update record + dbquery("UPDATE network "\ + "SET usesPihole = true "\ + "WHERE id = %i;",\ + dbID); + } + } + + char *hostname = NULL; + if(clientKnown) + hostname = getstr(clients[clientID].namepos); + if(hostname != NULL && strlen(hostname) > 0) + { + // Store host name + dbquery("UPDATE network "\ + "SET name = \"%s\" "\ + "WHERE id = %i;",\ + hostname, dbID); } } diff --git a/routines.h b/routines.h index c3485379..06c29ef9 100644 --- a/routines.h +++ b/routines.h @@ -29,7 +29,7 @@ void strtolower(char *str); int findOverTimeID(int overTimetimestamp); int findForwardID(const char * forward, bool count); int findDomainID(const char *domain); -int findClientID(const char *client); +int findClientID(const char *client, bool addNew); bool isValidIPv4(const char *addr); bool isValidIPv6(const char *addr); char *getDomainString(int queryID); From 82b43c332ba0bc1e871b88ecc2607c0c36327287 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Dec 2018 17:56:44 +0100 Subject: [PATCH 04/47] Parse ARP cache after storing queries into long-term database(usen the dedicated database thread) Signed-off-by: DL6ER --- FTL.h | 4 ++-- database.c | 3 +++ networktable.c | 60 ++++++++++++++++++++++++++++++-------------------- request.c | 5 ----- routines.h | 2 +- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/FTL.h b/FTL.h index 876910f7..e2a8f708 100644 --- a/FTL.h +++ b/FTL.h @@ -69,7 +69,7 @@ #define MAXITER 1000 // FTLDNS enums -enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER }; +enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER }; enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD }; enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN }; enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED }; @@ -213,7 +213,7 @@ typedef struct { } whitelistStruct; // Prepare timers, used mainly for debugging purposes -#define NUMTIMERS 5 +#define NUMTIMERS LAST_TIMER // Used to check memory integrity in various structs #define MAGICBYTE 0x57 diff --git a/database.c b/database.c index fe1dc64a..350cb3b7 100644 --- a/database.c +++ b/database.c @@ -656,6 +656,9 @@ void *DB_thread(void *val) delete_old_queries_in_DB(); DBdeleteoldqueries = false; } + + // Parse ARP cache (fill network table) + parse_arp_cache(); } sleepms(100); } diff --git a/networktable.c b/networktable.c index 1ef16916..dd24963a 100644 --- a/networktable.c +++ b/networktable.c @@ -9,6 +9,7 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "shmem.h" #define ARPCACHE "/proc/net/arp" bool create_network_table(void) @@ -33,7 +34,7 @@ bool create_network_table(void) } // Read kernel's ARP cache using procfs -void read_arp_cache(void) +void parse_arp_cache(void) { FILE* arpfp = NULL; // Try to access the kernel's ARP cache @@ -51,6 +52,9 @@ void read_arp_cache(void) return; } + // Start ARP timer + if(debug) timer_start(ARP_TIMER); + // Prepare buffers char * linebuffer = NULL; size_t linebuffersize = 0; @@ -86,52 +90,60 @@ void read_arp_cache(void) // is known to pihole-FTL // false = do not create a new record if the client is // unknown (only DNS requesting clients do this) + lock_shm(); int clientID = findClientID(ip, false); + unlock_shm(); bool clientKnown = clientID >= 0; + char *hostname = NULL; + if(clientKnown) + hostname = getstr(clients[clientID].namepos); + if(dbID == -1) { // Device not in database, add new entry dbquery("INSERT INTO network "\ - "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\ + "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole,name) "\ "VALUES "\ - "(\"%s\",\"%s\",\"%s\",%lu,%lu,%s);",\ + "(\"%s\",\"%s\",\"%s\",%lu, %lu, %s, \"%s\");",\ ip, hwaddr, iface, now, now, - clientKnown ? "true" : "false"); + clientKnown ? "true" : "false", + hostname == NULL ? "" : hostname); } else { - // Device already in database, update lastSeen + // Start collecting database commands + dbquery("BEGIN TRANSACTION"); + + // Update lastSeen dbquery("UPDATE network "\ "SET lastSeen = %lu "\ "WHERE id = %i;",\ now, dbID); // Store if device uses Pi-hole - if(clientKnown) - { - // Device uses Pi-hole, update record - dbquery("UPDATE network "\ - "SET usesPihole = true "\ - "WHERE id = %i;",\ - dbID); - } - } - - char *hostname = NULL; - if(clientKnown) - hostname = getstr(clients[clientID].namepos); - if(hostname != NULL && strlen(hostname) > 0) - { - // Store host name dbquery("UPDATE network "\ - "SET name = \"%s\" "\ + "SET usesPihole = %s "\ "WHERE id = %i;",\ - hostname, dbID); - } + clientKnown ? "true" : "false", dbID); + // Store hostname if available + if(hostname != NULL && strlen(hostname) > 0) + { + // Store host name + dbquery("UPDATE network "\ + "SET name = \"%s\" "\ + "WHERE id = %i;",\ + hostname, dbID); + } + + // Actually update the database + dbquery("COMMIT"); + } } + if(debug) logg("ARP table processing took %.1ems",timer_elapsed_msec(ARP_TIMER)); + // Close file handle fclose(arpfp); diff --git a/request.c b/request.c index 2e313b5a..f83b7c5e 100644 --- a/request.c +++ b/request.c @@ -168,11 +168,6 @@ void process_request(char *client_message, int *sock) read_regex_from_file(); unlock_shm(); } - else if(command(client_message, ">arp")) - { - processed = true; - read_arp_cache(); - } // Test only at the end if we want to quit or kill // so things can be processed before diff --git a/routines.h b/routines.h index 06c29ef9..c7630805 100644 --- a/routines.h +++ b/routines.h @@ -134,4 +134,4 @@ void addOverTimeClientSlot(); // networktable.c bool create_network_table(void); -void read_arp_cache(void); +void parse_arp_cache(void); From 95d3c02c5f6066100358d0627575c7fdecd140b7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Dec 2018 22:35:16 +0100 Subject: [PATCH 05/47] Store lastQuery property for known clients Signed-off-by: DL6ER --- FTL.h | 1 + database.c | 6 +++++- datastructure.c | 2 ++ dnsmasq_interface.c | 3 +++ networktable.c | 36 ++++++++++++++++-------------------- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/FTL.h b/FTL.h index e2a8f708..34135449 100644 --- a/FTL.h +++ b/FTL.h @@ -187,6 +187,7 @@ typedef struct { unsigned long long ippos; unsigned long long namepos; bool new; + time_t lastQuery; } clientsDataStruct; typedef struct { diff --git a/database.c b/database.c index 350cb3b7..2164bd6b 100644 --- a/database.c +++ b/database.c @@ -708,7 +708,7 @@ void read_data_from_DB(void) while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0); - int queryTimeStamp = sqlite3_column_int(stmt, 1); + time_t queryTimeStamp = sqlite3_column_int(stmt, 1); // 1483228800 = 01/01/2017 @ 12:00am (UTC) if(queryTimeStamp < 1483228800) { @@ -791,6 +791,7 @@ void read_data_from_DB(void) // Store this query in memory validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__); + validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); queries[queryIndex].magic = MAGICBYTE; queries[queryIndex].timestamp = queryTimeStamp; queries[queryIndex].type = type; @@ -805,6 +806,9 @@ void read_data_from_DB(void) queries[queryIndex].response = 0; queries[queryIndex].AD = false; + // Update lastQuery of corresponding client + clients[clientID].lastQuery = queryTimeStamp; + // Handle type counters if(type >= TYPE_A && type < TYPE_MAX) { diff --git a/datastructure.c b/datastructure.c index 610d4cd9..4819947a 100644 --- a/datastructure.c +++ b/datastructure.c @@ -222,6 +222,8 @@ int findClientID(const char *client, bool addNew) // to be done separately to be non-blocking clients[clientID].new = true; clients[clientID].namepos = 0; + // No query seen so far + clients[clientID].lastQuery = 0; // Increase counter by one counters->clients++; diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 45d8bac9..16bd40f9 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -179,6 +179,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Update overTime data structure with the new client overTimeClientData[clientID][timeidx]++; + // Set lastQuery timer for network table + clients[clientID].lastQuery = querytimestamp; + // Try blocking regex if configured validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__); if(domains[domainID].regexmatch == REGEX_UNKNOWN && blockingstatus != BLOCKING_DISABLED) diff --git a/networktable.c b/networktable.c index dd24963a..89eae08d 100644 --- a/networktable.c +++ b/networktable.c @@ -22,8 +22,7 @@ bool create_network_table(void) "interface TEXT NOT NULL, " \ "name TEXT, " \ "firstSeen INTEGER NOT NULL, " \ - "lastSeen INTEGER NOT NULL, " \ - "usesPihole BOOLEAN NOT NULL );"); + "lastQuery INTEGER NOT NULL);"); if(!ret){ dbclose(); return false; } // Update database version to 3 @@ -72,9 +71,6 @@ void parse_arp_cache(void) if (num < 4) continue; - entries++; - if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip); - // Get ID of this device in our network database. If it cannot be found, then this is a new device char querystr[256]; sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr); @@ -97,35 +93,34 @@ void parse_arp_cache(void) char *hostname = NULL; if(clientKnown) + { + validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); hostname = getstr(clients[clientID].namepos); + } if(dbID == -1) { // Device not in database, add new entry dbquery("INSERT INTO network "\ - "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole,name) "\ + "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\ "VALUES "\ - "(\"%s\",\"%s\",\"%s\",%lu, %lu, %s, \"%s\");",\ - ip, hwaddr, iface, now, now, - clientKnown ? "true" : "false", + "(\"%s\",\"%s\",\"%s\",%lu, 0, \"%s\");",\ + ip, hwaddr, iface, now, hostname == NULL ? "" : hostname); } - else + else if(clientKnown) { // Start collecting database commands dbquery("BEGIN TRANSACTION"); - // Update lastSeen + // Update lastQuery, only use new value if larger + // clients[clientID].lastQuery may be zero if this + // client is only known from a database entry but has + // not been seen since then dbquery("UPDATE network "\ - "SET lastSeen = %lu "\ + "SET lastQuery = MAX(lastQuery, %ld) "\ "WHERE id = %i;",\ - now, dbID); - - // Store if device uses Pi-hole - dbquery("UPDATE network "\ - "SET usesPihole = %s "\ - "WHERE id = %i;",\ - clientKnown ? "true" : "false", dbID); + clients[clientID].lastQuery, dbID); // Store hostname if available if(hostname != NULL && strlen(hostname) > 0) @@ -140,9 +135,10 @@ void parse_arp_cache(void) // Actually update the database dbquery("COMMIT"); } + entries++; } - if(debug) logg("ARP table processing took %.1ems",timer_elapsed_msec(ARP_TIMER)); + if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); // Close file handle fclose(arpfp); From 68a584ad8bf05bfe119aca0b10d9848afc16dbea Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Dec 2018 01:21:00 +0100 Subject: [PATCH 06/47] Reduce code duplication Signed-off-by: DL6ER --- database.c | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/database.c b/database.c index 2164bd6b..23bd2e46 100644 --- a/database.c +++ b/database.c @@ -252,43 +252,20 @@ void db_init(void) int db_get_FTL_property(unsigned int ID) { - int rc, ret = 0; - sqlite3_stmt* dbstmt; - char *querystring = NULL; - // Prepare SQL statement - ret = asprintf(&querystring, "SELECT VALUE FROM ftl WHERE id = %u;",ID); + char* querystr = NULL; + int ret = asprintf(&querystr, "SELECT VALUE FROM ftl WHERE id = %u;", ID); - if(querystring == NULL || ret < 0) + if(querystr == NULL || ret < 0) { - logg("Memory allocation failed in db_get_FTL_property, not saving query with ID = %u (%i)", ID, ret); + logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret); return false; } - rc = sqlite3_prepare(db, querystring, -1, &dbstmt, NULL); - if( rc ){ - logg("db_get_FTL_property() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); - logg("Query: \"%s\"", querystring); - dbclose(); - check_database(rc); - return -1; - } - free(querystring); + int value = db_query_int(querystr); + free(querystr); - // Evaluate SQL statement - rc = sqlite3_step(dbstmt); - if( rc != SQLITE_ROW ){ - logg("db_get_FTL_property() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return -1; - } - - int result = sqlite3_column_int(dbstmt, 0); - - sqlite3_finalize(dbstmt); - - return result; + return value; } bool db_set_FTL_property(unsigned int ID, int value) From 465a4900eca6935559c0b82f12ee827f94ffbe48 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Dec 2018 01:21:31 +0100 Subject: [PATCH 07/47] Use one big transaction for changes to be written to the network table and skip incomplete entires when parsing the ARP cache Signed-off-by: DL6ER --- networktable.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/networktable.c b/networktable.c index 89eae08d..2c918047 100644 --- a/networktable.c +++ b/networktable.c @@ -61,6 +61,9 @@ void parse_arp_cache(void) int type, flags, entries = 0; time_t now = time(NULL); + // Start collecting database commands + dbquery("BEGIN TRANSACTION"); + // Read ARP cache line by line while(getline(&linebuffer, &linebuffersize, arpfp) != -1) { @@ -71,10 +74,28 @@ void parse_arp_cache(void) if (num < 4) continue; + // Skip incomplete entires, i.e., entries without C (complete) flag + if(!(flags & 0x02)) + continue; + // Get ID of this device in our network database. If it cannot be found, then this is a new device - char querystr[256]; - sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr); + // We match both IP *and* MAC address + // Same MAC, two IPs: Non-deterministic DHCP server, treat as two entries + // Same IP, two MACs: Either non-deterministic DHCP server or (almost) full DHCP address pool + // We can run this SELECT inside the currently active transaction as only the + // changed to the database are collected for latter commitment. Read-only access + // such as this SELECT command will be executed immediately on the database. + char* querystr = NULL; + int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr); + + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in parse_arp_cache (%i)", ret); + break; + } + int dbID = db_query_int(querystr); + free(querystr); if(dbID == -2) { @@ -110,9 +131,6 @@ void parse_arp_cache(void) } else if(clientKnown) { - // Start collecting database commands - dbquery("BEGIN TRANSACTION"); - // Update lastQuery, only use new value if larger // clients[clientID].lastQuery may be zero if this // client is only known from a database entry but has @@ -131,13 +149,13 @@ void parse_arp_cache(void) "WHERE id = %i;",\ hostname, dbID); } - - // Actually update the database - dbquery("COMMIT"); } entries++; } + // Actually update the database + dbquery("COMMIT"); + if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); // Close file handle From 3a8c87907a6a07989cf88e98b4ef371a21c0fb6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Dec 2018 01:35:50 +0100 Subject: [PATCH 08/47] Initialize new devices with lastQuery property if available Signed-off-by: DL6ER --- database.c | 6 +----- networktable.c | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/database.c b/database.c index 23bd2e46..e0ccd4da 100644 --- a/database.c +++ b/database.c @@ -259,7 +259,7 @@ int db_get_FTL_property(unsigned int ID) if(querystr == NULL || ret < 0) { logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret); - return false; + return -2; } int value = db_query_int(querystr); @@ -289,10 +289,6 @@ bool db_update_counters(int total, int blocked) int db_query_int(const char* querystr) { - // Check if database is enabled - if(!database) - return -2; - sqlite3_stmt* stmt; int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); if( rc ){ diff --git a/networktable.c b/networktable.c index 2c918047..e3c0d89a 100644 --- a/networktable.c +++ b/networktable.c @@ -125,8 +125,9 @@ void parse_arp_cache(void) dbquery("INSERT INTO network "\ "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\ "VALUES "\ - "(\"%s\",\"%s\",\"%s\",%lu, 0, \"%s\");",\ + "(\"%s\",\"%s\",\"%s\",%lu, %lu, \"%s\");",\ ip, hwaddr, iface, now, + clientKnown ? clients[clientID].lastQuery : 0L, hostname == NULL ? "" : hostname); } else if(clientKnown) From 54253bd7d59cd7afc07eee51c2a72bcf10eab1c0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Dec 2018 11:25:41 +0100 Subject: [PATCH 09/47] Added more comments Signed-off-by: DL6ER --- networktable.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/networktable.c b/networktable.c index e3c0d89a..ce8c1df4 100644 --- a/networktable.c +++ b/networktable.c @@ -87,13 +87,13 @@ void parse_arp_cache(void) // such as this SELECT command will be executed immediately on the database. char* querystr = NULL; int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr); - if(querystr == NULL || ret < 0) { logg("Memory allocation failed in parse_arp_cache (%i)", ret); break; } + // Perform SQL query int dbID = db_query_int(querystr); free(querystr); @@ -110,8 +110,12 @@ void parse_arp_cache(void) lock_shm(); int clientID = findClientID(ip, false); unlock_shm(); + + // This client is known (by its IP address) to pihole-FTL if + // findClientID() returned a non-negative index bool clientKnown = clientID >= 0; + // Get hostname of this client if the client is known char *hostname = NULL; if(clientKnown) { @@ -119,17 +123,17 @@ void parse_arp_cache(void) hostname = getstr(clients[clientID].namepos); } + // Device not in database, add new entry if(dbID == -1) { - // Device not in database, add new entry dbquery("INSERT INTO network "\ "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\ - "VALUES "\ - "(\"%s\",\"%s\",\"%s\",%lu, %lu, \"%s\");",\ + "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, \"%s\");",\ ip, hwaddr, iface, now, clientKnown ? clients[clientID].lastQuery : 0L, hostname == NULL ? "" : hostname); } + // Device in database AND client known to Pi-hole else if(clientKnown) { // Update lastQuery, only use new value if larger @@ -151,12 +155,17 @@ void parse_arp_cache(void) hostname, dbID); } } + // else: + // Device in database but not known to Pi-hole: No action required + + // Count number of processed ARP cache entries entries++; } // Actually update the database dbquery("COMMIT"); + // Debug logging if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); // Close file handle From ac6ee9e1ef71a982a7e88ba0ae359c87896d6517 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Dec 2018 11:29:28 +0100 Subject: [PATCH 10/47] Add config option to disable ARP cache parsing Signed-off-by: DL6ER --- FTL.h | 1 + config.c | 13 +++++++++++++ database.c | 5 +++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/FTL.h b/FTL.h index 34135449..8628ea20 100644 --- a/FTL.h +++ b/FTL.h @@ -149,6 +149,7 @@ typedef struct { bool regex_debugmode; bool analyze_only_A_AAAA; bool DBimport; + bool parse_arp_cache; } ConfigStruct; // Dynamic structs diff --git a/config.c b/config.c index 2ba5b03b..fd000f03 100644 --- a/config.c +++ b/config.c @@ -319,6 +319,19 @@ void read_FTLconf(void) // AUDITLISTFILE getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist); + // PARSE_ARP_CACHE + // defaults to: Yes + config.parse_arp_cache = true; + buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE"); + + if(buffer != NULL && strcasecmp(buffer, "false") == 0) + config.parse_arp_cache = false; + + if(config.parse_arp_cache) + logg(" PARSE_ARP_CACHE: Active"); + else + logg(" PARSE_ARP_CACHE: Inactive"); + logg("Finished config file parsing"); // Release memory diff --git a/database.c b/database.c index e0ccd4da..1aed35b8 100644 --- a/database.c +++ b/database.c @@ -630,8 +630,9 @@ void *DB_thread(void *val) DBdeleteoldqueries = false; } - // Parse ARP cache (fill network table) - parse_arp_cache(); + // Parse ARP cache (fill network table) if enabled + if (config.parse_arp_cache) + parse_arp_cache(); } sleepms(100); } From d8fdf76cebb1e52c1e35348a0c53039762a828e4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 28 Dec 2018 12:18:35 +0100 Subject: [PATCH 11/47] Store number of queries per client in the database Signed-off-by: DL6ER --- FTL.h | 1 + database.c | 5 +++-- datastructure.c | 1 + dnsmasq_interface.c | 3 ++- networktable.c | 18 ++++++++++++++---- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/FTL.h b/FTL.h index 8628ea20..01544846 100644 --- a/FTL.h +++ b/FTL.h @@ -189,6 +189,7 @@ typedef struct { unsigned long long namepos; bool new; time_t lastQuery; + unsigned int numQueriesARP; } clientsDataStruct; typedef struct { diff --git a/database.c b/database.c index 1aed35b8..f4815fea 100644 --- a/database.c +++ b/database.c @@ -776,12 +776,13 @@ void read_data_from_DB(void) queries[queryIndex].timeidx = timeidx; queries[queryIndex].db = dbid; queries[queryIndex].id = 0; - queries[queryIndex].complete = true; // Mark as all information is avaiable + queries[queryIndex].complete = true; // Mark as all information is available queries[queryIndex].response = 0; queries[queryIndex].AD = false; - // Update lastQuery of corresponding client + // Set lastQuery timer and add one query for network table clients[clientID].lastQuery = queryTimeStamp; + clients[clientID].numQueriesARP++; // Handle type counters if(type >= TYPE_A && type < TYPE_MAX) diff --git a/datastructure.c b/datastructure.c index 4819947a..01fe403f 100644 --- a/datastructure.c +++ b/datastructure.c @@ -224,6 +224,7 @@ int findClientID(const char *client, bool addNew) clients[clientID].namepos = 0; // No query seen so far clients[clientID].lastQuery = 0; + clients[clientID].numQueriesARP = 0; // Increase counter by one counters->clients++; diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 16bd40f9..7c83108d 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -179,8 +179,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Update overTime data structure with the new client overTimeClientData[clientID][timeidx]++; - // Set lastQuery timer for network table + // Set lastQuery timer and add one query for network table clients[clientID].lastQuery = querytimestamp; + clients[clientID].numQueriesARP++; // Try blocking regex if configured validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__); diff --git a/networktable.c b/networktable.c index ce8c1df4..6d474d3e 100644 --- a/networktable.c +++ b/networktable.c @@ -22,7 +22,8 @@ bool create_network_table(void) "interface TEXT NOT NULL, " \ "name TEXT, " \ "firstSeen INTEGER NOT NULL, " \ - "lastQuery INTEGER NOT NULL);"); + "lastQuery INTEGER NOT NULL, " \ + "numQueries INTEGER NOT NULL);"); if(!ret){ dbclose(); return false; } // Update database version to 3 @@ -127,16 +128,17 @@ void parse_arp_cache(void) if(dbID == -1) { dbquery("INSERT INTO network "\ - "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\ - "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, \"%s\");",\ + "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name) "\ + "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\");",\ ip, hwaddr, iface, now, clientKnown ? clients[clientID].lastQuery : 0L, + clientKnown ? clients[clientID].numQueriesARP : 0u, hostname == NULL ? "" : hostname); } // Device in database AND client known to Pi-hole else if(clientKnown) { - // Update lastQuery, only use new value if larger + // Update lastQuery. Only use new value if larger // clients[clientID].lastQuery may be zero if this // client is only known from a database entry but has // not been seen since then @@ -145,6 +147,14 @@ void parse_arp_cache(void) "WHERE id = %i;",\ clients[clientID].lastQuery, dbID); + // Update numQueries. Add queries seen since last update + // and reset counter afterwards + dbquery("UPDATE network "\ + "SET numQueries = numQueries + %u "\ + "WHERE id = %i;",\ + clients[clientID].numQueriesARP, dbID); + clients[clientID].numQueriesARP = 0; + // Store hostname if available if(hostname != NULL && strlen(hostname) > 0) { From f2f543c8d6b87f8da6f1b7e54693119004ff9352 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 28 Dec 2018 20:39:03 +0100 Subject: [PATCH 12/47] Do not count in findClient() if search is triggered from parse_arp_cache() Signed-off-by: DL6ER --- datastructure.c | 10 ++++++---- gc.c | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/datastructure.c b/datastructure.c index 01fe403f..721dcf82 100644 --- a/datastructure.c +++ b/datastructure.c @@ -176,7 +176,7 @@ int findDomainID(const char *domain) return domainID; } -int findClientID(const char *client, bool addNew) +int findClientID(const char *client, bool count) { int i; // Compare content of client against known client IP addresses @@ -191,14 +191,16 @@ int findClientID(const char *client, bool addNew) // If so, compare the full IP using strcmp if(strcmp(getstr(clients[i].ippos), client) == 0) { - clients[i].count++; + // Add one if count == true (do not add one, e.g., during ARP table processing) + if(count) clients[i].count++; return i; } } - // Return -1 (= not found) if addNew is false - if(!addNew) + // Return -1 (= not found) if count is false ... + if(!count) return -1; + // ... otherwise proceed with adding a new client entry // If we did not return until here, then this client is definitely new // Store ID diff --git a/gc.c b/gc.c index 813b96aa..b5d3f915 100644 --- a/gc.c +++ b/gc.c @@ -51,7 +51,6 @@ void *GC_thread(void *val) if(queries[i].timestamp > mintime) break; - // Adjust total counters and total over time data // We cannot edit counters->queries directly as it is used // as max ID for the queries[] struct From 92d819899eb170031330c361b42da7229ecbddf4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Dec 2018 23:34:58 +0100 Subject: [PATCH 13/47] Add MAC->Vendor database support. We will provide the database as an optional auxiliary file. Signed-off-by: DL6ER --- FTL.h | 1 + config.c | 3 ++ memory.c | 1 + networktable.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/FTL.h b/FTL.h index 01544846..a66ad50e 100644 --- a/FTL.h +++ b/FTL.h @@ -98,6 +98,7 @@ typedef struct { char* port; char* db; char* socketfile; + char* macvendordb; } FTLFileNamesStruct; typedef struct { diff --git a/config.c b/config.c index fd000f03..3f5f71b2 100644 --- a/config.c +++ b/config.c @@ -319,6 +319,9 @@ void read_FTLconf(void) // AUDITLISTFILE getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist); + // MACVENDORDB + getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb); + // PARSE_ARP_CACHE // defaults to: Yes config.parse_arp_cache = true; diff --git a/memory.c b/memory.c index a959a022..30e8b2dd 100644 --- a/memory.c +++ b/memory.c @@ -20,6 +20,7 @@ FTLFileNamesStruct FTLfiles = { NULL, NULL, NULL, + NULL, NULL }; diff --git a/networktable.c b/networktable.c index 6d474d3e..c29cd7fc 100644 --- a/networktable.c +++ b/networktable.c @@ -12,6 +12,9 @@ #include "shmem.h" #define ARPCACHE "/proc/net/arp" +// Private prototypes +char* getMACVendor(const char* hwaddr); + bool create_network_table(void) { bool ret; @@ -23,7 +26,8 @@ bool create_network_table(void) "name TEXT, " \ "firstSeen INTEGER NOT NULL, " \ "lastQuery INTEGER NOT NULL, " \ - "numQueries INTEGER NOT NULL);"); + "numQueries INTEGER NOT NULL," \ + "macVendor TEXT);"); if(!ret){ dbclose(); return false; } // Update database version to 3 @@ -127,13 +131,17 @@ void parse_arp_cache(void) // Device not in database, add new entry if(dbID == -1) { + char* macVendor = getMACVendor(hwaddr); dbquery("INSERT INTO network "\ - "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name) "\ - "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\");",\ + "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\ + "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\", \"%s\");",\ ip, hwaddr, iface, now, clientKnown ? clients[clientID].lastQuery : 0L, clientKnown ? clients[clientID].numQueriesARP : 0u, - hostname == NULL ? "" : hostname); + hostname == NULL ? "" : hostname, + macVendor); + if(strlen(macVendor) > 0) + free(macVendor); } // Device in database AND client known to Pi-hole else if(clientKnown) @@ -184,3 +192,71 @@ void parse_arp_cache(void) // Close database connection dbclose(); } + +char* getMACVendor(const char* hwaddr) +{ + struct stat st; + if(stat(FTLfiles.macvendordb, &st) != 0 || strlen(hwaddr) != 17) + { + // File does not exist or MAC address is incomplete + if(debug) logg("getMACVenor(%s): %s does not exist or MAC invalid (length %lu)", hwaddr, FTLfiles.macvendordb, strlen(hwaddr)); + return ""; + } + + sqlite3 *macdb; + int rc = sqlite3_open_v2(FTLfiles.macvendordb, &macdb, SQLITE_OPEN_READONLY, NULL); + if( rc ){ + logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); + sqlite3_close(macdb); + return ""; + } + + char *querystr = NULL; + // Only keep "XX:YY:ZZ" (8 characters) + char * hwaddrshort = strdup(hwaddr); + hwaddrshort[8] = '\0'; + rc = asprintf(&querystr, "SELECT vendor FROM macvendor WHERE mac LIKE \"%s\";", hwaddrshort); + if(rc < 1) + { + logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc); + sqlite3_close(macdb); + return ""; + } + free(hwaddrshort); + + sqlite3_stmt* stmt; + rc = sqlite3_prepare_v2(macdb, querystr, -1, &stmt, NULL); + if( rc ){ + logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb)); + sqlite3_close(macdb); + return ""; + } + free(querystr); + + char *vendor = NULL; + rc = sqlite3_step(stmt); + if(rc == SQLITE_ROW) + { + const unsigned char *result = sqlite3_column_text(stmt, 0); + // Need to use sprintf(%s) to convert unsigned char* to + // standard C string literals (which are char*) + if(asprintf(&vendor, "%s", result) < 1) + logg("getMACVendor(%s) - Allocation error 2"); + } + else if(rc == SQLITE_DONE) + { + // Not found + vendor = ""; + } + else + { + // Error + logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); + vendor = ""; + } + + sqlite3_finalize(stmt); + sqlite3_close(macdb); + + return vendor; +} From a2b7a241e225dcbabead6b01bbc8db2bf7b67dca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 31 Dec 2018 00:18:23 +0100 Subject: [PATCH 14/47] Add routine to update all existing network table entries using the latest mac->vendor database Signed-off-by: DL6ER --- networktable.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/networktable.c b/networktable.c index c29cd7fc..e7a03c25 100644 --- a/networktable.c +++ b/networktable.c @@ -241,7 +241,7 @@ char* getMACVendor(const char* hwaddr) // Need to use sprintf(%s) to convert unsigned char* to // standard C string literals (which are char*) if(asprintf(&vendor, "%s", result) < 1) - logg("getMACVendor(%s) - Allocation error 2"); + logg("getMACVendor(%s) - Allocation error 2", hwaddr); } else if(rc == SQLITE_DONE) { @@ -260,3 +260,78 @@ char* getMACVendor(const char* hwaddr) return vendor; } + +void updateMACVendorRecords() +{ + struct stat st; + if(stat(FTLfiles.macvendordb, &st) != 0) + { + // File does not exist or MAC address is incomplete + if(debug) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb); + return; + } + + sqlite3 *db; + int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL); + if( rc ){ + logg("updateMACVendorRecords() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); + sqlite3_close(db); + return; + } + + sqlite3_stmt* stmt; + const char* querystr = "SELECT id,hwaddr FROM network;"; + rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc ){ + logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", querystr, rc, sqlite3_errmsg(db)); + sqlite3_close(db); + return; + } + + while((rc = sqlite3_step(stmt)) == SQLITE_ROW) + { + const int id = sqlite3_column_int(stmt, 0); + const unsigned char *hwaddr = sqlite3_column_text(stmt, 1); + // Need to use sprintf(%s) to convert unsigned char* to + // standard C string literals (which are char*) + char *querystr = NULL; + if(asprintf(&querystr, "%s", hwaddr) < 1) + { + logg("updateMACVendorRecords() - Allocation error 1"); + break; + } + + // Get vendor for MAC + char* vendor = getMACVendor(querystr); + free(querystr); + + // Prepare UPDATE statement + if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1) + { + logg("updateMACVendorRecords() - Allocation error 2"); + break; + } + + // Execute prepared statement + char *zErrMsg = NULL; + rc = sqlite3_exec(db, querystr, NULL, NULL, &zErrMsg); + if( rc != SQLITE_OK ){ + logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg); + sqlite3_free(zErrMsg); + break; + } + + // Free allocated memory + free(querystr); + if(strlen(vendor) > 0) + free(vendor); + } + if(rc != SQLITE_DONE) + { + // Error + logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); + } + + sqlite3_finalize(stmt); + sqlite3_close(db); +} From bd749f4a4c83114e77edf0afc0a7fdb721e49a7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 31 Dec 2018 10:33:33 +0100 Subject: [PATCH 15/47] Added python3 script to automatically generate the macvendor database used by FTL Signed-off-by: DL6ER --- aux/macvendor.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 aux/macvendor.py diff --git a/aux/macvendor.py b/aux/macvendor.py new file mode 100644 index 00000000..41cb97dc --- /dev/null +++ b/aux/macvendor.py @@ -0,0 +1,69 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine - auxiliary files +# MAC -> Vendor database generator +# +# This is a python3 script +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +import os, re, urllib.request, sqlite3 +import unicodecsv as unicodecsv + +# Download raw data from Wireshark's website +# We use the official URL recommended in the header of this file +print("Downloading...") +urllib.request.urlretrieve("https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf", "manuf.data") +print("...done") + +# Read file into memory and process lines +file = open("manuf.data", "r") +data = [] +print("Processing...") +for line in file: + line = line.strip() + + # Skip comments and empty lines + if line[:1] == "#" or line == "": + continue + # \s = Unicode whitespace characters, including [ \t\n\r\f\v] + cols = re.split("\s\s+|\t", line) + # Use try/except chain to catch empty/incomplete lines without failing hard + try: + # Strip whitespace and quotation marks (some entries are incomplete and cause errors with the CSV parser otherwise) + mac = cols[0].strip().strip("\"") + except: + continue + try: + desc_short = cols[1].strip().strip("\"") + except: + desc_short = "" + try: + desc_long = cols[2].strip().strip("\"") + except: + desc_long = "" + + # Only add long description where available + # There are a few vendors for which only the + # short description field is used + if(len(desc_long) > 0): + data.append([mac, desc_long]) + else: + data.append([mac, desc_short]) +print("...done") +file.close() + +# Create database +database = "macvendor.db" +os.remove(database) +print("Generating database...") +con = sqlite3.connect(database) +cur = con.cursor() +cur.execute("CREATE TABLE macvendor (mac TEXT NOT NULL, vendor TEXT NOT NULL, PRIMARY KEY (mac))") +cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data) +con.commit() +print("...done.") +print("Lines inserted into database:", cur.rowcount) From 85e954f4338cc9d8171849467b8343c97fb76f6f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 31 Dec 2018 11:48:05 +0100 Subject: [PATCH 16/47] Optimize database after creation (reduces filesize by about 10%) + don't fail if there was no previous database present Signed-off-by: DL6ER --- aux/macvendor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/aux/macvendor.py b/aux/macvendor.py index 41cb97dc..a36bc46f 100644 --- a/aux/macvendor.py +++ b/aux/macvendor.py @@ -58,7 +58,13 @@ file.close() # Create database database = "macvendor.db" -os.remove(database) + +# Try to delete old database file, pass if no old file exists +try: + os.remove(database) +except OSError: + pass + print("Generating database...") con = sqlite3.connect(database) cur = con.cursor() @@ -66,4 +72,7 @@ cur.execute("CREATE TABLE macvendor (mac TEXT NOT NULL, vendor TEXT NOT NULL, PR cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data) con.commit() print("...done.") +print("Optimizing database...") +con.execute("VACUUM") +print("...done") print("Lines inserted into database:", cur.rowcount) From 9c8f0704c1ddb767d1c8bc5a5301751a276fb42f Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 31 Dec 2018 17:44:14 -0500 Subject: [PATCH 17/47] Reuse the empty string at position 0 in string shared memory If an empty string is being added, reuse the one at position zero. This makes sure all empty strings used elsewhere in shared memory have a position of zero. Signed-off-by: Mcat12 --- shmem.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shmem.c b/shmem.c index 8fd333d0..e8cf89e3 100644 --- a/shmem.c +++ b/shmem.c @@ -55,6 +55,11 @@ unsigned long long addstr(const char *str) // Get string length size_t len = strlen(str); + // If this is an empty string, use the one at position zero + if(len == 0) { + return 0; + } + if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); // Reserve additional memory if necessary From 8ad22b283d81c3844d695d415173f5f37a4849c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 1 Jan 2019 10:36:45 +0100 Subject: [PATCH 18/47] Mac vendor database generator: Remove quotation marks as these might interfere with later INSERT / UPDATE commands Signed-off-by: DL6ER --- aux/macvendor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aux/macvendor.py b/aux/macvendor.py index a36bc46f..ce2e640b 100644 --- a/aux/macvendor.py +++ b/aux/macvendor.py @@ -29,6 +29,9 @@ for line in file: # Skip comments and empty lines if line[:1] == "#" or line == "": continue + + # Remove quotation marks as these might interfere with later INSERT / UPDATE commands + line = re.sub("\'|\"","", line) # \s = Unicode whitespace characters, including [ \t\n\r\f\v] cols = re.split("\s\s+|\t", line) # Use try/except chain to catch empty/incomplete lines without failing hard From 9151a876dda5d7bd43b5b4ff2833fb5fb0e826b3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 1 Jan 2019 13:53:38 +0100 Subject: [PATCH 19/47] Always pass line of command execution from dnsmasq codebase to FTL hooks for debugging (no functional change) Signed-off-by: DL6ER --- dnsmasq_interface.c | 28 ++++++++++++++-------------- dnsmasq_interface.h | 30 +++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 9aa4869d..dceb87ed 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -25,7 +25,7 @@ static int findQueryID(int id); unsigned char* pihole_privacylevel = &config.privacylevel; char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "}; -void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type) +void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -112,7 +112,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Log new query if in debug mode char *proto = (type == UDP) ? "UDP" : "TCP"; - if(debug) logg("**** new %s %s \"%s\" from %s (ID %i)", proto, types, domain, client, id); + if(debug) logg("**** new %s %s \"%s\" from %s (ID %i, %s:%i)", proto, types, domain, client, id, file, line); // Update counters int timeidx = findOverTimeID(overTimetimestamp); @@ -239,7 +239,7 @@ static int findQueryID(int id) return -1; } -void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id) +void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -257,7 +257,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id strtolower(forward); // Debug logging - if(debug) logg("**** forwarded %s to %s (ID %i)", name, forward, id); + if(debug) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, forward, id, file, line); // Save status and forwardID in corresponding query identified by dnsmasq's ID int i = findQueryID(id); @@ -372,7 +372,7 @@ void FTL_dnsmasq_reload(void) read_regex_from_file(); } -void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) +void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -399,7 +399,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) if(debug) { - logg("**** got reply %s is %s (ID %i)", name, answer, id); + logg("**** got reply %s is %s (ID %i, %s:%i)", name, answer, id, file, line); print_flags(flags); } @@ -584,7 +584,7 @@ static void query_externally_blocked(int i) queries[i].status = QUERY_EXTERNAL_BLOCKED; } -void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id) +void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -613,7 +613,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, free(domain); // Debug logging - if(debug) logg("**** got cache answer for %s / %s / %s (ID %i)", name, dest, arg, id); + if(debug) logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line); if(debug) print_flags(flags); // Get response time @@ -730,7 +730,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, unlock_shm(); } -void FTL_dnssec(int status, int id) +void _FTL_dnssec(int status, int id, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -752,7 +752,7 @@ void FTL_dnssec(int status, int id) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** got DNSSEC details for %s: %i (ID %i)", getstr(domains[domainID].domainpos), status, id); + logg("**** got DNSSEC details for %s: %i (ID %i, %s:%i)", getstr(domains[domainID].domainpos), status, id, file, line); } // Iterate through possible values @@ -766,7 +766,7 @@ void FTL_dnssec(int status, int id) unlock_shm(); } -void FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id) +void _FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -793,7 +793,7 @@ void FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** AD bit set for %s (ID %i, RCODE %u)", getstr(domains[domainID].domainpos), id, rcode); + logg("**** AD bit set for %s (ID %i, RCODE %u, %s:%i)", getstr(domains[domainID].domainpos), id, rcode, file, line); } // Store AD bit in query data @@ -977,7 +977,7 @@ void getCacheInformation(int *sock) // which hasn't been looked up for the longest time is evicted. } -void FTL_forwarding_failed(struct server *server) +void _FTL_forwarding_failed(struct server *server, const char* file, const int line) { // Don't analyze anything if in PRIVACY_NOSTATS mode if(config.privacylevel >= PRIVACY_NOSTATS) @@ -996,7 +996,7 @@ void FTL_forwarding_failed(struct server *server) strtolower(forward); int forwardID = findForwardID(forward, false); - if(debug) logg("**** forwarding to %s (ID %i) failed", dest, forwardID); + if(debug) logg("**** forwarding to %s (ID %i, %s:%i) failed", dest, forwardID, file, line); forwarded[forwardID].failed++; diff --git a/dnsmasq_interface.h b/dnsmasq_interface.h index 527b23ce..0a5000ba 100644 --- a/dnsmasq_interface.h +++ b/dnsmasq_interface.h @@ -11,15 +11,27 @@ extern int socketfd, telnetfd4, telnetfd6; extern unsigned char* pihole_privacylevel; enum { TCP, UDP }; -void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type); -void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id); -void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id); -void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id); -void FTL_dnssec(int status, int id); +#define FTL_new_query(flags, name, addr, types, id, type) _FTL_new_query(flags, name, addr, types, id, type, __FILE__, __LINE__) +void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line); + +#define FTL_forwarded(flags, name, addr, id) _FTL_forwarded(flags, name, addr, id, __FILE__, __LINE__) +void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line); + +#define FTL_reply(flags, name, addr, id) _FTL_reply(flags, name, addr, id, __FILE__, __LINE__) +void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line); + +#define FTL_cache(flags, name, addr, arg, id) _FTL_cache(flags, name, addr, arg, id, __FILE__, __LINE__) +void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id, const char* file, const int line); + +#define FTL_dnssec(status, id) _FTL_dnssec(status, id, __FILE__, __LINE__) +void _FTL_dnssec(int status, int id, const char* file, const int line); + +#define FTL_header_ADbit(header4, rcode, id) _FTL_header_ADbit(header4, rcode, id, __FILE__, __LINE__) +void _FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id, const char* file, const int line); + +#define FTL_forwarding_failed(server) _FTL_forwarding_failed(server, __FILE__, __LINE__) +void _FTL_forwarding_failed(struct server *server, const char* file, const int line); + void FTL_dnsmasq_reload(void); void FTL_fork_and_bind_sockets(struct passwd *ent_pw); - -void FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id); - -void FTL_forwarding_failed(struct server *server); int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz); From a3410a78f65b7955b4c3a3dd410f36218da2b96a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 1 Jan 2019 14:27:08 +0100 Subject: [PATCH 20/47] Add dnsmasq_interface.h as dependency of the dnsmasq objects. This forces a rebuild whenever interface definitions are changed. Signed-off-by: DL6ER --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6e27a0e7..c167bb27 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o -DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h +DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o # Get git commit version and date From 070feb630a9e16e95c01aacbbee6d71c851ca3fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Jan 2019 17:56:12 +0100 Subject: [PATCH 21/47] Add check for required Linux capabilities. Signed-off-by: DL6ER --- Makefile | 4 ++-- capabilities.c | 37 +++++++++++++++++++++++++++++++++++++ main.c | 7 +++++-- routines.h | 3 +++ 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 capabilities.c diff --git a/Makefile b/Makefile index 6e27a0e7..d74d593f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h -FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o +FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o @@ -51,7 +51,7 @@ CCFLAGS=-std=gnu11 -I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_ # for dnsmasq we need the nettle crypto library and the gmp maths library # We link the two libraries statically. Althougth this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier #LIBS=-pthread -lnettle -lgmp -lhogweed -LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic -lrt +LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic -lrt -lcap # Flags for compiling with libidn : -lidn # Flags for compiling with libidn2: -lidn2 diff --git a/capabilities.c b/capabilities.c new file mode 100644 index 00000000..633cd0f0 --- /dev/null +++ b/capabilities.c @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Linux capability check routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include + +bool check_capabilities() +{ + if(!cap_get_bound(CAP_NET_ADMIN)) + { + // Needed for ARP-injection (used when we're the DHCP server) + logg("FATAL: Required linux capability CAP_NET_ADMIN not available"); + return false; + } + if(!cap_get_bound(CAP_NET_RAW)) + { + // Needed for raw socket access (necessary for ICMP) + logg("FATAL: Required linux capability CAP_NET_RAW not available"); + return false; + } + if(!cap_get_bound(CAP_NET_BIND_SERVICE)) + { + // Necessary for dynamic port binding + logg("FATAL: Required linux capability CAP_NET_BIND_SERVICE not available"); + return false; + } + + // All okay! + return true; +} diff --git a/main.c b/main.c index a785a650..41664a45 100644 --- a/main.c +++ b/main.c @@ -69,8 +69,11 @@ int main (int argc, char* argv[]) log_counter_info(); check_setupVarsconf(); - // Preparations done - start the resolver - main_dnsmasq(argc_dnsmasq, argv_dnsmasq); + // Check for availability of advanced capabilities + // immediately before starting the resolver. If all + // capabilities are available, we start the resolver + if(check_capabiltities()) + main_dnsmasq(argc_dnsmasq, argv_dnsmasq); logg("Shutting down..."); diff --git a/routines.h b/routines.h index b4ad082b..a5c9b100 100644 --- a/routines.h +++ b/routines.h @@ -126,3 +126,6 @@ void newOverTimeClient(); * This also updates `overTimeClientData`. */ void addOverTimeClientSlot(); + +// capabilities.c +bool check_capabilities(void); From 724ab54aa363d879d8352363e1cb27693252b2c2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Jan 2019 18:07:05 +0100 Subject: [PATCH 22/47] Just complain loudly instead of failing badly if capabilities are not available Signed-off-by: DL6ER --- capabilities.c | 12 +++++++++--- main.c | 9 +++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/capabilities.c b/capabilities.c index 633cd0f0..1a33fee8 100644 --- a/capabilities.c +++ b/capabilities.c @@ -16,19 +16,25 @@ bool check_capabilities() if(!cap_get_bound(CAP_NET_ADMIN)) { // Needed for ARP-injection (used when we're the DHCP server) - logg("FATAL: Required linux capability CAP_NET_ADMIN not available"); + logg("**************************************************************"); + logg("WARNING: Required linux capability CAP_NET_ADMIN not available"); + logg("**************************************************************"); return false; } if(!cap_get_bound(CAP_NET_RAW)) { // Needed for raw socket access (necessary for ICMP) - logg("FATAL: Required linux capability CAP_NET_RAW not available"); + logg("************************************************************"); + logg("WARNING: Required linux capability CAP_NET_RAW not available"); + logg("************************************************************"); return false; } if(!cap_get_bound(CAP_NET_BIND_SERVICE)) { // Necessary for dynamic port binding - logg("FATAL: Required linux capability CAP_NET_BIND_SERVICE not available"); + logg("*********************************************************************"); + logg("WARNING: Required linux capability CAP_NET_BIND_SERVICE not available"); + logg("*********************************************************************"); return false; } diff --git a/main.c b/main.c index 41664a45..bbbb94b0 100644 --- a/main.c +++ b/main.c @@ -70,10 +70,11 @@ int main (int argc, char* argv[]) check_setupVarsconf(); // Check for availability of advanced capabilities - // immediately before starting the resolver. If all - // capabilities are available, we start the resolver - if(check_capabiltities()) - main_dnsmasq(argc_dnsmasq, argv_dnsmasq); + // immediately before starting the resolver. + check_capabilities(); + + // Start the resolver + main_dnsmasq(argc_dnsmasq, argv_dnsmasq); logg("Shutting down..."); From ea6b4162ad64efa8b4fd224ee8663a30184d3e73 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 3 Jan 2019 10:31:48 +0100 Subject: [PATCH 23/47] Add libcap-dev to Dockerfiles Signed-off-by: DL6ER --- docker/aarch64/Dockerfile | 2 +- docker/arm/Dockerfile | 2 +- docker/armhf/Dockerfile | 2 +- docker/x86_32/Dockerfile | 2 +- docker/x86_64/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/aarch64/Dockerfile b/docker/aarch64/Dockerfile index 22f74484..d27951cc 100644 --- a/docker/aarch64/Dockerfile +++ b/docker/aarch64/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture arm64 && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:arm64 gcc-aarch64-linux-gnu libc-dev-arm64-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev ENV CC aarch64-linux-gnu-gcc diff --git a/docker/arm/Dockerfile b/docker/arm/Dockerfile index c7505f60..63562f9a 100644 --- a/docker/arm/Dockerfile +++ b/docker/arm/Dockerfile @@ -3,7 +3,7 @@ FROM debian:stretch RUN dpkg --add-architecture armhf && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:armhf \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev # Use Raspbian's GCC # This command was taken from https://github.com/dockcross/dockcross/blob/master/linux-armv6/Dockerfile diff --git a/docker/armhf/Dockerfile b/docker/armhf/Dockerfile index 433d8f94..d2f73d4e 100644 --- a/docker/armhf/Dockerfile +++ b/docker/armhf/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture armhf && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:armhf gcc-arm-linux-gnueabihf libc6-dev-armhf-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev ENV CC arm-linux-gnueabihf-gcc diff --git a/docker/x86_32/Dockerfile b/docker/x86_32/Dockerfile index 320db885..8bad1782 100644 --- a/docker/x86_32/Dockerfile +++ b/docker/x86_32/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture i386 && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:i386 gcc gcc-multilib \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev ENV CC gcc diff --git a/docker/x86_64/Dockerfile b/docker/x86_64/Dockerfile index 433c6299..4b33a730 100644 --- a/docker/x86_64/Dockerfile +++ b/docker/x86_64/Dockerfile @@ -2,6 +2,6 @@ FROM debian:stretch RUN apt-get update && \ apt-get install -y --no-install-recommends nettle-dev gcc libc-dev \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev ENV CC gcc From b0e5e17cd0388c1d9b7e2eb974e64fd134b4b088 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 3 Jan 2019 12:25:40 -0500 Subject: [PATCH 24/47] Use architecture-specific libcap-dev packages Signed-off-by: Mcat12 --- docker/aarch64/Dockerfile | 2 +- docker/arm/Dockerfile | 2 +- docker/armhf/Dockerfile | 2 +- docker/x86_32/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/aarch64/Dockerfile b/docker/aarch64/Dockerfile index d27951cc..b03a955c 100644 --- a/docker/aarch64/Dockerfile +++ b/docker/aarch64/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture arm64 && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:arm64 gcc-aarch64-linux-gnu libc-dev-arm64-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:arm64 ENV CC aarch64-linux-gnu-gcc diff --git a/docker/arm/Dockerfile b/docker/arm/Dockerfile index 63562f9a..67014741 100644 --- a/docker/arm/Dockerfile +++ b/docker/arm/Dockerfile @@ -3,7 +3,7 @@ FROM debian:stretch RUN dpkg --add-architecture armhf && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:armhf \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:armhf # Use Raspbian's GCC # This command was taken from https://github.com/dockcross/dockcross/blob/master/linux-armv6/Dockerfile diff --git a/docker/armhf/Dockerfile b/docker/armhf/Dockerfile index d2f73d4e..7eee3d71 100644 --- a/docker/armhf/Dockerfile +++ b/docker/armhf/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture armhf && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:armhf gcc-arm-linux-gnueabihf libc6-dev-armhf-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:armhf ENV CC arm-linux-gnueabihf-gcc diff --git a/docker/x86_32/Dockerfile b/docker/x86_32/Dockerfile index 8bad1782..2555601a 100644 --- a/docker/x86_32/Dockerfile +++ b/docker/x86_32/Dockerfile @@ -3,6 +3,6 @@ FROM debian:stretch RUN dpkg --add-architecture i386 && \ apt-get update && \ apt-get install -y --no-install-recommends nettle-dev:i386 gcc gcc-multilib \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev + make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:i386 ENV CC gcc From 31df63488f031ad3702653d6f93832b51524454a Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 3 Jan 2019 15:16:15 -0500 Subject: [PATCH 25/47] Fix arm library search path Signed-off-by: Mcat12 --- docker/arm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/arm/Dockerfile b/docker/arm/Dockerfile index 67014741..2ae0bccd 100644 --- a/docker/arm/Dockerfile +++ b/docker/arm/Dockerfile @@ -19,4 +19,4 @@ RUN wget ftl.pi-hole.net/libraries/libgmp.a -O /usr/local/lib/libgmp.a && \ wget ftl.pi-hole.net/libraries/libhogweed.a -O /usr/local/lib/libhogweed.a # Allow libnettle to be used, because this GCC doesn't have all the right header and library directories -ENV CC "arm-linux-gnueabihf-gcc -I/usr/include -I/usr/include/arm-linux-gnueabihf" +ENV CC "arm-linux-gnueabihf-gcc -I/usr/include -I/usr/include/arm-linux-gnueabihf -L/usr/lib/arm-linux-gnueabihf" From f8db82cbd2b61cc1ff839a6bfadf35db45824894 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 3 Jan 2019 15:35:45 -0500 Subject: [PATCH 26/47] Move -m32 flag from CI config to docker container Signed-off-by: Mcat12 --- .circleci/config.yml | 3 +-- docker/x86_32/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38b49d6f..c09390bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ version: 2 command: | BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master") - make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" + make GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" file pihole-FTL - run: name: "Upload" @@ -59,7 +59,6 @@ jobs: <<: *job_template environment: BIN_NAME: "pihole-FTL-linux-x86_32" - CFLAGS: "-m32" workflows: version: 2 diff --git a/docker/x86_32/Dockerfile b/docker/x86_32/Dockerfile index 2555601a..87dae55e 100644 --- a/docker/x86_32/Dockerfile +++ b/docker/x86_32/Dockerfile @@ -5,4 +5,4 @@ RUN dpkg --add-architecture i386 && \ apt-get install -y --no-install-recommends nettle-dev:i386 gcc gcc-multilib \ make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:i386 -ENV CC gcc +ENV CC "gcc -m32" From 262fd8e440ee4fa008dd21a631bd75bd2e9713a9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 6 Jan 2019 20:36:20 +0100 Subject: [PATCH 27/47] Improve python aux script Signed-off-by: DL6ER --- aux/macvendor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/aux/macvendor.py b/aux/macvendor.py index ce2e640b..b255247d 100644 --- a/aux/macvendor.py +++ b/aux/macvendor.py @@ -10,7 +10,10 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -import os, re, urllib.request, sqlite3 +import os +import re +import urllib.request +import sqlite3 import unicodecsv as unicodecsv # Download raw data from Wireshark's website @@ -20,10 +23,10 @@ urllib.request.urlretrieve("https://code.wireshark.org/review/gitweb?p=wireshark print("...done") # Read file into memory and process lines -file = open("manuf.data", "r") +manuf = open("manuf.data", "r") data = [] print("Processing...") -for line in file: +for line in manuf: line = line.strip() # Skip comments and empty lines @@ -52,12 +55,12 @@ for line in file: # Only add long description where available # There are a few vendors for which only the # short description field is used - if(len(desc_long) > 0): + if(desc_long): data.append([mac, desc_long]) else: data.append([mac, desc_short]) print("...done") -file.close() +manuf.close() # Create database database = "macvendor.db" From 0f6c52036fde30d9c2e5a0e50da65a7b8cad2c1e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 01:57:49 +0100 Subject: [PATCH 28/47] Remove obsolete dependency Signed-off-by: DL6ER --- aux/macvendor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aux/macvendor.py b/aux/macvendor.py index b255247d..12fd3fd1 100644 --- a/aux/macvendor.py +++ b/aux/macvendor.py @@ -14,7 +14,6 @@ import os import re import urllib.request import sqlite3 -import unicodecsv as unicodecsv # Download raw data from Wireshark's website # We use the official URL recommended in the header of this file From 0742d4c21a3f98c6e5c08fa0f12638e1f98d179e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 01:58:09 +0100 Subject: [PATCH 29/47] Use simple cast of sqlite3_column_text() instead of using asprintf() Signed-off-by: DL6ER --- networktable.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/networktable.c b/networktable.c index e7a03c25..1c0de87b 100644 --- a/networktable.c +++ b/networktable.c @@ -237,11 +237,7 @@ char* getMACVendor(const char* hwaddr) rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) { - const unsigned char *result = sqlite3_column_text(stmt, 0); - // Need to use sprintf(%s) to convert unsigned char* to - // standard C string literals (which are char*) - if(asprintf(&vendor, "%s", result) < 1) - logg("getMACVendor(%s) - Allocation error 2", hwaddr); + vendor = strdup((char*)sqlite3_column_text(stmt, 0)); } else if(rc == SQLITE_DONE) { @@ -291,21 +287,15 @@ void updateMACVendorRecords() while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { const int id = sqlite3_column_int(stmt, 0); - const unsigned char *hwaddr = sqlite3_column_text(stmt, 1); - // Need to use sprintf(%s) to convert unsigned char* to - // standard C string literals (which are char*) - char *querystr = NULL; - if(asprintf(&querystr, "%s", hwaddr) < 1) - { - logg("updateMACVendorRecords() - Allocation error 1"); - break; - } + char* hwaddr = strdup((char*)sqlite3_column_text(stmt, 1)); // Get vendor for MAC - char* vendor = getMACVendor(querystr); - free(querystr); + char* vendor = getMACVendor(hwaddr); + free(hwaddr); + hwaddr = NULL; // Prepare UPDATE statement + char *querystr = NULL; if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1) { logg("updateMACVendorRecords() - Allocation error 2"); From 078b2ff26d901ce950c6a84fccaba43711fe9a5d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 02:08:52 +0100 Subject: [PATCH 30/47] Free memory when breaking Signed-off-by: DL6ER --- networktable.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/networktable.c b/networktable.c index 1c0de87b..62f186da 100644 --- a/networktable.c +++ b/networktable.c @@ -262,7 +262,7 @@ void updateMACVendorRecords() struct stat st; if(stat(FTLfiles.macvendordb, &st) != 0) { - // File does not exist or MAC address is incomplete + // File does not exist if(debug) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb); return; } @@ -299,6 +299,10 @@ void updateMACVendorRecords() if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1) { logg("updateMACVendorRecords() - Allocation error 2"); + + if(strlen(vendor) > 0) + free(vendor); + break; } @@ -308,6 +312,11 @@ void updateMACVendorRecords() if( rc != SQLITE_OK ){ logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg); sqlite3_free(zErrMsg); + + free(querystr); + if(strlen(vendor) > 0) + free(vendor); + break; } From d01d660bc588137bfb0e5749eba5f51700239ef4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 02:09:10 +0100 Subject: [PATCH 31/47] Separate error messages into two if-statements Signed-off-by: DL6ER --- networktable.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/networktable.c b/networktable.c index 62f186da..e1ad6c4d 100644 --- a/networktable.c +++ b/networktable.c @@ -196,10 +196,16 @@ void parse_arp_cache(void) char* getMACVendor(const char* hwaddr) { struct stat st; - if(stat(FTLfiles.macvendordb, &st) != 0 || strlen(hwaddr) != 17) + if(stat(FTLfiles.macvendordb, &st) != 0) { - // File does not exist or MAC address is incomplete - if(debug) logg("getMACVenor(%s): %s does not exist or MAC invalid (length %lu)", hwaddr, FTLfiles.macvendordb, strlen(hwaddr)); + // File does not exist + if(debug) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb); + return ""; + } + else if(strlen(hwaddr) != 17) + { + // MAC address is incomplete + if(debug) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr)); return ""; } From 9e6696575c75bbbd31cfed892cef6b0bed9306f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 02:10:06 +0100 Subject: [PATCH 32/47] Change code to ensure hostname is always set Signed-off-by: DL6ER --- networktable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networktable.c b/networktable.c index e1ad6c4d..8b5ad368 100644 --- a/networktable.c +++ b/networktable.c @@ -121,7 +121,7 @@ void parse_arp_cache(void) bool clientKnown = clientID >= 0; // Get hostname of this client if the client is known - char *hostname = NULL; + char *hostname = ""; if(clientKnown) { validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); @@ -138,7 +138,7 @@ void parse_arp_cache(void) ip, hwaddr, iface, now, clientKnown ? clients[clientID].lastQuery : 0L, clientKnown ? clients[clientID].numQueriesARP : 0u, - hostname == NULL ? "" : hostname, + hostname, macVendor); if(strlen(macVendor) > 0) free(macVendor); From 486e497d8a634ee780cea5516fa0b5e56513941b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 02:17:22 +0100 Subject: [PATCH 33/47] Review comments Signed-off-by: DL6ER --- FTL.h | 4 ++++ config.c | 2 +- database.c | 20 ++++++++++---------- networktable.c | 7 ++++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/FTL.h b/FTL.h index a66ad50e..a4a1095c 100644 --- a/FTL.h +++ b/FTL.h @@ -222,6 +222,10 @@ typedef struct { // Used to check memory integrity in various structs #define MAGICBYTE 0x57 +// Some magic database constants constants +#define DB_FAILED -2 +#define DB_NODATA -1 + extern logFileNamesStruct files; extern FTLFileNamesStruct FTLfiles; extern countersStruct *counters; diff --git a/config.c b/config.c index 3f5f71b2..95bf42fb 100644 --- a/config.c +++ b/config.c @@ -323,7 +323,7 @@ void read_FTLconf(void) getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb); // PARSE_ARP_CACHE - // defaults to: Yes + // defaults to: true config.parse_arp_cache = true; buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE"); diff --git a/database.c b/database.c index f4815fea..8de6983c 100644 --- a/database.c +++ b/database.c @@ -259,7 +259,7 @@ int db_get_FTL_property(unsigned int ID) if(querystr == NULL || ret < 0) { logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret); - return -2; + return DB_FAILED; } int value = db_query_int(querystr); @@ -295,7 +295,7 @@ int db_query_int(const char* querystr) logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -2; + return DB_FAILED; } rc = sqlite3_step(stmt); @@ -308,14 +308,14 @@ int db_query_int(const char* querystr) else if( rc == SQLITE_DONE ) { // No rows available - result = -1; + result = DB_NODATA; } else { logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -2; + return DB_FAILED; } sqlite3_finalize(stmt); @@ -333,7 +333,7 @@ int number_of_queries_in_DB(void) logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -1; + return DB_FAILED; } rc = sqlite3_step(stmt); @@ -341,7 +341,7 @@ int number_of_queries_in_DB(void) logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -1; + return DB_FAILED; } int result = sqlite3_column_int(stmt, 0); @@ -360,7 +360,7 @@ static sqlite3_int64 last_ID_in_DB(void) logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -1; + return DB_FAILED; } rc = sqlite3_step(stmt); @@ -368,7 +368,7 @@ static sqlite3_int64 last_ID_in_DB(void) logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); dbclose(); check_database(rc); - return -1; + return DB_FAILED; } sqlite3_int64 result = sqlite3_column_int64(stmt, 0); @@ -380,12 +380,12 @@ static sqlite3_int64 last_ID_in_DB(void) int get_number_of_queries_in_DB(void) { - int result = -1; + int result = DB_NODATA; if(!dbopen()) { logg("Failed to open DB in get_number_of_queries_in_DB()"); - return -2; + return DB_FAILED; } result = number_of_queries_in_DB(); diff --git a/networktable.c b/networktable.c index 8b5ad368..e8385ee1 100644 --- a/networktable.c +++ b/networktable.c @@ -18,7 +18,7 @@ char* getMACVendor(const char* hwaddr); bool create_network_table(void) { bool ret; - // Create FTL table in the database (holds properties like database version, etc.) + // Create network table in the database ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \ "ip TEXT NOT NULL, " \ "hwaddr TEXT NOT NULL, " \ @@ -53,6 +53,7 @@ void parse_arp_cache(void) if(!dbopen()) { logg("read_arp_cache() - Failed to open DB"); + fclose(arpfp); return; } @@ -102,7 +103,7 @@ void parse_arp_cache(void) int dbID = db_query_int(querystr); free(querystr); - if(dbID == -2) + if(dbID == DB_FAILED) { // SQLite error break; @@ -129,7 +130,7 @@ void parse_arp_cache(void) } // Device not in database, add new entry - if(dbID == -1) + if(dbID == DB_NODATA) { char* macVendor = getMACVendor(hwaddr); dbquery("INSERT INTO network "\ From 427f8ad8e169c0f545a30cf064133fad14298b7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 18:46:57 +0100 Subject: [PATCH 34/47] Further review comments. Always allocate vendor so we can always free this pointer. Add a callback for the update subroutine. Signed-off-by: DL6ER --- FTL.h | 2 +- aux/macvendor.py | 2 +- networktable.c | 25 +++++++++---------------- request.c | 6 ++++++ routines.h | 1 + 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/FTL.h b/FTL.h index a4a1095c..eda21af6 100644 --- a/FTL.h +++ b/FTL.h @@ -222,7 +222,7 @@ typedef struct { // Used to check memory integrity in various structs #define MAGICBYTE 0x57 -// Some magic database constants constants +// Some magic database constants #define DB_FAILED -2 #define DB_NODATA -1 diff --git a/aux/macvendor.py b/aux/macvendor.py index 12fd3fd1..344dab3e 100644 --- a/aux/macvendor.py +++ b/aux/macvendor.py @@ -29,7 +29,7 @@ for line in manuf: line = line.strip() # Skip comments and empty lines - if line[:1] == "#" or line == "": + if line[1] == "#" or line == "": continue # Remove quotation marks as these might interfere with later INSERT / UPDATE commands diff --git a/networktable.c b/networktable.c index e8385ee1..738e3cce 100644 --- a/networktable.c +++ b/networktable.c @@ -13,7 +13,7 @@ #define ARPCACHE "/proc/net/arp" // Private prototypes -char* getMACVendor(const char* hwaddr); +static char* getMACVendor(const char* hwaddr); bool create_network_table(void) { @@ -194,7 +194,7 @@ void parse_arp_cache(void) dbclose(); } -char* getMACVendor(const char* hwaddr) +static char* getMACVendor(const char* hwaddr) { struct stat st; if(stat(FTLfiles.macvendordb, &st) != 0) @@ -246,16 +246,16 @@ char* getMACVendor(const char* hwaddr) { vendor = strdup((char*)sqlite3_column_text(stmt, 0)); } - else if(rc == SQLITE_DONE) + else { // Not found - vendor = ""; + vendor = strdup(""); } - else + + if(rc != SQLITE_DONE && rc != SQLITE_ROW) { // Error logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); - vendor = ""; } sqlite3_finalize(stmt); @@ -306,10 +306,7 @@ void updateMACVendorRecords() if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1) { logg("updateMACVendorRecords() - Allocation error 2"); - - if(strlen(vendor) > 0) - free(vendor); - + free(vendor); break; } @@ -319,18 +316,14 @@ void updateMACVendorRecords() if( rc != SQLITE_OK ){ logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg); sqlite3_free(zErrMsg); - free(querystr); - if(strlen(vendor) > 0) - free(vendor); - + free(vendor); break; } // Free allocated memory free(querystr); - if(strlen(vendor) > 0) - free(vendor); + free(vendor); } if(rc != SQLITE_DONE) { diff --git a/request.c b/request.c index f83b7c5e..0245cbb9 100644 --- a/request.c +++ b/request.c @@ -168,6 +168,12 @@ void process_request(char *client_message, int *sock) read_regex_from_file(); unlock_shm(); } + else if(command(client_message, ">update-mac-vendor")) + { + processed = true; + logg("Received API request to update vendors in network table"); + updateMACVendorRecords(); + } // Test only at the end if we want to quit or kill // so things can be processed before diff --git a/routines.h b/routines.h index f293c70f..109b92c7 100644 --- a/routines.h +++ b/routines.h @@ -138,3 +138,4 @@ bool check_capabilities(void); // networktable.c bool create_network_table(void); void parse_arp_cache(void); +void updateMACVendorRecords(void); From 054dfce2037140b6b3ddc7888f7b9af6d9f4f288 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Jan 2019 19:22:49 +0100 Subject: [PATCH 35/47] Check for hostname != NULL is obsolete Signed-off-by: DL6ER --- networktable.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/networktable.c b/networktable.c index 738e3cce..bb5a785e 100644 --- a/networktable.c +++ b/networktable.c @@ -141,8 +141,7 @@ void parse_arp_cache(void) clientKnown ? clients[clientID].numQueriesARP : 0u, hostname, macVendor); - if(strlen(macVendor) > 0) - free(macVendor); + free(macVendor); } // Device in database AND client known to Pi-hole else if(clientKnown) @@ -165,7 +164,7 @@ void parse_arp_cache(void) clients[clientID].numQueriesARP = 0; // Store hostname if available - if(hostname != NULL && strlen(hostname) > 0) + if(strlen(hostname) > 0) { // Store host name dbquery("UPDATE network "\ From 787cf113007fd1797a8c03e442d59d59feb4cd67 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 8 Jan 2019 21:07:32 +0100 Subject: [PATCH 36/47] Add generated aux files to .gitignore Signed-off-by: DL6ER --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a952a2bb..005afc23 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ version* /pihole-FTL.conf /pihole-FTL.db /pihole-FTL.log + +# aux files +aux/manuf.data +aux/macvendor.db From 84cdf614735aacefb1349c54089a400f6ebeda88 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 12 Jan 2019 15:10:33 +0100 Subject: [PATCH 37/47] Always allocate strings to prevent subsequent free() to cause a failure of the process. Signed-off-by: DL6ER --- networktable.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/networktable.c b/networktable.c index bb5a785e..09624164 100644 --- a/networktable.c +++ b/networktable.c @@ -200,13 +200,13 @@ static char* getMACVendor(const char* hwaddr) { // File does not exist if(debug) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb); - return ""; + return strdup(""); } else if(strlen(hwaddr) != 17) { // MAC address is incomplete if(debug) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr)); - return ""; + return strdup(""); } sqlite3 *macdb; @@ -214,7 +214,7 @@ static char* getMACVendor(const char* hwaddr) if( rc ){ logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); sqlite3_close(macdb); - return ""; + return strdup(""); } char *querystr = NULL; @@ -226,7 +226,7 @@ static char* getMACVendor(const char* hwaddr) { logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc); sqlite3_close(macdb); - return ""; + return strdup(""); } free(hwaddrshort); @@ -235,7 +235,7 @@ static char* getMACVendor(const char* hwaddr) if( rc ){ logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb)); sqlite3_close(macdb); - return ""; + return strdup(""); } free(querystr); From 2c41ae7b738fafb6e32c8f7cffa1e291af91ca11 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Jan 2019 09:14:35 +0100 Subject: [PATCH 38/47] Offer fine grained debugging options through independentally configurable debug flags. The debugging output of FTL gre considerable over the last year and became overwhelming for users that are not used to it. With this addition, users will be able to switch on only what they want to see. Signed-off-by: DL6ER --- FTL.h | 13 ++++- config.c | 126 +++++++++++++++++++++++++++++++++++++++----- database.c | 12 ++--- dnsmasq_interface.c | 37 ++++++++----- gc.c | 6 +-- grep.c | 2 +- networktable.c | 10 ++-- regex.c | 10 ++-- routines.h | 1 + shmem.c | 17 +++--- socket.c | 6 +-- 11 files changed, 184 insertions(+), 56 deletions(-) diff --git a/FTL.h b/FTL.h index eda21af6..eee6e6ec 100644 --- a/FTL.h +++ b/FTL.h @@ -79,6 +79,17 @@ enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN }; +enum { + DEBUG_DATABASE = (1 << 0), /* 00000000 00000001 */ + DEBUG_NETWORKING = (1 << 1), /* 00000000 00000010 */ + DEBUG_LOCKS = (1 << 2), /* 00000000 00000100 */ + DEBUG_QUERIES = (1 << 3), /* 00000000 00001000 */ + DEBUG_FLAGS = (1 << 4), /* 00000000 00010000 */ + DEBUG_SHMEM = (1 << 5), /* 00000000 00100000 */ + DEBUG_GC = (1 << 6), /* 00000000 01000000 */ + DEBUG_ARP = (1 << 7), /* 00000000 10000000 */ + DEBUG_REGEX = (1 << 8), /* 00000001 00000000 */ +}; // Database table "ftl" enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP }; @@ -147,10 +158,10 @@ typedef struct { unsigned char privacylevel; bool ignore_localhost; unsigned char blockingmode; - bool regex_debugmode; bool analyze_only_A_AAAA; bool DBimport; bool parse_arp_cache; + int16_t debug; } ConfigStruct; // Dynamic structs diff --git a/config.c b/config.c index 95bf42fb..e072d842 100644 --- a/config.c +++ b/config.c @@ -255,19 +255,6 @@ void read_FTLconf(void) break; } - // REGEX_DEBUGMODE - // defaults to: No - config.regex_debugmode = false; - buffer = parse_FTLconf(fp, "REGEX_DEBUGMODE"); - - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.regex_debugmode = true; - - if(config.regex_debugmode) - logg(" REGEX_DEBUGMODE: Active. May increase log file size!"); - else - logg(" REGEX_DEBUGMODE: Inactive"); - // ANALYZE_ONLY_A_AND_AAAA // defaults to: No config.analyze_only_A_AAAA = false; @@ -335,6 +322,9 @@ void read_FTLconf(void) else logg(" PARSE_ARP_CACHE: Inactive"); + // Read DEBUG_... setting from pihole-FTL.conf + read_debuging_settings(fp); + logg("Finished config file parsing"); // Release memory @@ -509,3 +499,113 @@ void get_blocking_mode(FILE *fp) if(opened) fclose(fp); } + +void read_debuging_settings(FILE *fp) +{ + // Set default (no debug instructions set) + config.debug = 0; + + // See if we got a file handle, if not we have to open + // the config file ourselves + bool opened = false; + if(fp == NULL) + { + if((fp = fopen(FTLfiles.conf, "r")) == NULL) + // Return silently if there is no config file available + return; + opened = true; + } + + // DEBUG_DATABASE + // defaults to: false + char* buffer = parse_FTLconf(fp, "DEBUG_DATABASE"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_DATABASE; + + // DEBUG_NETWORKING + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_NETWORKING"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_NETWORKING; + + // DEBUG_LOCKS + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_LOCKS"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_LOCKS; + + // DEBUG_QUERIES + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_QUERIES"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_QUERIES; + + // DEBUG_FLAGS + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_FLAGS"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_FLAGS; + + // DEBUG_SHMEM + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_SHMEM"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_SHMEM; + + // DEBUG_GC + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_GC"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_GC; + + // DEBUG_ARP + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_ARP"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_ARP; + + // DEBUG_REGEX or REGEX_DEBUGMODE (legacy config option) + // defaults to: false + buffer = parse_FTLconf(fp, "DEBUG_REGEX"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_REGEX; + buffer = parse_FTLconf(fp, "REGEX_DEBUGMODE"); + if(buffer != NULL && strcasecmp(buffer, "true") == 0) + config.debug |= DEBUG_REGEX; + + if(config.debug) + { + logg("*********************"); + logg("* Debugging enabled *"); + if(config.debug & DEBUG_DATABASE) + logg("* DEBUG_DATABASE *"); + if(config.debug & DEBUG_NETWORKING) + logg("* DEBUG_NETWORKING *"); + if(config.debug & DEBUG_LOCKS) + logg("* DEBUG_LOCKS *"); + if(config.debug & DEBUG_QUERIES) + logg("* DEBUG_QUERIES *"); + if(config.debug & DEBUG_FLAGS) + logg("* DEBUG_FLAGS *"); + if(config.debug & DEBUG_SHMEM) + logg("* DEBUG_SHMEM *"); + if(config.debug & DEBUG_GC) + logg("* DEBUG_GC *"); + if(config.debug & DEBUG_ARP) + logg("* DEBUG_ARP *"); + if(config.debug & DEBUG_REGEX) + logg("* DEBUG_REGEX *"); + logg("*********************"); + } + + // Have to close the config file if we opened it + if(opened) + { + fclose(fp); + + // Release memory only when we opened the file + // Otherwise, it may still be needed outside of + // this function (initial config parsing) + release_config_memory(); + } +} diff --git a/database.c b/database.c index 8de6983c..5240d0da 100644 --- a/database.c +++ b/database.c @@ -88,7 +88,7 @@ bool dbquery(const char *format, ...) return false; } - if(debug) logg("dbquery: %s", query); + if(config.debug & DEBUG_DATABASE) logg("dbquery: %s", query); int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg); @@ -403,7 +403,7 @@ void save_to_DB(void) return; // Start database timer - if(debug) timer_start(DATABASE_WRITE_TIMER); + if(config.debug & DEBUG_DATABASE) timer_start(DATABASE_WRITE_TIMER); // Open database if(!dbopen()) @@ -554,7 +554,7 @@ void save_to_DB(void) // Close database dbclose(); - if(debug) + if(config.debug & DEBUG_DATABASE) { logg("Notice: Queries stored in DB: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID); if(saved_error > 0) @@ -585,7 +585,7 @@ void delete_old_queries_in_DB(void) int affected = sqlite3_changes(db); // Print final message only if there is a difference - if(debug || affected) + if((config.debug & DEBUG_DATABASE) || affected) logg("Notice: Database size is %.2f MB, deleted %i rows", get_db_filesize(), affected); // Close database @@ -666,7 +666,7 @@ void read_data_from_DB(void) return; } // Log DB query string in debug mode - if(debug) logg(rstr); + if(config.debug & DEBUG_DATABASE) logg(rstr); // Prepare SQLite3 statement sqlite3_stmt* stmt; @@ -691,7 +691,7 @@ void read_data_from_DB(void) } if(queryTimeStamp > now) { - if(debug) logg("DB warn: Skipping query logged in the future (%i)", queryTimeStamp); + if(config.debug & DEBUG_DATABASE) logg("DB warn: Skipping query logged in the future (%i)", queryTimeStamp); continue; } diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index b9c8c65e..96aa7fc5 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -61,7 +61,7 @@ void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char else { // Return early to avoid accessing querytypedata out of bounds - if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); + if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping unknown query type: %s (%i)", types, id); unlock_shm(); return; } @@ -69,7 +69,7 @@ void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char // Skip AAAA queries if user doesn't want to have them analyzed if(!config.analyze_AAAA && querytype == TYPE_AAAA) { - if(debug) logg("Not analyzing AAAA query"); + if(config.debug & DEBUG_QUERIES) logg("Not analyzing AAAA query"); unlock_shm(); return; } @@ -112,7 +112,7 @@ void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char // Log new query if in debug mode char *proto = (type == UDP) ? "UDP" : "TCP"; - if(debug) logg("**** new %s %s \"%s\" from %s (ID %i, %s:%i)", proto, types, domain, client, id, file, line); + if(config.debug & DEBUG_QUERIES) logg("**** new %s %s \"%s\" from %s (ID %i, %s:%i)", proto, types, domain, client, id, file, line); // Update counters int timeidx = findOverTimeID(overTimetimestamp); @@ -125,7 +125,7 @@ void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA) { // Don't process this query further here, we already counted it - if(debug) logg("Notice: Skipping new query: %s (%i)", types, id); + if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping new query: %s (%i)", types, id); free(domain); free(domainbuffer); free(client); @@ -261,7 +261,7 @@ void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int i strtolower(forward); // Debug logging - if(debug) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, forward, id, file, line); + if(config.debug & DEBUG_QUERIES) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, forward, id, file, line); // Save status and forwardID in corresponding query identified by dnsmasq's ID int i = findQueryID(id); @@ -374,6 +374,9 @@ void FTL_dnsmasq_reload(void) // Reread regex.list free_regex(); read_regex_from_file(); + + // Reread pihole-FTL.conf to see which debugging flags are set + read_debuging_settings(NULL); } void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line) @@ -401,7 +404,7 @@ void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, else if(flags & F_NEG) answer = "(NODATA)"; - if(debug) + if(config.debug & DEBUG_QUERIES) { logg("**** got reply %s is %s (ID %i, %s:%i)", name, answer, id, file, line); print_flags(flags); @@ -416,7 +419,7 @@ void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, if(i < 0) { // This may happen e.g. if the original query was "pi.hole" - if(debug) logg("FTL_reply(): Query %i has not been found", id); + if(config.debug & DEBUG_QUERIES) logg("FTL_reply(): Query %i has not been found", id); unlock_shm(); return; } @@ -617,8 +620,11 @@ void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg free(domain); // Debug logging - if(debug) logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line); - if(debug) print_flags(flags); + if(config.debug & DEBUG_QUERIES) + { + logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line); + print_flags(flags); + } // Get response time struct timeval response; @@ -752,7 +758,7 @@ void _FTL_dnssec(int status, int id, const char* file, const int line) } // Debug logging - if(debug) + if(config.debug & DEBUG_QUERIES) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); @@ -793,7 +799,7 @@ void _FTL_header_ADbit(unsigned char header4, unsigned int rcode, int id, const return; } - if(debug) + if(config.debug & DEBUG_QUERIES) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); @@ -829,6 +835,11 @@ void print_flags(unsigned int flags) { // Debug function, listing resolver flags in clear text // e.g. "Flags: F_FORWARD F_NEG F_IPV6" + + // Only print flags if corresponding debugging flag is set + if(!(config.debug & DEBUG_FLAGS)) + return; + unsigned int i; char *flagstr = calloc(256,sizeof(char)); for(i = 0; i < sizeof(flags)*8; i++) @@ -1000,7 +1011,7 @@ void _FTL_forwarding_failed(struct server *server, const char* file, const int l strtolower(forward); int forwardID = findForwardID(forward, false); - if(debug) logg("**** forwarding to %s (ID %i, %s:%i) failed", dest, forwardID, file, line); + if(config.debug & DEBUG_QUERIES) logg("**** forwarding to %s (ID %i, %s:%i) failed", dest, forwardID, file, line); forwarded[forwardID].failed++; @@ -1123,7 +1134,7 @@ static void block_single_domain(char *domain) regexlistname = files.regexlist; add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, SRC_REGEX); - if(debug) logg("Added %s to cache", domain); + if(config.debug & DEBUG_QUERIES) logg("Added %s to cache", domain); return; } diff --git a/gc.c b/gc.c index b5d3f915..342e4846 100644 --- a/gc.c +++ b/gc.c @@ -37,11 +37,11 @@ void *GC_thread(void *val) // Get minimum time stamp to keep time_t mintime = time(NULL) - config.maxlogage; - if(debug) timer_start(GC_TIMER); + if(config.debug & DEBUG_GC) timer_start(GC_TIMER); long int i; int removed = 0; - if(debug) logg("GC starting, mintime: %u %s", mintime, ctime(&mintime)); + if(config.debug & DEBUG_GC) logg("GC starting, mintime: %u %s", mintime, ctime(&mintime)); // Process all queries for(i=0; i < counters->queries; i++) @@ -159,7 +159,7 @@ void *GC_thread(void *val) // Zero out remaining memory (marked as "F" in the above example) memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries)); - if(debug) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); + if(config.debug & DEBUG_GC) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); // Release thread lock unlock_shm(); diff --git a/grep.c b/grep.c index 95551e5b..a640f906 100644 --- a/grep.c +++ b/grep.c @@ -142,5 +142,5 @@ void check_blocking_status(void) message = "disabled"; } - if(debug) logg("Blocking status is %s", message); + logg("Blocking status is %s", message); } diff --git a/networktable.c b/networktable.c index 09624164..67f564cb 100644 --- a/networktable.c +++ b/networktable.c @@ -58,7 +58,7 @@ void parse_arp_cache(void) } // Start ARP timer - if(debug) timer_start(ARP_TIMER); + if(config.debug & DEBUG_ARP) timer_start(ARP_TIMER); // Prepare buffers char * linebuffer = NULL; @@ -184,7 +184,7 @@ void parse_arp_cache(void) dbquery("COMMIT"); // Debug logging - if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); + if(config.debug & DEBUG_ARP) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); // Close file handle fclose(arpfp); @@ -199,13 +199,13 @@ static char* getMACVendor(const char* hwaddr) if(stat(FTLfiles.macvendordb, &st) != 0) { // File does not exist - if(debug) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb); + if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb); return strdup(""); } else if(strlen(hwaddr) != 17) { // MAC address is incomplete - if(debug) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr)); + if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr)); return strdup(""); } @@ -269,7 +269,7 @@ void updateMACVendorRecords() if(stat(FTLfiles.macvendordb, &st) != 0) { // File does not exist - if(debug) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb); + if(config.debug & DEBUG_ARP) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb); return; } diff --git a/regex.c b/regex.c index 5b62385e..93988740 100644 --- a/regex.c +++ b/regex.c @@ -40,7 +40,7 @@ static bool init_regex(const char *regexin, int index) } // Store compiled regex string in buffer if in regex debug mode - if(config.regex_debugmode) + if(config.debug & DEBUG_REGEX) { regexbuffer[index] = strdup(regexin); } @@ -98,8 +98,8 @@ bool match_regex(char *input) matched = true; // Print match message when in regex debug mode - if(config.regex_debugmode) - logg("DEBUG: Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input); + if(config.debug & DEBUG_REGEX) + logg("Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input); break; } else if (errcode != REG_NOMATCH) @@ -134,7 +134,7 @@ void free_regex(void) regfree(®ex[index]); // Also free buffered regex strings if in regex debug mode - if(config.regex_debugmode) + if(config.debug & DEBUG_REGEX) { free(regexbuffer[index]); regexbuffer[index] = NULL; @@ -245,7 +245,7 @@ void read_regex_from_file(void) regexconfigured = calloc(num_regex, sizeof(bool)); // Buffer strings if in regex debug mode - if(config.regex_debugmode) + if(config.debug & DEBUG_REGEX) regexbuffer = calloc(num_regex, sizeof(char*)); // Search through file diff --git a/routines.h b/routines.h index 109b92c7..9bbdd627 100644 --- a/routines.h +++ b/routines.h @@ -73,6 +73,7 @@ void getLogFilePath(void); void read_FTLconf(void); void get_privacy_level(FILE *fp); void get_blocking_mode(FILE *fp); +void read_debuging_settings(FILE *fp); // gc.c void *GC_thread(void *val); diff --git a/shmem.c b/shmem.c index 924dc0ca..dd94ae8f 100644 --- a/shmem.c +++ b/shmem.c @@ -60,7 +60,7 @@ unsigned long long addstr(const char *str) return 0; } - if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); + if(config.debug & DEBUG_SHMEM) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); // Reserve additional memory if necessary size_t required_size = next_pos + len + 1; @@ -164,11 +164,13 @@ void _lock_shm(const char* function, const int line, const char * file) { // Signal that FTL is waiting for a lock shmLock->waitingForLock = true; - if(debug) logg("Waiting for lock in %s() (%s:%i)", function, file, line); + if(config.debug & DEBUG_LOCKS) + logg("Waiting for lock in %s() (%s:%i)", function, file, line); int result = pthread_mutex_lock(&shmLock->lock); - if(debug) logg("Obtained lock for %s() (%s:%i)", function, file, line); + if(config.debug & DEBUG_LOCKS) + logg("Obtained lock for %s() (%s:%i)", function, file, line); // Turn off the waiting for lock signal to notify everyone who was // deferring to FTL that they can jump in the lock queue. @@ -187,7 +189,8 @@ void _lock_shm(const char* function, const int line, const char * file) { void _unlock_shm(const char* function, const int line, const char * file) { int result = pthread_mutex_unlock(&shmLock->lock); - if(debug) logg("Removed lock in %s() (%s:%i)", function, file, line); + if(config.debug & DEBUG_LOCKS) + logg("Removed lock in %s() (%s:%i)", function, file, line); if(result != 0) logg("Failed to unlock SHM lock: %s", strerror(result)); @@ -289,7 +292,8 @@ void destroy_shmem(void) SharedMemory create_shm(char *name, size_t size) { - if(debug) logg("Creating shared memory with name \"%s\" and size %zu", name, size); + if(config.debug & DEBUG_SHMEM) + logg("Creating shared memory with name \"%s\" and size %zu", name, size); SharedMemory sharedMemory = { .name = name, @@ -397,7 +401,8 @@ void *enlarge_shmem_struct(char type) } bool realloc_shm(SharedMemory *sharedMemory, size_t size) { - logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); + if(config.debug & DEBUG_SHMEM) + logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); int result = munmap(sharedMemory->ptr, sharedMemory->size); if(result != 0) diff --git a/socket.c b/socket.c index 74470c5e..a7354156 100644 --- a/socket.c +++ b/socket.c @@ -552,14 +552,14 @@ bool ipv6_available(void) { iface[addr->sa_family == AF_INET6 ? 1 : 0]++; - // For now unused debug statement - // logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4"); + if(config.debug & DEBUG_NETWORKING) + logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4"); } } freeifaddrs(allInterfaces); } - if(debug) + if(config.debug & DEBUG_NETWORKING) { logg("Found %i IPv4 and %i IPv6 capable interfaces", iface[0], iface[1]); } From fc17373f7de341b3cc636256b04868f8fd1150b0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Jan 2019 10:45:05 +0100 Subject: [PATCH 39/47] Print all available debug settings together with "YES" or "NO" when at least one flag is set Signed-off-by: DL6ER --- config.c | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/config.c b/config.c index e072d842..7d9673ba 100644 --- a/config.c +++ b/config.c @@ -575,27 +575,18 @@ void read_debuging_settings(FILE *fp) if(config.debug) { - logg("*********************"); - logg("* Debugging enabled *"); - if(config.debug & DEBUG_DATABASE) - logg("* DEBUG_DATABASE *"); - if(config.debug & DEBUG_NETWORKING) - logg("* DEBUG_NETWORKING *"); - if(config.debug & DEBUG_LOCKS) - logg("* DEBUG_LOCKS *"); - if(config.debug & DEBUG_QUERIES) - logg("* DEBUG_QUERIES *"); - if(config.debug & DEBUG_FLAGS) - logg("* DEBUG_FLAGS *"); - if(config.debug & DEBUG_SHMEM) - logg("* DEBUG_SHMEM *"); - if(config.debug & DEBUG_GC) - logg("* DEBUG_GC *"); - if(config.debug & DEBUG_ARP) - logg("* DEBUG_ARP *"); - if(config.debug & DEBUG_REGEX) - logg("* DEBUG_REGEX *"); - logg("*********************"); + logg("************************"); + logg("* Debugging enabled *"); + logg("* DEBUG_DATABASE %s *", (config.debug & DEBUG_DATABASE)? "YES":"NO "); + logg("* DEBUG_NETWORKING %s *", (config.debug & DEBUG_NETWORKING)? "YES":"NO "); + logg("* DEBUG_LOCKS %s *", (config.debug & DEBUG_LOCKS)? "YES":"NO "); + logg("* DEBUG_QUERIES %s *", (config.debug & DEBUG_QUERIES)? "YES":"NO "); + logg("* DEBUG_FLAGS %s *", (config.debug & DEBUG_FLAGS)? "YES":"NO "); + logg("* DEBUG_SHMEM %s *", (config.debug & DEBUG_SHMEM)? "YES":"NO "); + logg("* DEBUG_GC %s *", (config.debug & DEBUG_GC)? "YES":"NO "); + logg("* DEBUG_ARP %s *", (config.debug & DEBUG_ARP)? "YES":"NO "); + logg("* DEBUG_REGEX %s *", (config.debug & DEBUG_REGEX)? "YES":"NO "); + logg("************************"); } // Have to close the config file if we opened it From f518fa1c080ad29df609c3ac41390fde1c0d83e6 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 19 Jan 2019 10:57:36 -0800 Subject: [PATCH 40/47] Add an "/FTL-settings" shared memory block It currently contains the version of shared memory that FTL is exposing. The current version of shared memory is 1. Whenever a change is made to structures stored in shared memory, or the layout of shared memory, the version should be incremented (like how the database version is incremented when the database changes). This version number will be used by the API to verify it is using the same version of shared memory as FTL. Signed-off-by: Mcat12 --- FTL.h | 4 ++++ shmem.c | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/FTL.h b/FTL.h index eda21af6..333e9d79 100644 --- a/FTL.h +++ b/FTL.h @@ -216,6 +216,10 @@ typedef struct { char **domains; } whitelistStruct; +typedef struct { + int version; +} ShmSettings; + // Prepare timers, used mainly for debugging purposes #define NUMTIMERS LAST_TIMER diff --git a/shmem.c b/shmem.c index 924dc0ca..3fa355b0 100644 --- a/shmem.c +++ b/shmem.c @@ -11,6 +11,9 @@ #include "FTL.h" #include "shmem.h" +/// The version of shared memory used +#define SHARED_MEMORY_VERSION 1 + /// The name of the shared memory. Use this when connecting to the shared memory. #define SHARED_LOCK_NAME "/FTL-lock" #define SHARED_STRINGS_NAME "/FTL-strings" @@ -20,6 +23,7 @@ #define SHARED_QUERIES_NAME "/FTL-queries" #define SHARED_FORWARDED_NAME "/FTL-forwarded" #define SHARED_OVERTIME_NAME "/FTL-overTime" +#define SHARED_SETTINGS_NAME "/FTL-settings" #define SHARED_OVERTIMECLIENT_PREFIX "/FTL-client-" /// The pointer in shared memory to the shared string buffer @@ -31,6 +35,7 @@ static SharedMemory shm_clients = { 0 }; static SharedMemory shm_queries = { 0 }; static SharedMemory shm_forwarded = { 0 }; static SharedMemory shm_overTime = { 0 }; +static SharedMemory shm_settings = { 0 }; static SharedMemory *shm_overTimeClients = NULL; static int overTimeClientCount = 0; @@ -264,6 +269,14 @@ bool init_shmem(void) overTime = (overTimeDataStruct*)shm_overTime.ptr; counters->overTime_MAX = pagesize; + /****************************** shared settings struct ******************************/ + // Try to create shared memory object + shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings)); + if(shm_settings.ptr == NULL) + return false; + ShmSettings *settings = (ShmSettings*)shm_settings.ptr; + settings->version = SHARED_MEMORY_VERSION; + return true; } @@ -280,6 +293,7 @@ void destroy_shmem(void) delete_shm(&shm_queries); delete_shm(&shm_forwarded); delete_shm(&shm_overTime); + delete_shm(&shm_settings); for(int i = 0; i < overTimeClientCount; i++) { delete_shm(&shm_overTimeClients[i]); From 09522edba71b8b7f56e17716343f5af552acd10a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 19 Jan 2019 20:25:18 +0100 Subject: [PATCH 41/47] Remove extra debug variable Signed-off-by: DL6ER --- FTL.h | 1 - args.c | 2 -- log.c | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/FTL.h b/FTL.h index eee6e6ec..8828d517 100644 --- a/FTL.h +++ b/FTL.h @@ -259,7 +259,6 @@ extern char ** setupVarsArray; extern int setupVarsElements; extern bool initialscan; -extern bool debug; extern bool threadwritelock; extern bool threadreadlock; extern unsigned char blockingstatus; diff --git a/args.c b/args.c index 13b464a4..ae04a5ff 100644 --- a/args.c +++ b/args.c @@ -11,7 +11,6 @@ #include "FTL.h" #include "version.h" -bool debug = false; bool daemonmode = true; bool travis = false; int argc_dnsmasq = 0; @@ -34,7 +33,6 @@ void parse_args(int argc, char* argv[]) if(strcmp(argv[i], "d") == 0 || strcmp(argv[i], "debug") == 0) { - debug = true; daemonmode = false; ok = true; diff --git a/log.c b/log.c index 83e73e11..5452abe3 100644 --- a/log.c +++ b/log.c @@ -92,7 +92,7 @@ void logg(const char *format, ...) va_end(args); fputc('\n',logfile); } - else if(debug) + else if(!daemonmode) { printf("!!! WARNING: Writing to FTL\'s log file failed!\n"); syslog(LOG_ERR, "Writing to FTL\'s log file failed!"); From c269bdfe414b330ab31726e4854f3515270f036d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 19 Jan 2019 20:28:36 +0100 Subject: [PATCH 42/47] We need the debug variable internally in parse_args(). However, as we need it only in there, we mark it static. Signed-off-by: DL6ER --- args.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/args.c b/args.c index ae04a5ff..262ccf6e 100644 --- a/args.c +++ b/args.c @@ -11,6 +11,7 @@ #include "FTL.h" #include "version.h" +static bool debug = false; bool daemonmode = true; bool travis = false; int argc_dnsmasq = 0; @@ -33,6 +34,7 @@ void parse_args(int argc, char* argv[]) if(strcmp(argv[i], "d") == 0 || strcmp(argv[i], "debug") == 0) { + debug = true; daemonmode = false; ok = true; From d688ae3de66e2bfe4c13695613a858417fed1fd4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Jan 2019 09:42:01 +0100 Subject: [PATCH 43/47] Trim any whitespace characters around config keys Signed-off-by: DL6ER --- config.c | 11 +++++++---- routines.h | 2 ++ setupVars.c | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/config.c b/config.c index 7d9673ba..ae0a2641 100644 --- a/config.c +++ b/config.c @@ -389,9 +389,6 @@ static char *parse_FTLconf(FILE *fp, const char * key) errno = 0; while(getline(&conflinebuffer, &size, fp) != -1) { - // Strip (possible) newline - conflinebuffer[strcspn(conflinebuffer, "\n")] = '\0'; - // Skip comment lines if(conflinebuffer[0] == '#' || conflinebuffer[0] == ';') continue; @@ -402,7 +399,13 @@ static char *parse_FTLconf(FILE *fp, const char * key) // otherwise: key found free(keystr); - return (find_equals(conflinebuffer) + 1); + // Note: value is still a pointer into the conflinebuffer + // its memory will get released in release_config_memory() + char* value = find_equals(conflinebuffer) + 1; + // Trim whitespace at beginning and end, this function + // modifies the string inplace + trim_whitespace(value); + return value; } if(errno == ENOMEM) diff --git a/routines.h b/routines.h index 9bbdd627..4b6c570d 100644 --- a/routines.h +++ b/routines.h @@ -66,7 +66,9 @@ bool getSetupVarsBool(char * input); void parse_args(int argc, char* argv[]); +// setupVars.c char* find_equals(const char* s); +void trim_whitespace(char *string); // config.c void getLogFilePath(void); diff --git a/setupVars.c b/setupVars.c index 17602e6e..7570aeb3 100644 --- a/setupVars.c +++ b/setupVars.c @@ -38,6 +38,26 @@ char* find_equals(const char* s) return (char*)s; } +void trim_whitespace(char *string) +{ + // isspace(char*) man page: + // checks for white-space characters. In the "C" and "POSIX" + // locales, these are: space, form-feed ('\f'), newline ('\n'), + // carriage return ('\r'), horizontal tab ('\t'), and vertical tab + // ('\v'). + char *original = string, *modified = string; + // Trim any whitespace characters (see above) at the beginning by increasing the pointer address + while (isspace((unsigned char)*original)) + original++; + // Copy the content of original into modified as long as there is something in original + while ((*modified = *original++) != '\0') + modified++; + // Trim any whitespace characters (see above) at the end of the string by overwriting it + // with the zero character (marking the end of a C string) + while (modified > string && isspace((unsigned char)*--modified)) + *modified = '\0'; +} + // This will hold the read string // in memory and will serve the space // we will point to in the rest of the From 9d0346a4241b0e8462dfe2010294195e2cd46965 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 22 Jan 2019 17:23:34 +0100 Subject: [PATCH 44/47] Ensure lastdbindex in properly set after reading in queries from the database Signed-off-by: DL6ER --- database.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/database.c b/database.c index 5240d0da..d63c4c05 100644 --- a/database.c +++ b/database.c @@ -439,7 +439,7 @@ void save_to_DB(void) int total = 0, blocked = 0; time_t currenttimestamp = time(NULL); time_t newlasttimestamp = 0; - for(i = lastdbindex; i < counters->queries; i++) + for(i = MAX(0, lastdbindex); i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); if(queries[i].db != 0) @@ -837,6 +837,10 @@ void read_data_from_DB(void) } logg("Imported %i queries from the long-term database", counters->queries); + // Update lastdbindex so that the next call to save_to_DB() + // skips the queries that we just imported from the database + lastdbindex = counters->queries; + if( rc != SQLITE_DONE ){ logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); dbclose(); From 608458edfda6ef669b8768a3d10c19db399fecdd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 22 Jan 2019 17:26:25 +0100 Subject: [PATCH 45/47] Only include sqlite3.h in those files that need it. This should improve the compiling time slightly Signed-off-by: DL6ER --- FTL.h | 4 +--- api.c | 2 ++ database.c | 1 + networktable.c | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/FTL.h b/FTL.h index 356a223a..9bcf7cba 100644 --- a/FTL.h +++ b/FTL.h @@ -36,8 +36,6 @@ #include // syslog #include -// SQLite -#include "sqlite3.h" // tolower() #include // Unix socket @@ -174,7 +172,7 @@ typedef struct { int domainID; int clientID; int forwardID; - sqlite3_int64 db; + int64_t db; int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here bool complete; unsigned char privacylevel; diff --git a/api.c b/api.c index 621e73f4..16ee405d 100644 --- a/api.c +++ b/api.c @@ -11,6 +11,8 @@ #include "FTL.h" #include "api.h" #include "version.h" +// needed for sqlite3_libversion() +#include "sqlite3.h" #define min(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) diff --git a/database.c b/database.c index d63c4c05..cd20e871 100644 --- a/database.c +++ b/database.c @@ -10,6 +10,7 @@ #include "FTL.h" #include "shmem.h" +#include "sqlite3.h" static sqlite3 *db; bool database = false; diff --git a/networktable.c b/networktable.c index 67f564cb..d6cf0b86 100644 --- a/networktable.c +++ b/networktable.c @@ -10,6 +10,7 @@ #include "FTL.h" #include "shmem.h" +#include "sqlite3.h" #define ARPCACHE "/proc/net/arp" // Private prototypes From 247e8ce5b0c4c1c250f66e6345b1bc1460dfab8f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 24 Jan 2019 11:08:34 +0100 Subject: [PATCH 46/47] Micro optimizations for adding blocking domains. Most important change is that we check if we need to rehash at the end of hosts file parsing (this is rather unlikely) Signed-off-by: DL6ER --- dnsmasq_interface.c | 47 ++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 96aa7fc5..0887bd57 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -17,7 +17,7 @@ void print_flags(unsigned int flags); void save_reply_type(unsigned int flags, int queryID, struct timeval response); unsigned long converttimeval(struct timeval time); -static void block_single_domain(char *domain); +static void block_single_domain_regex(char *domain); static void detect_blocked_IP(unsigned short flags, char* answer, int queryID); static void query_externally_blocked(int i); static int findQueryID(int id); @@ -200,7 +200,7 @@ void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char if(match_regex(domainbuffer) && !in_whitelist(domainbuffer)) { // We have to block this domain - block_single_domain(domainbuffer); + block_single_domain_regex(domainbuffer); domains[domainID].regexmatch = REGEX_BLOCKED; } else @@ -1074,14 +1074,15 @@ void rehash(int size); // This routine adds one domain to the resolver's cache. Depending on the configured blocking mode it may create // a single entry valid for IPv4 & IPv6 or two entries one for IPv4 and one for IPv6. // When IPv6 is not available on the machine, we do not add IPv6 cache entries (likewise for IPv4) -static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *addr6, bool has_IPv4, bool has_IPv6, - char *domain, struct crec **rhash, int hashsz, unsigned int index) +static int add_blocked_domain(struct all_addr *addr4, struct all_addr *addr6, bool has_IPv4, bool has_IPv6, + char *domain, int len, struct crec **rhash, int hashsz, unsigned int index) { int name_count = 0; struct crec *cache4,*cache6; - // Add IPv4 record + // Add IPv4 record, allocate enough space for cache entry including arbitrary domain name length + // (the domain name is stored at the end of struct crec) if(has_IPv4 && - (cache4 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME))) + (cache4 = malloc(sizeof(struct crec) + len+1-SMALLDNAME))) { strcpy(cache4->name.sname, domain); cache4->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV4; @@ -1111,7 +1112,7 @@ static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *add } // Add IPv6 record only if we respond with a non-NULL IP address to blocked domains if(has_IPv6 && (config.blockingmode == MODE_IP || config.blockingmode == MODE_IP_NODATA_AAAA) && - (cache6 = malloc(sizeof(struct crec) + strlen(domain)+1-SMALLDNAME))) + (cache6 = malloc(sizeof(struct crec) + len+1-SMALLDNAME))) { strcpy(cache6->name.sname, domain); cache6->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV6; @@ -1120,11 +1121,15 @@ static int add_blocked_domain_cache(struct all_addr *addr4, struct all_addr *add add_hosts_entry(cache6, addr6, IN6ADDRSZ, index, rhash, hashsz); name_count++; } + + // Return 1 if only one cache slot was allocated (IPv4) or 2 if two slots were allocated (IPv4 + IPv6) return name_count; } // Add a single domain to resolver's cache. This respects the configured blocking mode -static void block_single_domain(char *domain) +// Note: This routine is meant for adding a single domain at a time. It should not be +// invoked for batch processing +static void block_single_domain_regex(char *domain) { struct all_addr addr4 = {{{ 0 }}}, addr6 = {{{ 0 }}}; bool has_IPv4 = false, has_IPv6 = false; @@ -1132,7 +1137,7 @@ static void block_single_domain(char *domain) // Get IPv4/v6 addresses for blocking depending on user configures blocking mode prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6); regexlistname = files.regexlist; - add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, NULL, 0, SRC_REGEX); + add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, strlen(domain), NULL, 0, SRC_REGEX); if(config.debug & DEBUG_QUERIES) logg("Added %s to cache", domain); @@ -1160,7 +1165,8 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s // Get IPv4/v6 addresses for blocking depending on user configured blocking mode prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6); - // If we have neither a valid IPv4 nor a valid IPv6, then we cannot add any entries here + // If we have neither a valid IPv4 nor a valid IPv6 but the user asked for + // blocking modes MODE_IP or MODE_IP_NODATA_AAAA then we cannot add any entries here if(!has_IPv4 && !has_IPv6) { logg("ERROR: found neither a valid IPV4_ADDRESS nor IPV6_ADDRESS in setupVars.conf"); @@ -1181,7 +1187,8 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s // Check for spaces or tabs // If found, then this list is still in HOSTS format and we - // don't analyze it here. + // don't analyze it here. We only check the first line for + // efficiency reasons (strstr() is slow) if(firstline && (strstr(domain, " ") != NULL || strstr(domain, "\t") != NULL)) { @@ -1193,12 +1200,16 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s firstline = false; // Skip empty lines - if(strlen(domain) == 0) + int len = strlen(domain); + if(len == 0) continue; // Strip newline character at the end of line we just read - if(domain[strlen(domain)-1] == '\n') - domain[strlen(domain)-1] = '\0'; + if(domain[len-1] == '\n') + { + domain[len-1] = '\0'; + len -= 1; + } // As of here we assume the entry to be valid // Rehash every 1000 valid names @@ -1208,11 +1219,17 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s cache_size = name_count; } - name_count += add_blocked_domain_cache(&addr4, &addr6, has_IPv4, has_IPv6, domain, rhash, hashsz, index); + // Add domain + name_count += add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, len, rhash, hashsz, index); + // Count added domain added++; } + // Rehash after having read all entries + if(rhash) + rehash(name_count); + // Free allocated memory if(buffer != NULL) { From 52c2148c78bc3314c37a8299382580c96144778f Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 27 Jan 2019 19:40:23 -0800 Subject: [PATCH 47/47] Fix counters->clients getting incremented twice due to v4.2 merge In release/v4.2, the `counters->clients++` call was moved to be after `newOverTimeClient`. When that change was merged to development, the first call was accidentally not deleted, causing it to happen twice (before and after `newOverTimeClient`). Signed-off-by: Mcat12 --- datastructure.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/datastructure.c b/datastructure.c index 14eb5595..aa745e40 100644 --- a/datastructure.c +++ b/datastructure.c @@ -227,8 +227,6 @@ int findClientID(const char *client, bool count) // No query seen so far clients[clientID].lastQuery = 0; clients[clientID].numQueriesARP = 0; - // Increase counter by one - counters->clients++; // Create new overTime client data newOverTimeClient(clientID);