From 1c2e44d519182fe3e3681cd46620407daede1f4d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 09:19:31 +0200 Subject: [PATCH 1/5] Use client->firstSeen for new cients for the netDB if available Signed-off-by: DL6ER --- src/database/network-table.c | 43 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 754d1e93..ec464f2e 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -510,8 +510,8 @@ static int add_netDB_network_address(sqlite3 *db, const int network_id, const ch } // Insert a new record into the network table -static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time_t lastQuery, - unsigned int numQueriesARP, const char *macVendor) +static int insert_netDB_device(sqlite3 *db, const char *hwaddr, const time_t firstSeen, const time_t lastQuery, + const unsigned int numQueriesARP, const char *macVendor) { // Return early if database is known to be broken if(FTLDBerror()) @@ -526,29 +526,29 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if(rc != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\") - SQL error prepare (%i): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); checkFTLDBrc(rc); return rc; } log_debug(DEBUG_DATABASE, "dbquery: \"%s\" with arguments ?1-?5 = (\"%s\", %lu, %lu, %u, \"%s\")", - querystr, hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor); + querystr, hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor); // Bind hwaddr to prepared statement (1st argument) if((rc = sqlite3_bind_text(query_stmt, 1, hwaddr, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\"): Failed to bind hwaddr (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; } - // Bind now to prepared statement (2nd argument) - if((rc = sqlite3_bind_int(query_stmt, 2, now)) != SQLITE_OK) + // Bind firstSeen to prepared statement (2nd argument) + if((rc = sqlite3_bind_int(query_stmt, 2, firstSeen)) != SQLITE_OK) { - log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind now (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind firstSeen (error %d): %s", + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -558,7 +558,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 3, lastQuery)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind lastQuery (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -568,7 +568,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 4, numQueriesARP)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind numQueriesARP (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -578,7 +578,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_text(query_stmt, 5, macVendor, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind macVendor (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -588,7 +588,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_step(query_stmt)) != SQLITE_DONE) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to step (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -598,7 +598,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_finalize(query_stmt)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to finalize (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -901,9 +901,10 @@ static bool add_FTL_clients_to_network_table(sqlite3 *db, const enum arp_status // Add new device to database const time_t lastQuery = client->lastQuery; + const time_t firstSeen = client->firstSeen; const unsigned int numQueriesARP = client->numQueriesARP; unlock_shm(); - insert_netDB_device(db, hwaddr, now, lastQuery, numQueriesARP, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueriesARP, macVendor); lock_shm(); // Reacquire client pointer (if may have changed when unlocking above) @@ -1370,10 +1371,12 @@ void parse_neighbor_cache(sqlite3* db) lock_shm(); int clientID = findClientID(ip, false, false); - // Get hostname of this client if the client is known + // Set default values for a new device, may be updated + // below if the client is known to pihole-FTL char *hostname = NULL; bool client_valid = false; time_t lastQuery = 0; + time_t firstSeen = now; unsigned int numQueries = 0; // This client is known (by its IP address) to pihole-FTL if @@ -1384,14 +1387,20 @@ void parse_neighbor_cache(sqlite3* db) if(!client) continue; + // Client is known to Pi-hole, update properties + // with their real values client_valid = true; hostname = strdup(getstr(client->namepos)); + firstSeen = client->firstSeen; lastQuery = client->lastQuery; numQueries = client->numQueriesARP; client_status[clientID] = CLIENT_ARP_COMPLETE; } else { + // Client is not known to Pi-hole, create a + // mock-device with the default values set above + // and an empty hostname hostname = strdup(""); } unlock_shm(); @@ -1413,7 +1422,7 @@ void parse_neighbor_cache(sqlite3* db) hwaddr, ip, hostname, macVendor); // Create new record (INSERT) - insert_netDB_device(db, hwaddr, now, lastQuery, numQueries, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); clientsData *client = getClient(clientID, true); From 73634762ed04e5b6101de465a3baee35dad40d46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 09:36:54 +0200 Subject: [PATCH 2/5] Remove deprecated asprintf() calls from network-table code Signed-off-by: DL6ER --- src/database/common.c | 92 +++++++++++++++++++++++ src/database/common.h | 2 + src/database/network-table.c | 142 +++++++++++++---------------------- 3 files changed, 145 insertions(+), 91 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index c29d889f..dee1019c 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -758,6 +758,98 @@ int db_query_int(sqlite3 *db, const char* querystr) return result; } +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_arg: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_int(stmt, 1, arg)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_str: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, arg, -1, SQLITE_STATIC)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + double db_query_double(sqlite3 *db, const char* querystr) { log_debug(DEBUG_DATABASE, "dbquery: \"%s\"", querystr); diff --git a/src/database/common.h b/src/database/common.h index 5dd0c875..b5673f91 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -45,6 +45,8 @@ void _dbclose(sqlite3 **db, const char *func, const int line, const char *file); void piholeFTLDB_reopen(void); int db_query_int(sqlite3 *db, const char *querystr); +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg); +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg); double db_query_double(sqlite3 *db, const char *querystr); int db_query_int_from_until(sqlite3 *db, const char* querystr, const double from, const double until); int db_query_int_from_until_type(sqlite3 *db, const char* querystr, const double from, const double until, const int type); diff --git a/src/database/network-table.c b/src/database/network-table.c index ec464f2e..f33279a1 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -210,23 +210,13 @@ static int find_device_by_recent_ip(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return -1; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT network_id FROM network_addresses " - "WHERE ip = \'%s\' AND " - "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " - "ORDER BY lastSeen DESC LIMIT 1;", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_ip(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT network_id FROM network_addresses " + "WHERE ip = ?1 AND " + "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " + "ORDER BY lastSeen DESC LIMIT 1;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - querystr = NULL; + int network_id = db_query_int_str(db, querystr, ipaddr); if(network_id == DB_FAILED) { @@ -252,20 +242,10 @@ static int find_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'ip-%s\';", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = concat('ip-',?1)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Try to find device by hardware address @@ -275,20 +255,10 @@ static int find_device_by_hwaddr(sqlite3 *db, const char hwaddr[]) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'%s\' COLLATE NOCASE;", hwaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_hwaddr(\"%s\"): %i", - hwaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = ?1 COLLATE NOCASE;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, hwaddr); } // Try to find device by RECENT mock hardware address (generated from IP address) @@ -298,24 +268,12 @@ static int find_recent_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT id FROM network WHERE " - "hwaddr = \'ip-%s\' AND " - "firstSeen > (cast(strftime('%%s', 'now') as int)-3600);", - ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE " + "hwaddr = concat('ip-',?1) AND " + "firstSeen > (cast(strftime('%%s', 'now') as int)-3600)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Store hostname of device identified by dbID @@ -1137,30 +1095,9 @@ static bool add_local_interfaces_to_network_table(sqlite3 *db, time_t now, unsig int lastQuery = 0, firstSeen = now, numQueries = 0; if(mockID >= 0) { - char *querystr = NULL; - if(asprintf(&querystr, "SELECT lastQuery from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - lastQuery = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT firstSeen from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - firstSeen = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT numQueries from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - numQueries = db_query_int(db, querystr); - free(querystr); + lastQuery = db_query_int_int(db, "SELECT lastQuery from network where id = ?1", mockID); + firstSeen = db_query_int_int(db, "SELECT firstSeen from network where id = ?1", mockID); + numQueries = db_query_int_int(db, "SELECT numQueries from network where id = ?1", mockID); } // Add new device to database @@ -1800,33 +1737,56 @@ void updateMACVendorRecords(sqlite3 *db) // Get vendor for MAC char *vendor = getMACVendor(hwaddr); + + // Free allocated memory free(hwaddr); hwaddr = NULL; - // Prepare UPDATE statement - char *updatestr = NULL; - if(asprintf(&updatestr, "UPDATE network SET macVendor = \'%s\' WHERE id = %i", vendor, id) < 1) + // Prepare statement + sqlite3_stmt *stmt2 = NULL; + const char *updatestr = "UPDATE network SET macVendor = ?1 WHERE id = ?2"; + rc = sqlite3_prepare_v2(db, updatestr, -1, &stmt2, NULL); + if(rc != SQLITE_OK) { - log_err("updateMACVendorRecords() - Allocation error"); + log_err("updateMACVendorRecords() - SQL error prepare \"%s\": %s", updatestr, sqlite3_errstr(rc)); + checkFTLDBrc(rc); free(vendor); break; } - // Execute prepared statement - char *zErrMsg = NULL; - rc = sqlite3_exec(db, updatestr, NULL, NULL, &zErrMsg); - if(rc != SQLITE_OK) + // Bind vendor to prepared statement + if((rc = sqlite3_bind_text(stmt2, 1, vendor, -1, SQLITE_STATIC)) != SQLITE_OK) { - log_err("updateMACVendorRecords() - SQL exec error: \"%s\": %s", updatestr, zErrMsg); + log_err("updateMACVendorRecords() - Failed to bind vendor: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Bind id to prepared statement + if((rc = sqlite3_bind_int(stmt2, 2, id)) != SQLITE_OK) + { + log_err("updateMACVendorRecords() - Failed to bind id: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Execute statement + rc = sqlite3_step(stmt2); + if(rc != SQLITE_DONE) + { + log_err("updateMACVendorRecords() - SQL error step: %s", sqlite3_errstr(rc)); checkFTLDBrc(rc); - sqlite3_free(zErrMsg); - free(updatestr); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); free(vendor); break; } // Free allocated memory - free(updatestr); free(vendor); } if(rc != SQLITE_DONE) From 04f255927a825848433f23056a404980d56be46d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 14:08:50 +0200 Subject: [PATCH 3/5] Update firstSeen on importing clients from the database. Before, their firstSeen was the timestamp of their creation, i.e., the time FTL imported the database. Now the timestamp is set to the first query we import for this client from the database Signed-off-by: DL6ER --- src/database/aliasclients.c | 3 ++- src/database/network-table.c | 14 +++++++++----- src/database/query-table.c | 2 +- src/datastructure.c | 5 +++-- src/datastructure.h | 6 +++--- src/dnsmasq_interface.c | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index 7e8fddeb..e258f5c6 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -109,6 +109,7 @@ bool import_aliasclients(sqlite3 *db) // Loop until no further data is available int imported = 0; + const double now = double_time(); while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { // Check if we ran into an error @@ -132,7 +133,7 @@ bool import_aliasclients(sqlite3 *db) } // Try to open existing client - const int clientID = findClientID(aliasclient_str, false, true); + const int clientID = findClientID(aliasclient_str, false, true, now); clientsData *client = getClient(clientID, true); if(client == NULL) diff --git a/src/database/network-table.c b/src/database/network-table.c index f33279a1..c92fd70a 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1189,7 +1189,7 @@ void parse_neighbor_cache(sqlite3* db) char *linebuffer = NULL; size_t linebuffersize = 0u; unsigned int entries = 0u, additional_entries = 0u; - time_t now = time(NULL); + const time_t now = time(NULL); // Start ARP timer if(config.debug.arp.v.b) @@ -1269,8 +1269,11 @@ void parse_neighbor_cache(sqlite3* db) { // This line is incomplete, remember this to skip // mock-device creation after ARP processing + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); unlock_shm(); if(clientID >= 0 && clientID < clients) client_status[clientID] = CLIENT_ARP_INCOMPLETE; @@ -1303,10 +1306,11 @@ void parse_neighbor_cache(sqlite3* db) // If we reach this point, we can check if this client // is known to pihole-FTL - // false = do not create a new record if the client is - // unknown (only DNS requesting clients do this) + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); // Set default values for a new device, may be updated // below if the client is known to pihole-FTL diff --git a/src/database/query-table.c b/src/database/query-table.c index 8aa16790..1b661e77 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1148,7 +1148,7 @@ void DB_read_queries(void) // Obtain IDs only after filtering which queries we want to keep const int timeidx = getOverTimeID(queryTimeStamp); const int domainID = findDomainID(domainname, true); - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, queryTimeStamp); // Set index for this query const int queryIndex = counters->queries; diff --git a/src/datastructure.c b/src/datastructure.c index 28a7be4b..72376a9f 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -239,7 +239,8 @@ static int get_next_free_clientID(void) return counters->clients; } -int _findClientID(const char *clientIP, const bool count, const bool aliasclient, int line, const char *func, const char *file) +int _findClientID(const char *clientIP, const bool count, const bool aliasclient, + const double now, int line, const char *func, const char *file) { // Compare content of client against known client IP addresses for(int clientID=0; clientID < counters->clients; clientID++) @@ -308,7 +309,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient // some time after adding a client to ensure we pick up possible // group configuration though hostname, MAC address or interface client->reread_groups = 0u; - client->firstSeen = time(NULL); + client->firstSeen = now; // Interface is not yet known client->ifacepos = 0; // Set all MAC address bytes to zero diff --git a/src/datastructure.h b/src/datastructure.h index 43b2a5c1..b2b66772 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -93,7 +93,7 @@ typedef struct { size_t ippos; size_t namepos; size_t ifacepos; - time_t firstSeen; + double firstSeen; double lastQuery; } clientsData; @@ -124,8 +124,8 @@ int findQueryID(const int id); int _findUpstreamID(const char *upstream, const in_port_t port, int line, const char *func, const char *file); #define findDomainID(domain, count) _findDomainID(domain, count, __LINE__, __FUNCTION__, __FILE__) int _findDomainID(const char *domain, const bool count, int line, const char *func, const char *file); -#define findClientID(client, count, aliasclient) _findClientID(client, count, aliasclient, __LINE__, __FUNCTION__, __FILE__) -int _findClientID(const char *client, const bool count, const bool aliasclient, int line, const char *func, const char *file); +#define findClientID(client, count, aliasclient, now) _findClientID(client, count, aliasclient, now, __LINE__, __FUNCTION__, __FILE__) +int _findClientID(const char *client, const bool count, const bool aliasclient, const double now, int line, const char *func, const char *file); #define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__) int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d5506eaf..fa9ca8d1 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -647,7 +647,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, const int queryID = counters->queries; // Find client IP - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, querytimestamp); // Get client pointer clientsData* client = getClient(clientID, true); From 4d71d88e7f704adfb7e3483e825fd90405a7c23f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 2 Aug 2024 14:20:06 +0200 Subject: [PATCH 4/5] Add exception for the case where the device is not yet in the database: Use total count of queries as the number of queries for the new device instead of the special ARP cache counter to add also the number of queries in the DNS history imported from the long-term database. Signed-off-by: DL6ER --- src/database/network-table.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index c92fd70a..9ca8fdf5 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1318,7 +1318,7 @@ void parse_neighbor_cache(sqlite3* db) bool client_valid = false; time_t lastQuery = 0; time_t firstSeen = now; - unsigned int numQueries = 0; + unsigned int numQueries = 0, totalQueries = 0; // This client is known (by its IP address) to pihole-FTL if // findClientID() returned a non-negative index @@ -1335,6 +1335,7 @@ void parse_neighbor_cache(sqlite3* db) firstSeen = client->firstSeen; lastQuery = client->lastQuery; numQueries = client->numQueriesARP; + totalQueries = client->count; client_status[clientID] = CLIENT_ARP_COMPLETE; } else @@ -1356,6 +1357,15 @@ void parse_neighbor_cache(sqlite3* db) // and the ARP entry just came a bit delayed (reported by at least one user) dbID = find_recent_device_by_mock_hwaddr(db, ip); + // Exception for the case where the device is + // not yet in the database: Use total count of + // queries as the number of queries for the new + // device instead of the special ARP cache + // counter to add also the number of queries in + // the DNS history imported from the long-term + // database + numQueries = totalQueries; + if(dbID == DB_NODATA) { // Device not known AND no recent mock-device found ---> create new device record From 1fb9df910be4efd07f7e178ca4181d077f7250fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 3 Aug 2024 14:36:10 +0200 Subject: [PATCH 5/5] 127.0.0.1 is not in the ARP table and handled specially Signed-off-by: DL6ER --- src/database/network-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 9ca8fdf5..686e1ed4 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -860,9 +860,9 @@ static bool add_FTL_clients_to_network_table(sqlite3 *db, const enum arp_status // Add new device to database const time_t lastQuery = client->lastQuery; const time_t firstSeen = client->firstSeen; - const unsigned int numQueriesARP = client->numQueriesARP; + const unsigned int numQueries = client->count; unlock_shm(); - insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueriesARP, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); // Reacquire client pointer (if may have changed when unlocking above)