From 7a0ce6b2a230536804db65023835e92394a4898d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Sep 2017 20:44:29 +0200 Subject: [PATCH 1/6] Add overTime/clients data structure --- FTL.h | 3 +++ parser.c | 40 ++++++++++++++++++++++++++++++++++++++-- routines.h | 1 + 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/FTL.h b/FTL.h index f07dbe4a..776296f4 100644 --- a/FTL.h +++ b/FTL.h @@ -168,6 +168,8 @@ typedef struct { int forwardnum; int *forwarddata; int *querytypedata; + int clientnum; + int *clientdata; } overTimeDataStruct; typedef struct { @@ -178,6 +180,7 @@ typedef struct { int forwardedips; int forwardednames; int forwarddata; + int clientdata; int querytypedata; } memoryStruct; diff --git a/parser.c b/parser.c index bebe5205..2aa0bb41 100644 --- a/parser.c +++ b/parser.c @@ -272,6 +272,8 @@ void process_pihole_log(int file) overTime[timeidx].forwardnum = 0; overTime[timeidx].forwarddata = NULL; overTime[timeidx].querytypedata = calloc(2, sizeof(int)); + overTime[timeidx].clientnum = 0; + overTime[timeidx].clientdata = NULL; memory.querytypedata += 2*sizeof(int); counters.overTime++; @@ -589,6 +591,27 @@ void process_pihole_log(int file) break; } + // Determine if there is enough space for saving the current + // clientID in the overTime data structure, allocate space otherwise + validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); + if(overTime[timeidx].clientnum <= clientID) + { + // Reallocate more space for clientdata + overTime[timeidx].clientdata = realloc(overTime[timeidx].clientdata, (clientID+1)*sizeof(*overTime[timeidx].clientdata)); + // Initialize new data fields with zeroes + for(i = overTime[timeidx].clientnum; i <= clientID; i++) + { + overTime[timeidx].clientdata[i] = 0; + memory.clientdata++; + } + // Update counter + overTime[timeidx].clientnum = clientID + 1; + } + + // Update overTime data structure with the new client + validate_access_oTcl(timeidx, clientID, __LINE__, __FUNCTION__, __FILE__); + overTime[timeidx].clientdata[clientID]++; + // Free allocated memory free(client); free(domain); @@ -644,11 +667,13 @@ void process_pihole_log(int file) overTime[timeidx].forwardnum = 0; overTime[timeidx].forwarddata = NULL; overTime[timeidx].querytypedata = calloc(2, sizeof(int)); + overTime[timeidx].clientnum = 0; + overTime[timeidx].clientdata = NULL; memory.querytypedata += 2*sizeof(int); counters.overTime++; } // Determine if there is enough space for saving the current - // forwardID in the overTime data structure -allocate space otherwise + // forwardID in the overTime data structure, allocate space otherwise validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); if(overTime[timeidx].forwardnum <= forwardID) { @@ -1020,7 +1045,18 @@ void validate_access_oTfd(int timeidx, int pos, int line, const char * function, if(pos >= limit || pos < 0) { logg("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - logg("FATAL ERROR: Trying to access overTime.forwardata[%i], but maximum is %i", pos, limit); + logg("FATAL ERROR: Trying to access overTime.forwarddata[%i], but maximum is %i", pos, limit); + logg(" found in %s() (line %i) in %s", function, line, file); + } +} + +void validate_access_oTcl(int timeidx, int pos, int line, const char * function, const char * file) +{ + int limit = overTime[timeidx].clientnum; + if(pos >= limit || pos < 0) + { + logg("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); + logg("FATAL ERROR: Trying to access overTime.clientdata[%i], but maximum is %i", pos, limit); logg(" found in %s() (line %i) in %s", function, line, file); } } diff --git a/routines.h b/routines.h index f2f03983..fba6a5de 100644 --- a/routines.h +++ b/routines.h @@ -31,6 +31,7 @@ void process_pihole_log(int file); void *pihole_log_thread(void *val); void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file); void validate_access_oTfd(int timeidx, int pos, int line, const char * function, const char * file); +void validate_access_oTcl(int timeidx, int pos, int line, const char * function, const char * file); void reresolveHostnames(void); void pihole_log_flushed(bool message); From 199fccf643f47eb5c095046caae4d0ed598ccb5a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Sep 2017 20:50:21 +0200 Subject: [PATCH 2/6] Add ">ClientsoverTime" request --- request.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/request.c b/request.c index 5119d80e..80cfaa56 100644 --- a/request.c +++ b/request.c @@ -30,6 +30,7 @@ void getClientID(int *sock); void getQueryTypesOverTime(int *sock); void getVersion(int *sock); void getDBstats(int *sock); +void getClientsOverTime(int *sock); void process_request(char *client_message, int *sock) { @@ -113,6 +114,11 @@ void process_request(char *client_message, int *sock) processed = true; getDBstats(sock); } + else if(command(client_message, ">ClientsoverTime")) + { + processed = true; + getClientsOverTime(sock); + } // End of queryable commands if(processed) @@ -1024,3 +1030,65 @@ void getDBstats(int *sock) if(debugclients) logg("Sent DB info to client, ID: %i", *sock); } + +void getClientsOverTime(int *sock) +{ + char server_message[SOCKETBUFFERLEN]; + int i, sendit = -1; + + for(i = 0; i < counters.overTime; i++) + { + validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); + if((overTime[i].total > 0 || overTime[i].blocked > 0)) + { + sendit = i; + break; + } + } + if(sendit < 0) + return; + + for(i = sendit; i < counters.overTime; i++) + { + double percentage; + + validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); + sprintf(server_message, "%i", overTime[i].timestamp); + + int j, allclients = 0; + + // Compute forwardedsum used for later normalization + for(j = 0; j < overTime[i].clientnum; j++) + { + allclients += overTime[i].clientdata[j]; + } + + // Loop over forward destinations to generate output to be sent to the client + for(j = 0; j < counters.clients; j++) + { + int thisclient = 0; + + if(j < overTime[i].clientnum) + { + // This client entry does already exist at this timestamp + // -> use counter of requests sent to this destination + thisclient = overTime[i].clientdata[j]; + } + + // Avoid floating point exceptions + if(allclients > 0 && overTime[i].total > 0 && thisclient > 0) + { + percentage = 1e2 * thisclient / allclients; + } + else + { + percentage = 0.0; + } + + sprintf(server_message + strlen(server_message), " %.2f", percentage); + } + + sprintf(server_message + strlen(server_message), " %.2f\n", percentage); + swrite(server_message, *sock); + } +} From a14167fcf9210e852299771f03d37c67f49b2ca7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Sep 2017 13:03:25 +0200 Subject: [PATCH 3/6] Return absolute instead of relative data in client over time requests --- request.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/request.c b/request.c index 80cfaa56..5fffbf6d 100644 --- a/request.c +++ b/request.c @@ -31,6 +31,7 @@ void getQueryTypesOverTime(int *sock); void getVersion(int *sock); void getDBstats(int *sock); void getClientsOverTime(int *sock); +void getClientNames(int *sock); void process_request(char *client_message, int *sock) { @@ -119,6 +120,11 @@ void process_request(char *client_message, int *sock) processed = true; getClientsOverTime(sock); } + else if(command(client_message, ">client-names")) + { + processed = true; + getClientNames(sock); + } // End of queryable commands if(processed) @@ -1050,20 +1056,11 @@ void getClientsOverTime(int *sock) for(i = sendit; i < counters.overTime; i++) { - double percentage; - validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); sprintf(server_message, "%i", overTime[i].timestamp); - int j, allclients = 0; - - // Compute forwardedsum used for later normalization - for(j = 0; j < overTime[i].clientnum; j++) - { - allclients += overTime[i].clientdata[j]; - } - // Loop over forward destinations to generate output to be sent to the client + int j; for(j = 0; j < counters.clients; j++) { int thisclient = 0; @@ -1075,20 +1072,24 @@ void getClientsOverTime(int *sock) thisclient = overTime[i].clientdata[j]; } - // Avoid floating point exceptions - if(allclients > 0 && overTime[i].total > 0 && thisclient > 0) - { - percentage = 1e2 * thisclient / allclients; - } - else - { - percentage = 0.0; - } - - sprintf(server_message + strlen(server_message), " %.2f", percentage); + sprintf(server_message + strlen(server_message), " %i", thisclient); } - sprintf(server_message + strlen(server_message), " %.2f\n", percentage); + sprintf(server_message + strlen(server_message), "\n"); + swrite(server_message, *sock); + } +} + +void getClientNames(int *sock) +{ + char server_message[SOCKETBUFFERLEN]; + int i; + + // Loop over clients to generate output to be sent to the client + for(i = 0; i < counters.clients; i++) + { + validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + sprintf(server_message,"%i %s %s\n", i, clients[i].ip, clients[i].name); swrite(server_message, *sock); } } From 42c0dcc1e69348abf7910cc5d3f5c5cd0711e690 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 29 Sep 2017 22:34:13 +0200 Subject: [PATCH 4/6] Apply API_EXCLUDE_CLIENTS filtering to algorithm that returns clients over time data Signed-off-by: DL6ER --- request.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/request.c b/request.c index 5fffbf6d..947ef137 100644 --- a/request.c +++ b/request.c @@ -449,7 +449,7 @@ void getTopClients(char *client_message, int *sock) // Sort temporary array qsort(temparray, counters.clients, sizeof(int[2]), cmpasc); - // Get domains which the user doesn't want to see + // Get clients which the user doesn't want to see char * excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); if(excludeclients != NULL) { @@ -1054,6 +1054,30 @@ void getClientsOverTime(int *sock) if(sendit < 0) return; + // Get clients which the user doesn't want to see + char * excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); + // Array of clients to be skipped in the output + // if skipclient[i] == true then this client should be hidden from + // returned data. We initialize it with false + bool skipclient[counters.clients]; + memset(skipclient, false, counters.clients*sizeof(bool)); + + if(excludeclients != NULL) + { + getSetupVarsArray(excludeclients); + + for(i=0; i < counters.clients; i++) + { + validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + // Check if this client should be skipped + if(insetupVarsArray(clients[i].ip) || insetupVarsArray(clients[i].name)) + { + skipclient[i] = true; + } + } + } + + // Main return loop for(i = sendit; i < counters.overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); @@ -1065,6 +1089,9 @@ void getClientsOverTime(int *sock) { int thisclient = 0; + if(skipclient[j]) + continue; + if(j < overTime[i].clientnum) { // This client entry does already exist at this timestamp @@ -1078,6 +1105,9 @@ void getClientsOverTime(int *sock) sprintf(server_message + strlen(server_message), "\n"); swrite(server_message, *sock); } + + if(excludeclients != NULL) + clearSetupVarsArray(); } void getClientNames(int *sock) @@ -1085,11 +1115,37 @@ void getClientNames(int *sock) char server_message[SOCKETBUFFERLEN]; int i; + // Get clients which the user doesn't want to see + char * excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); + // Array of clients to be skipped in the output + // if skipclient[i] == true then this client should be hidden from + // returned data. We initialize it with false + bool skipclient[counters.clients]; + memset(skipclient, false, counters.clients*sizeof(bool)); + + if(excludeclients != NULL) + { + getSetupVarsArray(excludeclients); + + for(i=0; i < counters.clients; i++) + { + validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + // Check if this client should be skipped + + } + } + // Loop over clients to generate output to be sent to the client for(i = 0; i < counters.clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + if(insetupVarsArray(clients[i].ip) || insetupVarsArray(clients[i].name)) + continue; + sprintf(server_message,"%i %s %s\n", i, clients[i].ip, clients[i].name); swrite(server_message, *sock); } + + if(excludeclients != NULL) + clearSetupVarsArray(); } From 88e5e2941ebc652fe8449f14c80592c3ba0f1306 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 29 Sep 2017 22:50:05 +0200 Subject: [PATCH 5/6] Merge branch 'development' into new/clientsovertime Signed-off-by: DL6ER --- request.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/request.c b/request.c index 947ef137..a25581fe 100644 --- a/request.c +++ b/request.c @@ -126,13 +126,6 @@ void process_request(char *client_message, int *sock) getClientNames(sock); } - // End of queryable commands - if(processed) - { - // Send EOM - seom(server_message, *sock); - } - // Test only at the end if we want to quit or kill // so things can be processed before if(command(client_message, ">quit") || command(client_message, EOT)) @@ -153,9 +146,16 @@ void process_request(char *client_message, int *sock) if(!processed) { - sprintf(server_message,"unknown command: %s\n",client_message); + sprintf(server_message,"unknown command: %s",client_message); swrite(server_message, *sock); } + + // End of queryable commands + if(*sock != 0) + { + // Send EOM + seom(server_message, *sock); + } } bool command(char *client_message, const char* cmd) From 2baad5c1a4346bf9b8bb9e2cb54687cb343acc8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Sep 2017 10:37:31 +0200 Subject: [PATCH 6/6] Fix that API is expecting a different format of FTL's answer in FTL itself for the sake of convenience (the output format is now similar to ">forward-names") Signed-off-by: DL6ER --- request.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.c b/request.c index a25581fe..a7ed6a06 100644 --- a/request.c +++ b/request.c @@ -1142,7 +1142,7 @@ void getClientNames(int *sock) if(insetupVarsArray(clients[i].ip) || insetupVarsArray(clients[i].name)) continue; - sprintf(server_message,"%i %s %s\n", i, clients[i].ip, clients[i].name); + sprintf(server_message,"%i %i %s %s\n", i, clients[i].count, clients[i].ip, clients[i].name); swrite(server_message, *sock); }