From c7803df60474a59a2d09afe69a76821e9beff0d7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Jan 2023 21:19:11 +0100 Subject: [PATCH] Add /api/stats/upstreams and /api/stats/database/upstreams documentation Signed-off-by: DL6ER --- src/api/docs/content/specs/history.yaml | 7 ++ src/api/docs/content/specs/main.yaml | 6 ++ src/api/docs/content/specs/stats.yaml | 119 +++++++++++++++++++++++- src/api/stats.c | 39 ++------ src/api/stats_database.c | 56 ++++++++++- src/database/query-table.c | 7 +- src/dnsmasq_interface.c | 26 ++++-- src/overTime.c | 3 + src/webserver/http-common.c | 2 +- test/api/checkAPI.py | 2 +- 10 files changed, 220 insertions(+), 47 deletions(-) diff --git a/src/api/docs/content/specs/history.yaml b/src/api/docs/content/specs/history.yaml index a61cbd88..9c72e9b7 100644 --- a/src/api/docs/content/specs/history.yaml +++ b/src/api/docs/content/specs/history.yaml @@ -16,6 +16,13 @@ components: application/json: schema: $ref: 'history.yaml#/components/schemas/total_history' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: 'common.yaml#/components/errors/unauthorized' + clients: get: summary: Get per-client activity graph data diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index d4a0a642..446774cb 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -54,6 +54,12 @@ paths: /stats/summary: $ref: 'stats.yaml#/components/paths/summary' + /stats/upstreams: + $ref: 'stats.yaml#/components/paths/upstreams' + + /stats/database/upstreams: + $ref: 'stats.yaml#/components/paths/database_upstreams' + /history: $ref: 'history.yaml#/components/paths/history' diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index c95444c5..84811640 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -15,10 +15,56 @@ components: content: application/json: schema: - allOf: - - $ref: 'stats.yaml#/components/schemas/queries' - - $ref: 'ftl.yaml#/components/schemas/system' - - $ref: 'ftl.yaml#/components/schemas/ftl' + $ref: 'stats.yaml#/components/schemas/queries' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: 'common.yaml#/components/errors/unauthorized' + + upstreams: + get: + summary: Get metrics about Pi-hole's upstream destinations + tags: + - Metrics + operationId: "get_metrics_upstreams" + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: 'stats.yaml#/components/schemas/upstreams' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: 'common.yaml#/components/errors/unauthorized' + + database_upstreams: + get: + summary: Get metrics about Pi-hole's upstream destinations (database) + tags: + - Metrics + operationId: "get_metrics_upstreams_database" + parameters: + - $ref: 'stats.yaml#/components/parameters/database/from' + - $ref: 'stats.yaml#/components/parameters/database/until' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: 'stats.yaml#/components/schemas/upstreams' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: 'common.yaml#/components/errors/unauthorized' schemas: queries: @@ -143,3 +189,68 @@ components: type: integer description: text responses example: 16 + upstreams: + type: object + properties: + upstreams: + type: array + description: Array of upstream destinations + items: + type: object + properties: + ip: + type: string + description: Upstream destination's IP address (can be either IPv4 or IPv6) + nullable: true + example: "127.0.0.1" + name: + type: string + description: Upstream destination's hostname (if available) + nullable: true + example: "localhost" + port: + type: integer + description: Upstream destination's destination port (-1 if not applicable, e.g., for the local chace) + example: 53 + count: + type: integer + description: Number of queries this upstream destination has been used for + example: 65445 + statistics: + type: object + properties: + response: + type: string + description: Average response time of this upstream destination in seconds (0 if not applicable) + example: 0.0254856 + variance: + type: string + description: Standard deviation of the average response time (0 if not applicable) + example: 0.02058 + forwarded_queries: + type: integer + description: Number of forwarded queries + example: 6379 + total_queries: + type: integer + description: Total number of queries + example: 29160 + + parameters: + database: + from: + in: query + description: Unix timestamp from when the data should be requested + name: from + schema: + type: integer + required: true + example: 1672580025 + until: + in: query + description: Unix timestamp from when the data should be requested + name: until + schema: + type: integer + required: true + example: 1672688280 \ No newline at end of file diff --git a/src/api/stats.c b/src/api/stats.c index 02c1b917..7efb4027 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -108,27 +108,6 @@ int api_stats_summary(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); - - // Get system object - cJSON *system = JSON_NEW_OBJECT(); - ret = get_system_obj(api, system); - if(ret != 0) - { - unlock_shm(); - return ret; - } - JSON_ADD_ITEM_TO_OBJECT(json, "system", system); - - // Get FTL object - cJSON *ftl = JSON_NEW_OBJECT(); - ret = get_ftl_obj(api, ftl, true); - if(ret != 0) - { - unlock_shm(); - return ret; - } - JSON_ADD_ITEM_TO_OBJECT(json, "ftl", ftl); - JSON_SEND_OBJECT_UNLOCK(json); } @@ -419,14 +398,14 @@ int api_stats_top_clients(struct ftl_conn *api) int api_stats_upstreams(struct ftl_conn *api) { - const int forwarded = get_forwarded_count(); - unsigned int totalcount = 0; - int temparray[forwarded][2]; - // Verify requesting client is allowed to see this ressource if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) return send_json_unauthorized(api); + const int forwarded = get_forwarded_count(); + unsigned int totalcount = 0; + int temparray[forwarded][2]; + // Lock shared memory lock_shm(); @@ -455,7 +434,7 @@ int api_stats_upstreams(struct ftl_conn *api) { int count = 0; const char* ip, *name; - unsigned short port = 53; + int port = -1; double responsetime = 0.0, uncertainty = 0.0; if(i == -2) @@ -511,12 +490,14 @@ int api_stats_upstreams(struct ftl_conn *api) if(count > 0 || i < 0) { cJSON *upstream = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(upstream, "name", name); JSON_REF_STR_IN_OBJECT(upstream, "ip", ip); + JSON_REF_STR_IN_OBJECT(upstream, "name", name); JSON_ADD_NUMBER_TO_OBJECT(upstream, "port", port); JSON_ADD_NUMBER_TO_OBJECT(upstream, "count", count); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "responsetime", responsetime); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "uncertainty", uncertainty); + cJSON *statistics = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty); + JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics); JSON_ADD_ITEM_TO_ARRAY(upstreams, upstream); } } diff --git a/src/api/stats_database.c b/src/api/stats_database.c index 9821cb70..1fa2f677 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -21,6 +21,10 @@ int api_stats_database_overTime_history(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + double from = 0, until = 0; const int interval = 600; if(api->request->query_string != NULL) @@ -149,6 +153,10 @@ int api_stats_database_overTime_history(struct ftl_conn *api) int api_stats_database_top_items(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + unsigned int show = 10; double from = 0.0, until = 0.0; @@ -322,6 +330,10 @@ int api_stats_database_top_items(struct ftl_conn *api) int api_stats_database_summary(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + double from = 0, until = 0; if(api->request->query_string != NULL) { @@ -392,6 +404,10 @@ int api_stats_database_summary(struct ftl_conn *api) int api_stats_database_overTime_clients(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + double from = 0, until = 0; const int interval = 600; if(api->request->query_string != NULL) @@ -607,6 +623,10 @@ int api_stats_database_overTime_clients(struct ftl_conn *api) int api_stats_database_query_types(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + double from = 0, until = 0; if(api->request->query_string != NULL) { @@ -655,6 +675,10 @@ int api_stats_database_query_types(struct ftl_conn *api) int api_stats_database_upstreams(struct ftl_conn *api) { + // Verify requesting client is allowed to see this ressource + if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) + return send_json_unauthorized(api); + double from = 0, until = 0; if(api->request->query_string != NULL) { @@ -749,10 +773,26 @@ int api_stats_database_upstreams(struct ftl_conn *api) { const char* upstream = (char*)sqlite3_column_text(stmt, 0); const int count = sqlite3_column_int(stmt, 1); + cJSON *item = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(item, "ip", upstream); + unsigned int port = -1; + char buffer[512] = { 0 }; + if(sscanf(upstream, "%511[^#]#%u", buffer, &port) == 2) + { + buffer[sizeof(buffer)-1] = '\0'; + JSON_COPY_STR_TO_OBJECT(item, "ip", buffer) + } + else + JSON_COPY_STR_TO_OBJECT(item, "ip", upstream); JSON_REF_STR_IN_OBJECT(item, "name", ""); + JSON_ADD_NUMBER_TO_OBJECT(item, "port", port); JSON_ADD_NUMBER_TO_OBJECT(item, "count", count); + + cJSON *statistics = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", 0); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", 0); + JSON_ADD_ITEM_TO_OBJECT(item, "statistics", statistics); + JSON_ADD_ITEM_TO_ARRAY(upstreams, item); forwarded_queries += count; } @@ -763,15 +803,25 @@ int api_stats_database_upstreams(struct ftl_conn *api) // Add cache and blocklist as upstreams cJSON *cached = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(cached, "ip", ""); + JSON_REF_STR_IN_OBJECT(cached, "ip", "cache"); JSON_REF_STR_IN_OBJECT(cached, "name", "cache"); + JSON_ADD_NUMBER_TO_OBJECT(cached, "port", -1); JSON_ADD_NUMBER_TO_OBJECT(cached, "count", cached_queries); + + cJSON *statistics = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", 0); + JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", 0); + JSON_ADD_ITEM_TO_OBJECT(cached, "statistics", statistics); + JSON_ADD_ITEM_TO_ARRAY(upstreams, cached); cJSON *blocked = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(blocked, "ip", ""); + JSON_REF_STR_IN_OBJECT(blocked, "ip", "blocklist"); JSON_REF_STR_IN_OBJECT(blocked, "name", "blocklist"); + JSON_ADD_NUMBER_TO_OBJECT(blocked, "port", -1); JSON_ADD_NUMBER_TO_OBJECT(blocked, "count", blocked_queries); + JSON_ADD_ITEM_TO_OBJECT(blocked, "statistics", statistics); + JSON_ADD_ITEM_TO_ARRAY(upstreams, blocked); // Close (= unlock) database connection diff --git a/src/database/query-table.c b/src/database/query-table.c index df757123..2a123970 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -882,7 +882,7 @@ void DB_read_queries(void) query->dnssec = dnssec; query->reply = reply; counters->reply[query->reply]++; - query->response = reply_time * 1e4; // convert to tenth-millisecond unit + query->response = reply_time; query->CNAME_domainID = -1; // Initialize flags query->flags.complete = true; // Mark as all information is available @@ -981,6 +981,7 @@ void DB_read_queries(void) { upstream->overTime[timeidx]++; upstream->lastQuery = queryTimeStamp; + upstream->count++; } } break; @@ -1315,8 +1316,8 @@ bool queries_to_database(void) // REPLY_TIME if(query->flags.response_calculated) - // Store difference (in milliseconds) when applicable - sqlite3_bind_double(query_stmt, 12, 1e-4*query->response); + // Store difference (in seconds) when applicable + sqlite3_bind_double(query_stmt, 12, query->response); else // Store NULL otherwise sqlite3_bind_null(query_stmt, 12); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index a84d83cf..7d71b627 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -722,7 +722,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, query->flags.database.stored = false; query->flags.database.changed = true; query->flags.complete = false; - query->response = converttimeval(request); + query->response = querytimestamp; query->flags.response_calculated = false; // Initialize reply type query->reply = REPLY_UNKNOWN; @@ -1584,6 +1584,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio unsigned short port, const int id, const char* file, const int line) { // Save that this query got forwarded to an upstream server + const double now = double_time(); // Lock shared memory lock_shm(); @@ -1660,6 +1661,8 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio upstream->overTime[timeidx]++; // Update lastQuery timestamp upstream->lastQuery = time(NULL); + // Count forwarded query + upstream->count++; } // Proceed only if @@ -1701,7 +1704,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio // can go back in time to measure both the initial cache // lookup and the (now starting) time it takes for the // upstream to respond - query->response = converttimeval(response) - query->response; + query->response = now - query->response; query->flags.response_calculated = false; } } @@ -1779,14 +1782,14 @@ static void mysockaddr_extract_ip_port(union mysockaddr *server, char ip[ADDRSTR } // Compute cache/upstream response time -static inline void set_response_time(queriesData *query, const struct timeval response) +static inline void set_response_time(queriesData *query, const double now) { // Do this only if this is the first time we set a reply if(query->flags.response_calculated) return; // Convert absolute timestamp to relative timestamp - query->response = converttimeval(response) - query->response; + query->response = now - query->response; query->flags.response_calculated = true; } @@ -1828,6 +1831,7 @@ static void update_upstream(queriesData *query, const int id) static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char *arg, const int id, const char* file, const int line) { + const double now = double_time(); // If domain is "pi.hole", we skip this query // We compare case-insensitive here // Hint: name can be NULL, e.g. for NODATA/NXDOMAIN replies @@ -1963,7 +1967,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Save response time // Skipped internally if already computed - set_response_time(query, response); + set_response_time(query, now); // We only process the first reply further in here // Check if reply type is still UNKNOWN @@ -2052,6 +2056,14 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al } else if((flags & (F_FORWARD | F_UPSTREAM)) && isExactMatch) { + upstreamsData *upstream = getUpstream(query->upstreamID, true); + upstream->responses++; + + // Re-compute upstream average response time and uncertainty + upstream->rtime += query->response; + const double mean = upstream->rtime / upstream->responses; + upstream->rtuncertainty += (mean - query->response)*(mean - query->response); + // Only proceed if query is not already known // to have been blocked by Quad9 if(query->status == QUERY_EXTERNAL_BLOCKED_IP || @@ -2247,6 +2259,7 @@ static void query_blocked(queriesData* query, domainsData* domain, clientsData* { const int timeidx = getOverTimeID(query->timestamp); upstream->overTime[timeidx]--; + upstream->count--; } } else if(is_blocked(query->status)) @@ -2558,6 +2571,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep const char *file, const int line) { enum reply_type new_reply = REPLY_UNKNOWN; + const double now = double_time(); // If reply is set, we use it directly instead of interpreting the flags if(reply != 0) { @@ -2634,7 +2648,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep // Save response time // Skipped internally if already computed - set_response_time(query, response); + set_response_time(query, now); } static void init_pihole_PTR(void) diff --git a/src/overTime.c b/src/overTime.c index dd2dbbcc..446711c4 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -218,6 +218,9 @@ void moveOverTimeMemory(const time_t mintime) if(!upstream) continue; + // Adjust upstream's queries counter + upstream->count -= upstream->overTime[0]; + // Move upstream-specific overTime memory memmove(&(upstream->overTime[0]), &(upstream->overTime[moveOverTime]), diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index 52eeaea2..60b9dfe0 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -77,7 +77,7 @@ int send_json_error(struct ftl_conn *api, const int code, const char *key, const char* message, const char *hint) { - log_err("API error: %s, hint: %s", message, hint); + log_warn("API error: %s, hint: %s", message, hint); cJSON *error = JSON_NEW_OBJECT(); JSON_REF_STR_IN_OBJECT(error, "key", key); diff --git a/test/api/checkAPI.py b/test/api/checkAPI.py index 4339e967..6d4d39b1 100644 --- a/test/api/checkAPI.py +++ b/test/api/checkAPI.py @@ -27,7 +27,7 @@ def read_yaml_maybe_cache(file: str) -> dict: try: yamls[file] = yaml.safe_load(stream) except Exception as e: - print("Exception when reading " + file + ": " + e) + print("Exception when reading " + file + ": " + str(e)) exit(1) return yamls[file]