From e4383775d11898d7cf2bdd6e89ee0bdf151e4113 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 24 Jan 2023 20:15:45 +0100 Subject: [PATCH] Add /api/action/gravity which can be used to trigger a run of pihole -g. The output is live streamed using HTTP/1.1 chunked encoding. Signed-off-by: DL6ER --- src/api/CMakeLists.txt | 2 + src/api/action.c | 107 +++++++++++++++++++++++++ src/api/api.c | 3 +- src/api/api.h | 3 + src/api/docs/CMakeLists.txt | 1 + src/api/docs/content/specs/action.yaml | 65 +++++++++++++++ src/api/docs/content/specs/main.yaml | 5 ++ src/api/teleporter.c | 3 + src/database/common.c | 11 ++- test/api/checkAPI.py | 5 ++ test/api/libs/responseVerifyer.py | 9 ++- 11 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 src/api/action.c create mode 100644 src/api/docs/content/specs/action.yaml diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index f47c9ba5..32293c06 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -9,6 +9,8 @@ # Please see LICENSE file for your rights under this license. set(sources + action.c + api_helper.h api.h api.c auth.c diff --git a/src/api/action.c b/src/api/action.c new file mode 100644 index 00000000..733cb9c9 --- /dev/null +++ b/src/api/action.c @@ -0,0 +1,107 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* API Implementation /api/action +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "webserver/http-common.h" +#include "webserver/json_macros.h" +#include "api/api.h" +// wait() +#include + +static int run_and_stream_command(struct ftl_conn *api, const char *path, const char *const args[]) +{ + // Create a pipe for communication with our child + int pipefd[2]; + if(pipe(pipefd) !=0) + { + log_err("Cannot create pipe while running gravity action: %s", strerror(errno)); + return false; + } + + // Fork! + pid_t cpid = fork(); + int code = -1; + bool crashed = false; + if (cpid == 0) + { + /*** CHILD ***/ + // Close the reading end of the pipe + close(pipefd[0]); + + // Disable logging + log_ctrl(false, false); + + // Flush STDERR + fflush(stderr); + + // Redirect STDERR into our pipe + dup2(pipefd[1], STDERR_FILENO); + dup2(pipefd[1], STDOUT_FILENO); + + // Run pihole -g + execv(path, (char *const *)args); + + // Exit the fork + exit(EXIT_SUCCESS); + } + else + { + /*** PARENT ***/ + // Close the writing end of the pipe + close(pipefd[1]); + + // Send 200 OK with chunked size (-1) + mg_send_http_ok(api->conn, "text/plain", -1); + + // Read readirected STDOUT/STDERR until EOF + // We are only interested in the last pipe line + char errbuf[1024] = ""; + while(read(pipefd[0], errbuf, sizeof(errbuf)) > 0) + { + // Send chunked data + // The chunked size is the length of the string in hex and has to be + // transferred in advance, followed by \r\n as line separator and + // followed by a chunk of data (the string itself) of the specified + // size + mg_printf(api->conn, "%zX\r\n%s\r\n", strlen(errbuf), errbuf); + + // Reset buffer + memset(errbuf, 0, sizeof(errbuf)); + } + + // Wait until child has exited to get its return code + int status; + waitpid(cpid, &status, 0); + code = WEXITSTATUS(status); + + if(WIFSIGNALED(status)) + { + crashed = true; + log_err("gravity failed with signal %d %s", + WTERMSIG(status), + WCOREDUMP(status) ? "(core dumped)" : ""); + } + + log_debug(DEBUG_API, "Gravity return code: %d", code); + + // Close the reading end of the pipe + close(pipefd[0]); + } + + // Send final chunk of size 0 showing end of data + mg_printf(api->conn, "0\r\n\r\n"); + + return code == EXIT_SUCCESS && !crashed ? 200 : 500; +} + +int api_action_gravity(struct ftl_conn *api) +{ + return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL }); +} \ No newline at end of file diff --git a/src/api/api.c b/src/api/api.c index af782020..d68a30d8 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -78,7 +78,8 @@ static struct { { "/api/network/interfaces", "", api_network_interfaces, { false, 0 }, true, HTTP_GET }, { "/api/network/devices", "", api_network_devices, { false, 0 }, true, HTTP_GET }, { "/api/endpoints", "", api_endpoints, { false, 0 }, true, HTTP_GET }, - { "/api/teleporter", "", api_teleporter, { false, 0 }, false, HTTP_GET | HTTP_POST }, + { "/api/teleporter", "", api_teleporter, { false, 0 }, true, HTTP_GET | HTTP_POST }, + { "/api/action/gravity", "", api_action_gravity, { false, 0 }, true, HTTP_GET }, { "/api/docs", "", api_docs, { false, 0 }, false, HTTP_GET }, }; diff --git a/src/api/api.h b/src/api/api.h index 920f9ca1..acab6642 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -85,4 +85,7 @@ int api_docs(struct ftl_conn *api); // Teleporter methods int api_teleporter(struct ftl_conn *api); +// Action methods +int api_action_gravity(struct ftl_conn *api); + #endif // ROUTES_H diff --git a/src/api/docs/CMakeLists.txt b/src/api/docs/CMakeLists.txt index 5692d423..74ab6783 100644 --- a/src/api/docs/CMakeLists.txt +++ b/src/api/docs/CMakeLists.txt @@ -18,6 +18,7 @@ set(sources hex/external/highlight-default.min.css hex/external/geraintluff-sha256.min.js hex/images/logo.svg + hex/specs/action.yaml hex/specs/auth.yaml hex/specs/clients.yaml hex/specs/common.yaml diff --git a/src/api/docs/content/specs/action.yaml b/src/api/docs/content/specs/action.yaml new file mode 100644 index 00000000..7c24e4dc --- /dev/null +++ b/src/api/docs/content/specs/action.yaml @@ -0,0 +1,65 @@ +openapi: 3.0.2 +components: + paths: + gravity: + get: + summary: Run gravity + tags: + - Actions + operationId: "action_gravity" + description: | + Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding. + responses: + '200': + description: OK + content: + text/plain: + schema: + type: string + example: | + [i] Neutrino emissions detected... + + [✓] Pulling blocklist source list into range + + [i] Preparing new gravity database... + [✓] Preparing new gravity database + [i] Using libz compression + + [i] Target: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts + [✓] Status: Retrieval successful + [i] Imported 172502 domains, ignoring 3 non-domain entries + Sample of non-domain entries: + - 0.0.0.0 + - fe + - ff + [i] List stayed unchanged + + [i] Target: https://v.firebog.net/hosts/AdguardDNS.txt + [✓] Status: No changes detected + [i] Imported 47225 domains + + [✓] Creating new gravity databases + [✓] Storing downloaded domains in new gravity database + [✓] Building tree + [✓] Swapping databases + [✓] The old database remains available. + [i] Number of gravity domains: 219727 (215440 unique domains) + [i] Number of exact blacklisted domains: 0 + [i] Number of regex blacklist filters: 2 + [i] Number of exact whitelisted domains: 0 + [i] Number of regex whitelist filters: 0 + [✓] Cleaning up stray matter + + [✓] FTL is listening on port + [✓] UDP (IPv4) + [✓] TCP (IPv4) + [✓] UDP (IPv6) + [✓] TCP (IPv6) + + [✓] Pi-hole blocking is enabled + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: 'common.yaml#/components/errors/unauthorized' \ No newline at end of file diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index dde1c005..766dfa16 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -48,6 +48,8 @@ tags: description: Methods used to configure your Pi-hole - name: "Network information" description: Methods used to gather advanced information about your network + - name: "Actions" + description: Methods used to trigger certain actions on your Pi-hole paths: /auth: @@ -197,6 +199,9 @@ paths: /teleporter: $ref: 'teleporter.yaml#/components/paths/teleporter' + /action/gravity: + $ref: 'action.yaml#/components/paths/gravity' + components: securitySchemes: sidHeader: diff --git a/src/api/teleporter.c b/src/api/teleporter.c index cc79e1f8..0d3ec266 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -38,6 +38,9 @@ static int api_teleporter_GET(struct ftl_conn *api) // Send 200 OK with appropriate headers mg_send_http_ok(api->conn, "application/zip", size); + // Clear extra headers + pi_hole_extra_headers[0] = '\0'; + // Send raw (binary) ZIP content mg_write(api->conn, ptr, size); diff --git a/src/database/common.c b/src/database/common.c index 0fe5fd0b..44048652 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -230,7 +230,16 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) // Note: pArg is NULL and not used // See https://sqlite.org/rescode.html#extrc for details // concerning the return codes returned here - log_err("SQLite3 message: %s (%d)", zMsg, iErrCode); + if(strncmp(zMsg, "file renamed while open: ", sizeof("file renamed while open: ")-1) == 0) + { + // This happens when gravity.db is replaced while FTL is running + // We can safely ignore this warning + return; + } + if(iErrCode == SQLITE_WARNING) + log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode); + else + log_err("SQLite3 message: %s (%d)", zMsg, iErrCode); } void db_init(void) diff --git a/test/api/checkAPI.py b/test/api/checkAPI.py index 2adc8e47..64d1788e 100644 --- a/test/api/checkAPI.py +++ b/test/api/checkAPI.py @@ -50,6 +50,11 @@ if __name__ == "__main__": # matches the OpenAPI specs. print("Verifying the individual endpoint properties...") for path in openapi.endpoints["get"]: + # We do not check the action endpoints as they'd trigger + # possibly unwanted action such as restarting FTL, running + # gravity, stutting down the system, etc. + if path.startswith("/api/action"): + continue verifyer = ResponseVerifyer(ftl, openapi) errors = verifyer.verify_endpoint(path) if len(errors) == 0: diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index ed921faf..8f92e53e 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -20,6 +20,7 @@ class ResponseVerifyer(): # Translate between OpenAPI and Python types YAML_TYPES = { "string": [str], "integer": [int], "number": [int, float], "boolean": [bool], "array": [list] } + TELEPORTER_FILES = ["etc/pihole/gravity.db", "etc/pihole/pihole.toml", "etc/pihole/pihole-FTL.db", "etc/hosts"] def __init__(self, ftl: FTLAPI, openapi: openApi): self.ftl = ftl @@ -141,9 +142,13 @@ class ResponseVerifyer(): # Try to read pihole.toml and see if it starts with the expected # header block try: - pihole_toml = zipfile_obj.read("pihole.toml") + # Check if all expected files are present + for expected_file in self.TELEPORTER_FILES: + if expected_file not in zipfile_obj.namelist(): + self.errors.append("File " + expected_file + " is missing in received archive.") + pihole_toml = zipfile_obj.read("etc/pihole/pihole.toml") if not pihole_toml.startswith(b"# This file is managed by pihole-FTL"): - self.errors.append("Received ZIP file starts with wrong header") + self.errors.append("Received ZIP file's pihole.toml starts with wrong header") except Exception as err: self.errors.append("Error during ZIP analysis: " + str(err)) else: