From a96c283c0cc19ccce2504cff11e79bbfe2c603f3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Nov 2023 19:25:21 +0100 Subject: [PATCH 01/55] Add authentication via query string Signed-off-by: DL6ER --- src/api/auth.c | 16 ++++++++++++++++ test/api/libs/FTLAPI.py | 13 ++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index d48d171b..2f95b337 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -140,6 +140,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } } + // If not, does the client provide a session ID via COOKIE? bool cookie_auth = false; if(!sid_avail) { @@ -151,7 +152,22 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // Mark SID as available sid_avail = true; } + } + // If not, does the client provide a session ID via URI? + if(!sid_avail && api->request->query_string && GET_VAR("sid", sid, api->request->query_string) > 0) + { + // "+" may have been replaced by " ", undo this here + for(unsigned int i = 0; i < SID_SIZE; i++) + if(sid[i] == ' ') + sid[i] = '+'; + + // Zero terminate SID string + sid[SID_SIZE-1] = '\0'; + // Mention source of SID + sid_source = "URI"; + // Mark SID as available + sid_avail = true; } if(!sid_avail) diff --git a/test/api/libs/FTLAPI.py b/test/api/libs/FTLAPI.py index a71bcec2..c8d19e8b 100644 --- a/test/api/libs/FTLAPI.py +++ b/test/api/libs/FTLAPI.py @@ -15,6 +15,7 @@ import requests from typing import List import json from hashlib import sha256 +import urllib.parse url = "http://pi.hole/api/auth" @@ -23,6 +24,7 @@ class AuthenticationMethods(Enum): HEADER = 1 BODY = 2 COOKIE = 3 + QUERY_STR = 4 # Class to query the FTL API class FTLAPI(): @@ -103,13 +105,18 @@ class FTLAPI(): def GET(self, uri: str, params: List[str] = [], expected_mimetype: str = "application/json", authenticate: AuthenticationMethods = AuthenticationMethods.BODY): self.errors = [] try: + # Get json_data, headers and cookies + json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate) + + # Add session ID to the request if authenticating via query string + if self.auth_method == AuthenticationMethods.QUERY_STR.name: + encoded_sid = urllib.parse.quote(self.session['sid'], safe='') + params.append("sid=" + encoded_sid) + # Add parameters to the URI (if any) if len(params) > 0: uri = uri + "?" + "&".join(params) - # Get json_data, headers and cookies - json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate) - if self.verbose: print("GET " + self.api_url + uri + " with json_data: " + json.dumps(json_data)) From 00a9bc8d17652ad6182fd451906272f02c441514 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 18:43:08 +0100 Subject: [PATCH 02/55] Generalize upload handing scripts to possibly accept other files than ZIP archives Signed-off-by: DL6ER --- src/api/teleporter.c | 78 ++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 87688f62..470829f4 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -16,7 +16,7 @@ // ERRBUF_SIZE #include "config/dnsmasq_config.h" -#define MAXZIPSIZE (50u*1024*1024) +#define MAXFILESIZE (50u*1024*1024) static int api_teleporter_GET(struct ftl_conn *api) { @@ -58,9 +58,9 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; - char *zip_data; - char *zip_filename; - size_t zip_size; + char *data; + char *filename; + size_t filesize; }; // Callback function for CivetWeb to determine which fields we want to receive @@ -79,7 +79,7 @@ static int field_found(const char *key, is_sid = false; if(strcasecmp(key, "file") == 0 && filename && *filename) { - data->zip_filename = strdup(filename); + data->filename = strdup(filename); is_file = true; return MG_FORM_FIELD_STORAGE_GET; } @@ -103,21 +103,21 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * if(is_file) { - if(data->zip_size + valuelen > MAXZIPSIZE) + if(data->filesize + valuelen > MAXFILESIZE) { - log_warn("Uploaded Teleporter ZIP archive is too large (limit is %u bytes)", - MAXZIPSIZE); + log_warn("Uploaded Teleporter file is too large (limit is %u bytes)", + MAXFILESIZE); data->too_large = true; return MG_FORM_FIELD_HANDLE_ABORT; } - // Allocate memory for the raw ZIP archive data - data->zip_data = realloc(data->zip_data, data->zip_size + valuelen); - // Copy the raw ZIP archive data - memcpy(data->zip_data + data->zip_size, value, valuelen); - // Store the size of the ZIP archive raw data - data->zip_size += valuelen; - log_debug(DEBUG_API, "Received ZIP archive (%zu bytes, buffer is now %zu bytes)", - valuelen, data->zip_size); + // Allocate memory for the raw file data + data->data = realloc(data->data, data->filesize + valuelen); + // Copy the raw file data + memcpy(data->data + data->filesize, value, valuelen); + // Store the size of the file raw data + data->filesize += valuelen; + log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)", + valuelen, data->filesize); } else if(is_sid) { @@ -143,24 +143,27 @@ static int field_stored(const char *path, long long file_size, void *user_data) static int free_upload_data(struct upload_data *data) { // Free allocated memory - if(data->zip_filename) + if(data->filename) { - free(data->zip_filename); - data->zip_filename = NULL; + free(data->filename); + data->filename = NULL; } if(data->sid) { free(data->sid); data->sid = NULL; } - if(data->zip_data) + if(data->data) { - free(data->zip_data); - data->zip_data = NULL; + free(data->data); + data->data = NULL; } return 0; } +// Private function prototypes +static int process_received_zip(struct ftl_conn *api, struct upload_data *data); + static int api_teleporter_POST(struct ftl_conn *api) { struct upload_data data; @@ -170,7 +173,7 @@ static int api_teleporter_POST(struct ftl_conn *api) // Disallow large ZIP archives (> 50 MB) to prevent DoS attacks. // Typically, the ZIP archive size should be around 30-100 kB. - if(req_info->content_length > MAXZIPSIZE) + if(req_info->content_length > MAXFILESIZE) { free_upload_data(&data); return send_json_error(api, 400, @@ -191,7 +194,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something we consider being a file - if(data.zip_data == NULL || data.zip_size == 0) + if(data.data == NULL || data.filesize == 0) { free_upload_data(&data); return send_json_error(api, 400, @@ -209,28 +212,17 @@ static int api_teleporter_POST(struct ftl_conn *api) "ZIP archive too large", NULL); } -/* - // Set the payload to the SID we received (if available) - if(data.sid != NULL) - { - const size_t bufsize = strlen(data.sid) + 5; - api->payload.raw = calloc(bufsize, sizeof(char)); - strncpy(api->payload.raw, "sid=", 5); - strncat(api->payload.raw, data.sid, bufsize - 4); - } - // Check if the client is authorized to use this API endpoint - if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) - { - free_upload_data(&data); - return send_json_unauthorized(api); - } -*/ // Process what we received + return process_received_zip(api, &data); +} + +static int process_received_zip(struct ftl_conn *api, struct upload_data *data) +{ char hint[ERRBUF_SIZE]; memset(hint, 0, sizeof(hint)); cJSON *json_files = JSON_NEW_ARRAY(); - const char *error = read_teleporter_zip(data.zip_data, data.zip_size, hint, json_files); + const char *error = read_teleporter_zip(data->data, data->filesize, hint, json_files); if(error != NULL) { const size_t msglen = strlen(error) + strlen(hint) + 4; @@ -242,7 +234,7 @@ static int api_teleporter_POST(struct ftl_conn *api) strcat(msg, ": "); strcat(msg, hint); } - free_upload_data(&data); + free_upload_data(data); return send_json_error_free(api, 400, "bad_request", "Invalid ZIP archive", @@ -250,7 +242,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Free allocated memory - free_upload_data(&data); + free_upload_data(data); // Send response cJSON *json = JSON_NEW_OBJECT(); From b9fa29c18ce84b752eefa1f65d28053fa00a363b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 19:04:00 +0100 Subject: [PATCH 03/55] Add some rule along which we will decide the user supplied somethings that (superficially) looks like a Teleporter v6 ZIP file Signed-off-by: DL6ER --- src/api/teleporter.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 470829f4..856da89c 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -213,8 +213,32 @@ static int api_teleporter_POST(struct ftl_conn *api) NULL); } - // Process what we received - return process_received_zip(api, &data); + // Check if we received something that claims to be a ZIP archive + // - filename + // - shoud be at least 12 characters long, + // - should start in "pi-hole_", + // - have "_teleporter_" in the middle, and + // - end in ".zip" + // - the data itself + // - should be at least 40 bytes long + // - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT) + if(strlen(data.filename) >= 12 && + strncmp(data.filename, "pi-hole_", 8) == 0 && + strstr(data.filename, "_teleporter_") != NULL && + strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 && + data.filesize >= 40 && + memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0) + { + return process_received_zip(api, &data); + } + else + { + free_upload_data(&data); + return send_json_error(api, 400, + "bad_request", + "Invalid file", + "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); + } } static int process_received_zip(struct ftl_conn *api, struct upload_data *data) From 50a72afcef99d980dc9fbb4e39a4c82f61ba564b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:21:48 +0100 Subject: [PATCH 04/55] Add TAR routines for efficient parsing of a tar archive in memory Signed-off-by: DL6ER --- src/api/teleporter.c | 72 ++++++++++++++++++++--- src/zip/CMakeLists.txt | 2 + src/zip/gzip.c | 5 +- src/zip/gzip.h | 4 ++ src/zip/tar.c | 129 +++++++++++++++++++++++++++++++++++++++++ src/zip/tar.h | 19 ++++++ src/zip/teleporter.c | 2 +- src/zip/teleporter.h | 2 +- 8 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 src/zip/tar.c create mode 100644 src/zip/tar.h diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 856da89c..b82cd535 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -15,6 +15,10 @@ #include "api/api.h" // ERRBUF_SIZE #include "config/dnsmasq_config.h" +// inflate_buffer() +#include "zip/gzip.h" +// find_file_in_tar() +#include "zip/tar.h" #define MAXFILESIZE (50u*1024*1024) @@ -58,7 +62,7 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; - char *data; + uint8_t *data; char *filename; size_t filesize; }; @@ -163,6 +167,7 @@ static int free_upload_data(struct upload_data *data) // Private function prototypes static int process_received_zip(struct ftl_conn *api, struct upload_data *data); +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data); static int api_teleporter_POST(struct ftl_conn *api) { @@ -231,14 +236,31 @@ static int api_teleporter_POST(struct ftl_conn *api) { return process_received_zip(api, &data); } - else + // Check if we received something that claims to be a TAR.GZ archive + // - filename + // - shoud be at least 12 characters long, + // - should start in "pi-hole-", + // - have "-teleporter_" in the middle, and + // - end in ".tar.gz" + // - the data itself + // - should be at least 40 bytes long + // - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt) + else if(strlen(data.filename) >= 12 && + strncmp(data.filename, "pi-hole-", 8) == 0 && + strstr(data.filename, "-teleporter_") != NULL && + strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 && + data.filesize >= 40 && + memcmp(data.data, "\x1f\x8b", 2) == 0) { - free_upload_data(&data); - return send_json_error(api, 400, - "bad_request", - "Invalid file", - "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); + return process_received_tar_gz(api, &data); } + + // else: invalid file + free_upload_data(&data); + return send_json_error(api, 400, + "bad_request", + "Invalid file", + "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); } static int process_received_zip(struct ftl_conn *api, struct upload_data *data) @@ -274,6 +296,42 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) JSON_SEND_OBJECT(json); } +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data) +{ + // Try to decompress the received data + uint8_t *archive = NULL; + mz_ulong archive_size = 0u; + if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size)) + { + free_upload_data(data); + return send_json_error(api, 400, + "bad_request", + "Invalid GZIP archive", + "The uploaded file does not appear to be a valid gzip archive - decompression failed"); + } + + // Check if the decompressed data is a valid TAR archive + cJSON *json_files = list_files_in_tar(archive, archive_size); + + // Print all files in the TAR archive + cJSON *file = NULL; + cJSON_ArrayForEach(file, json_files) + { + cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); + cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); + log_info("Found file in TAR archive: \"%s\" (%d bytes)", + name->valuestring, size->valueint); + } + + // Free allocated memory + free_upload_data(data); + + // Send response + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); + JSON_SEND_OBJECT(json); +} + int api_teleporter(struct ftl_conn *api) { if(api->method == HTTP_GET) diff --git a/src/zip/CMakeLists.txt b/src/zip/CMakeLists.txt index 8042ed5e..3abeb7ad 100644 --- a/src/zip/CMakeLists.txt +++ b/src/zip/CMakeLists.txt @@ -11,6 +11,8 @@ set(sources gzip.c gzip.h + tar.c + tar.h teleporter.c teleporter.h ) diff --git a/src/zip/gzip.c b/src/zip/gzip.c index bf15ea86..f7fbc571 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -14,7 +14,6 @@ #include // le32toh and friends #include -#include "miniz/miniz.h" #include "gzip.h" #include "log.h" @@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul return true; } -static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, - unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) { // Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08) if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B) diff --git a/src/zip/gzip.h b/src/zip/gzip.h index 7839602c..b0c916b6 100644 --- a/src/zip/gzip.h +++ b/src/zip/gzip.h @@ -11,6 +11,10 @@ #define GZIP_H #include +#include "miniz/miniz.h" + +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed); bool deflate_file(const char *in, const char *out, bool verbose); bool inflate_file(const char *infile, const char *outfile, bool verbose); diff --git a/src/zip/tar.c b/src/zip/tar.c new file mode 100644 index 00000000..639af231 --- /dev/null +++ b/src/zip/tar.c @@ -0,0 +1,129 @@ +/* Pi-hole: A black hole for Internet advertisements + * (c) 2023 Pi-hole, LLC (https://pi-hole.net) + * Network-wide ad blocking via your own hardware. + * + * FTL Engine + * In-memory tar reading routines + * + * This file is copyright under the latest version of the EUPL. + * Please see LICENSE file for your rights under this license. */ + +#include "zip/tar.h" +#include "log.h" + +// TAR offsets +#define TAR_NAME_OFFSET 0 +#define TAR_SIZE_OFFSET 124 +#define TAR_MAGIC_OFFSET 257 + +// TAR constants +#define TAR_BLOCK_SIZE 512 +#define TAR_NAME_SIZE 100 +#define TAR_SIZE_SIZE 12 +#define TAR_MAGIC_SIZE 5 + +static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ + +/** + * Find a file in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @param fileName Name of the file to find + * @param fileSize Pointer to a size_t variable to store the file size in + * @return Pointer to the file data or NULL if not found + */ +const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, + const char *fileName, size_t *fileSize) +{ + bool found = false; + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Initialize fileSize to 0 + *fileSize = 0; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return NULL; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + found = strncmp(name, fileName, TAR_NAME_SIZE) == 0; + } while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize)); + + if (!found) + return NULL; // No file found in TAR - return NULL + + // File found in TAR - return pointer to file data and set fileSize + *fileSize = size; + return tarData + p + TAR_BLOCK_SIZE; +} + +/** + * List all files in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @return Pointer to a cJSON array containing all file names with file size + */ +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) +{ + cJSON *files = cJSON_CreateArray(); + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return files; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + // Add file name to cJSON array + log_info("Found file '%s' with size %zu", name, size); + cJSON *file = cJSON_CreateObject(); + cJSON_AddItemToObject(file, "name", cJSON_CreateString(name)); + cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size)); + cJSON_AddItemToArray(files, file); + } while (p + newOffset + TAR_BLOCK_SIZE <= tarSize); + + return files; +} \ No newline at end of file diff --git a/src/zip/tar.h b/src/zip/tar.h new file mode 100644 index 00000000..426a4037 --- /dev/null +++ b/src/zip/tar.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* TAR reading routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef TAR_H +#define TAR_H + +#include "FTL.h" +#include "webserver/cJSON/cJSON.h" + +const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); + +#endif // TAR_H \ No newline at end of file diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 62281d89..5935b6de 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -524,7 +524,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char * return NULL; } -const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files) +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files) { // Initialize ZIP archive mz_zip_archive zip = { 0 }; diff --git a/src/zip/teleporter.h b/src/zip/teleporter.h index b20567b9..a5743028 100644 --- a/src/zip/teleporter.h +++ b/src/zip/teleporter.h @@ -15,7 +15,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size); bool free_teleporter_zip(mz_zip_archive *zip); -const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files); +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files); bool write_teleporter_zip_to_disk(void); bool read_teleporter_zip_from_disk(const char *filename); From 68d6f4ab9416bb9c8db10c76514830c145e89014 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 23:33:53 +0100 Subject: [PATCH 05/55] Test parse adlist.json Signed-off-by: DL6ER --- src/api/teleporter.c | 20 ++++++++++++++++++++ src/zip/tar.c | 4 ++-- src/zip/tar.h | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index b82cd535..51183ca1 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -323,6 +323,26 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat name->valuestring, size->valueint); } + // Parse adlist.json + size_t fileSize = 0u; + const char *adlist_json = find_file_in_tar(archive, archive_size, "adlist.json", &fileSize); + if(adlist_json != NULL) + { + cJSON *adlists = cJSON_ParseWithLength(adlist_json, fileSize); + if(adlists != NULL) + { + cJSON *adlist = NULL; + cJSON_ArrayForEach(adlist, adlists) + { + cJSON *address = cJSON_GetObjectItemCaseSensitive(adlist, "address"); + cJSON *comment = cJSON_GetObjectItemCaseSensitive(adlist, "comment"); + log_info("Found adlist in TAR archive: \"%s\" (%s)", + address->valuestring, comment->valuestring); + } + cJSON_Delete(adlists); + } + } + // Free allocated memory free_upload_data(data); diff --git a/src/zip/tar.c b/src/zip/tar.c index 639af231..ee64b9d0 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -32,7 +32,7 @@ static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ * @param fileSize Pointer to a size_t variable to store the file size in * @return Pointer to the file data or NULL if not found */ -const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, +const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, const char *fileName, size_t *fileSize) { bool found = false; @@ -76,7 +76,7 @@ const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, // File found in TAR - return pointer to file data and set fileSize *fileSize = size; - return tarData + p + TAR_BLOCK_SIZE; + return tar + p + TAR_BLOCK_SIZE; } /** diff --git a/src/zip/tar.h b/src/zip/tar.h index 426a4037..0e23625f 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -13,7 +13,7 @@ #include "FTL.h" #include "webserver/cJSON/cJSON.h" -const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); +const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); #endif // TAR_H \ No newline at end of file From 6c62d3e3f95eeb374dae75497b49795fd8c20158 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 14:06:44 +0100 Subject: [PATCH 06/55] Add import_json_table() routine that can read, parse, and import .json files from Pi-hole v5.x Teleporter archives Signed-off-by: DL6ER --- src/api/list.c | 2 +- src/api/teleporter.c | 356 ++++++++++++++++++++++++++++++++++++++++--- src/zip/tar.c | 6 +- src/zip/tar.h | 4 +- 4 files changed, 337 insertions(+), 31 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 3c0b6dc0..2d9931c2 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -524,7 +524,7 @@ int api_list(struct ftl_conn *api) } else if((api->item = startsWith("/api/domains/allow", api)) != NULL) { - listtype = GRAVITY_DOMAINLIST_ALLOW_ALL; + listtype = GRAVITY_DOMAINLIST_ALLOW_ALL; } else if((api->item = startsWith("/api/domains/deny/exact", api)) != NULL) { diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 51183ca1..6cda1d52 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -19,6 +19,10 @@ #include "zip/gzip.h" // find_file_in_tar() #include "zip/tar.h" +// sqlite3_open_v2() +#include "database/sqlite3.h" +// dbquery() +#include "database/common.h" #define MAXFILESIZE (50u*1024*1024) @@ -296,6 +300,306 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) JSON_SEND_OBJECT(json); } +static struct teleporter_files { + const char *filename; // Filename of the file in the archive + const char *table_name; // Name of the table in the database + const int listtype; // Type of list (only used for domainlist table) + const size_t num_columns; // Number of columns in the table + const char *columns[10]; // List of columns in the table +} teleporter_v5_files[] = { + { + .filename = "adlist.json", + .table_name = "adlist", + .listtype = -1, + .num_columns = 10, + .columns = { "id", "address", "enabled", "date_added", "date_modified", "comment", "date_updated", "number", "invalid_domains", "status" } // abp_entries and type are not defined in Pi-hole v5.x + },{ + .filename = "adlist_by_group.json", + .table_name = "adlist_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "adlist_id" } + },{ + .filename = "blacklist.exact.json", + .table_name = "domainlist", + .listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "blacklist.regex.json", + .table_name = "domainlist", + .listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "client.json", + .table_name = "client", + .listtype = -1, + .num_columns = 5, + .columns = { "id", "ip", "date_added", "date_modified", "comment" } + },{ + .filename = "client_by_group.json", + .table_name = "client_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "client_id" } + },{ + .filename = "domainlist_by_group.json", + .table_name = "domainlist_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "domainlist_id" } + },{ + .filename = "group.json", + .table_name = "group", + .listtype = -1, + .num_columns = 6, + .columns = { "id", "enabled", "name", "date_added", "date_modified", "description" } + },{ + .filename = "whitelist.exact.json", + .table_name = "domainlist", + .listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "whitelist.regex.json", + .table_name = "domainlist", + .listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + } +}; + +static bool import_json_table(cJSON *json, struct teleporter_files *file) +{ + // Check if the JSON object is an array + if(!cJSON_IsArray(json)) + { + log_err("import_json_table(%s): JSON object is not an array", file->filename); + return false; + } + + // Check if the JSON array is empty, if so, we can return early + const int num_entries = cJSON_GetArraySize(json); + if(num_entries == 0) + { + log_info("import_json_table(%s): JSON array is empty", file->filename); + return true; + } + + // Check if all the JSON entries contain all the expected columns + cJSON *json_object = NULL; + cJSON_ArrayForEach(json_object, json) + { + if(!cJSON_IsObject(json_object)) + { + log_err("import_json_table(%s): JSON array does not contain objects", file->filename); + return false; + } + + // If this is a record for the domainlist table, add type/kind + if(strcmp(file->table_name, "domainlist") == 0) + { + // Add type/kind to the JSON object + cJSON_AddNumberToObject(json_object, "type", file->listtype); + } + + // Check if the JSON object contains the expected columns + for(size_t i = 0; i < file->num_columns; i++) + { + if(cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]) == NULL) + { + log_err("import_json_table(%s): JSON object does not contain column \"%s\"", file->filename, file->columns[i]); + return false; + } + } + } + + log_info("import_json_table(%s): JSON array contains %d entr%s", file->filename, num_entries, num_entries == 1 ? "y" : "ies"); + + // Open database connection + sqlite3 *db = NULL; + if(sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to open database file \"%s\": %s", + file->filename, config.files.database.v.s, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Disable foreign key constraints + if(sqlite3_exec(db, "PRAGMA foreign_keys = OFF;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to disable foreign key constraints: %s", file->filename, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Start transaction + if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to start transaction: %s", file->filename, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Clear existing table entries + if(file->listtype < 0) + { + // Delete all entries in the table + if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", + file->filename, file->table_name, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + } + else + { + // Delete all entries in the table of the same type + if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", + file->filename, file->table_name, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + } + + // Build dynamic SQL insertion statement + // "INSERT OR IGNORE INTO table (column1, column2, ...) VALUES (?, ?, ...);" + char *sql = sqlite3_mprintf("INSERT OR IGNORE INTO \"%s\" (", file->table_name); + for(size_t i = 0; i < file->num_columns; i++) + { + char *sql2 = sqlite3_mprintf("%s%s", sql, file->columns[i]); + sqlite3_free(sql); + sql = NULL; + if(i < file->num_columns - 1) + { + sql = sqlite3_mprintf("%s, ", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + else + { + sql = sqlite3_mprintf("%s) VALUES (", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + } + for(size_t i = 0; i < file->num_columns; i++) + { + char *sql2 = sqlite3_mprintf("%s?", sql); + sqlite3_free(sql); + sql = NULL; + if(i < file->num_columns - 1) + { + sql = sqlite3_mprintf("%s, ", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + else + { + sql = sqlite3_mprintf("%s);", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + } + + // Prepare SQL statement + sqlite3_stmt *stmt = NULL; + if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) + { + log_err("Unable to prepare SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_free(sql); + sqlite3_close(db); + return false; + } + + // Free allocated memory + sqlite3_free(sql); + sql = NULL; + + // Iterate over all JSON objects + cJSON_ArrayForEach(json_object, json) + { + // Bind values to SQL statement + for(size_t i = 0; i < file->num_columns; i++) + { + cJSON *json_value = cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]); + if(cJSON_IsString(json_value)) + { + // Bind string value + if(sqlite3_bind_text(stmt, i + 1, json_value->valuestring, -1, SQLITE_STATIC) != SQLITE_OK) + { + log_err("Unable to bind text value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + else if(cJSON_IsNumber(json_value)) + { + // Bind integer value + if(sqlite3_bind_int(stmt, i + 1, json_value->valueint) != SQLITE_OK) + { + log_err("Unable to bind integer value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + else + { + log_err("Unable to bind value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + + // Execute SQL statement + if(sqlite3_step(stmt) != SQLITE_DONE) + { + log_err("Unable to execute SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + + // Reset SQL statement + if(sqlite3_reset(stmt) != SQLITE_OK) + { + log_err("Unable to reset SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + + // Finalize SQL statement + if(sqlite3_finalize(stmt) != SQLITE_OK) + { + log_err("Unable to finalize SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Commit transaction + if(sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("Unable to commit transaction: %s", sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Close database connection + sqlite3_close(db); + + return true; +} + static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data) { // Try to decompress the received data @@ -313,36 +617,38 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Check if the decompressed data is a valid TAR archive cJSON *json_files = list_files_in_tar(archive, archive_size); - // Print all files in the TAR archive - cJSON *file = NULL; - cJSON_ArrayForEach(file, json_files) + // Print all files in the TAR archive if in debug mode + if(config.debug.api.v.b) { - cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); - cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); - log_info("Found file in TAR archive: \"%s\" (%d bytes)", - name->valuestring, size->valueint); - } - - // Parse adlist.json - size_t fileSize = 0u; - const char *adlist_json = find_file_in_tar(archive, archive_size, "adlist.json", &fileSize); - if(adlist_json != NULL) - { - cJSON *adlists = cJSON_ParseWithLength(adlist_json, fileSize); - if(adlists != NULL) + cJSON *file = NULL; + cJSON_ArrayForEach(file, json_files) { - cJSON *adlist = NULL; - cJSON_ArrayForEach(adlist, adlists) - { - cJSON *address = cJSON_GetObjectItemCaseSensitive(adlist, "address"); - cJSON *comment = cJSON_GetObjectItemCaseSensitive(adlist, "comment"); - log_info("Found adlist in TAR archive: \"%s\" (%s)", - address->valuestring, comment->valuestring); - } - cJSON_Delete(adlists); + const cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); + const cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); + if(name == NULL || size == NULL) + continue; + + log_debug(DEBUG_API, "Found file in TAR archive: \"%s\" (%d bytes)", + name->valuestring, size->valueint); } } + // Parse JSON files in the TAR archive + for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) + { + size_t fileSize = 0u; + cJSON *json = NULL; + const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) + import_json_table(json, &teleporter_v5_files[i]); + } + + // Further files to process if present: + // custom.list + // dhcp.leases + // pihole-FTL.conf + // setupVars.conf + // Free allocated memory free_upload_data(data); diff --git a/src/zip/tar.c b/src/zip/tar.c index ee64b9d0..be9a9e51 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -32,8 +32,8 @@ static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ * @param fileSize Pointer to a size_t variable to store the file size in * @return Pointer to the file data or NULL if not found */ -const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, - const char *fileName, size_t *fileSize) +const char * __attribute__((nonnull (1,3,4))) find_file_in_tar(const uint8_t *tarData, const size_t tarSize, + const char *fileName, size_t *fileSize) { bool found = false; size_t size, p = 0, newOffset = 0; @@ -85,7 +85,7 @@ const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, * @param tarSize Size of the TAR archive in memory in bytes * @return Pointer to a cJSON array containing all file names with file size */ -cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) +cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, const size_t tarSize) { cJSON *files = cJSON_CreateArray(); size_t size, p = 0, newOffset = 0; diff --git a/src/zip/tar.h b/src/zip/tar.h index 0e23625f..11f5e200 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -13,7 +13,7 @@ #include "FTL.h" #include "webserver/cJSON/cJSON.h" -const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); -cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); +const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize) __attribute__((nonnull (1,3,4))); +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) __attribute__((nonnull (1))); #endif // TAR_H \ No newline at end of file From 40e2e97259f54c015f5434fe8d7c5d244c990233 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:31:22 +0100 Subject: [PATCH 07/55] Install also remaining files Signed-off-by: DL6ER --- src/api/teleporter.c | 64 ++++++++++++++++++++++++++++++++------ src/config/config.h | 3 ++ src/config/legacy_reader.c | 2 +- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 6cda1d52..4d7c8e86 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -224,7 +224,7 @@ static int api_teleporter_POST(struct ftl_conn *api) // Check if we received something that claims to be a ZIP archive // - filename - // - shoud be at least 12 characters long, + // - should be at least 12 characters long, // - should start in "pi-hole_", // - have "_teleporter_" in the middle, and // - end in ".zip" @@ -242,7 +242,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something that claims to be a TAR.GZ archive // - filename - // - shoud be at least 12 characters long, + // - should be at least 12 characters long, // - should start in "pi-hole-", // - have "-teleporter_" in the middle, and // - end in ".tar.gz" @@ -634,27 +634,73 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat } // Parse JSON files in the TAR archive + cJSON *imported_files = JSON_NEW_ARRAY(); for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { size_t fileSize = 0u; cJSON *json = NULL; const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) - import_json_table(json, &teleporter_v5_files[i]); + if(import_json_table(json, &teleporter_v5_files[i])) + JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); } - // Further files to process if present: - // custom.list - // dhcp.leases - // pihole-FTL.conf - // setupVars.conf + // Temporarily write further files to to disk so we can import them on restart + struct { + const char *archive_name; + const char *destination; + } extract_files[] = { + { + .archive_name = "custom.list", + .destination = DNSMASQ_CUSTOM_LIST_LEGACY + },{ + .archive_name = "dhcp.leases", + .destination = DHCPLEASESFILE + },{ + .archive_name = "pihole-FTL.conf", + .destination = GLOBALCONFFILE_LEGACY + },{ + .archive_name = "setupVars.conf", + .destination = config.files.setupVars.v.s + } + }; + for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++) + { + size_t fileSize = 0u; + const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize); + if(file != NULL && fileSize > 0u) + { + // Write file to disk + FILE *fp = fopen(extract_files[i].destination, "wb"); + if(fp == NULL) + { + log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno)); + continue; + } + if(fwrite(file, fileSize, 1, fp) != 1) + { + log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno)); + fclose(fp); + continue; + } + fclose(fp); + JSON_COPY_STR_TO_ARRAY(imported_files, extract_files[i].destination); + } + } + + // Remove pihole.toml to prevent it from being imported on restart + if(remove(GLOBALTOMLPATH) != 0) + log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); // Free allocated memory free_upload_data(data); + // Signal FTL we want to restart for re-import + api->ftl.restart = true; + // Send response cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); + JSON_ADD_ITEM_TO_OBJECT(json, "files", imported_files); JSON_SEND_OBJECT(json); } diff --git a/src/config/config.h b/src/config/config.h index dc493739..18fa6e14 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -38,6 +38,9 @@ // characters will be replaced by their UTF-8 escape sequences (UCS-2) #define TOML_UTF8 +// Location of the legacy (pre-v6.0) config file +#define GLOBALCONFFILE_LEGACY "/etc/pihole/pihole-FTL.conf" + union conf_value { bool b; // boolean value int i; // integer value diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index bf4600f8..ce09f3df 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -43,7 +43,7 @@ static FILE * __attribute__((nonnull(1), malloc, warn_unused_result)) openFTLcon return fp; // Local file not present, try system file - *path = "/etc/pihole/pihole-FTL.conf"; + *path = GLOBALCONFFILE_LEGACY; fp = fopen(*path, "r"); return fp; From 41f01ae4c9fabdb86ab8008d3988ca81bd2d4449 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:44:40 +0100 Subject: [PATCH 08/55] Relax filename constraints in archive type detection Signed-off-by: DL6ER --- src/api/teleporter.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 4d7c8e86..f0916340 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -223,17 +223,11 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something that claims to be a ZIP archive - // - filename - // - should be at least 12 characters long, - // - should start in "pi-hole_", - // - have "_teleporter_" in the middle, and - // - end in ".zip" + // - filename should end in ".zip" // - the data itself // - should be at least 40 bytes long // - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT) - if(strlen(data.filename) >= 12 && - strncmp(data.filename, "pi-hole_", 8) == 0 && - strstr(data.filename, "_teleporter_") != NULL && + if(strlen(data.filename) > 4 && strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 && data.filesize >= 40 && memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0) @@ -241,17 +235,11 @@ static int api_teleporter_POST(struct ftl_conn *api) return process_received_zip(api, &data); } // Check if we received something that claims to be a TAR.GZ archive - // - filename - // - should be at least 12 characters long, - // - should start in "pi-hole-", - // - have "-teleporter_" in the middle, and - // - end in ".tar.gz" + // - filename should end in ".tar.gz" // - the data itself // - should be at least 40 bytes long // - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt) - else if(strlen(data.filename) >= 12 && - strncmp(data.filename, "pi-hole-", 8) == 0 && - strstr(data.filename, "-teleporter_") != NULL && + else if(strlen(data.filename) > 7 && strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 && data.filesize >= 40 && memcmp(data.data, "\x1f\x8b", 2) == 0) From f30c1e89a8fa2dab6efcc4bf618cf0b1bf839b79 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:56:13 +0100 Subject: [PATCH 09/55] Also remove all rotated files in light of the upcoming https://github.com/pi-hole/FTL/pull/1738 Signed-off-by: DL6ER --- src/api/teleporter.c | 18 ++++++++++++++++++ src/files.c | 2 -- src/files.h | 1 + 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index f0916340..6279c5c3 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -23,6 +23,8 @@ #include "database/sqlite3.h" // dbquery() #include "database/common.h" +// MAX_ROTATIONS +#include "files.h" #define MAXFILESIZE (50u*1024*1024) @@ -680,6 +682,22 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(remove(GLOBALTOMLPATH) != 0) log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); + // Remove all rotated pihole.toml files to avoid automatic config + // restore on restart + for(unsigned int i = MAX_ROTATIONS; i > 0; i--) + { + const char *fname = GLOBALTOMLPATH; + const char *filename = basename(fname); + // extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999") + const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6; + char *path = calloc(buflen, sizeof(char)); + snprintf(path, buflen, BACKUP_DIR"/%s.%u", filename, i); + + // Remove file (if it exists) + if(remove(path) != 0 && errno != ENOENT) + log_err("Unable to remove file \"%s\": %s", path, strerror(errno)); + } + // Free allocated memory free_upload_data(data); diff --git a/src/files.c b/src/files.c index 525f57e3..bce995c3 100644 --- a/src/files.c +++ b/src/files.c @@ -32,8 +32,6 @@ #include #include -#define BACKUP_DIR "/etc/pihole/config_backups" - // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an // octal number representing the bit pattern for the new mode bits diff --git a/src/files.h b/src/files.h index 329ce876..2e99b6ee 100644 --- a/src/files.h +++ b/src/files.h @@ -17,6 +17,7 @@ #define ZIP_ROTATIONS 3 #define MAX_ROTATIONS 15 +#define BACKUP_DIR "/etc/pihole/config_backups" bool chmod_file(const char *filename, const mode_t mode); bool file_exists(const char *filename); From 65aef156cd06575f76ed4516a0b0c8ab6339f11b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 19:38:59 +0100 Subject: [PATCH 10/55] Add parsing support for NULL-values Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 19 +++++++++++------- src/api/teleporter.c | 35 +++++++++++++++++++++------------ src/zip/tar.c | 1 - 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2710e65a..8c2469a6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,14 +2,19 @@ "name": "FTL x86_64 Build Env", "image": "ghcr.io/pi-hole/ftl-build:v2.3-alpine", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], - "extensions": [ - "jetmartin.bats", - "ms-vscode.cpptools", - "ms-vscode.cmake-tools", - "eamodio.gitlens" - ], + "customizations": { + "vscode": { + "extensions": [ + "jetmartin.bats", + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "eamodio.gitlens" + ] + } + }, "mounts": [ - "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly" + "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", + "type=bind,source=/var/www/html,target=/var/www/html,readonly" ] } diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 6279c5c3..f726318b 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -313,13 +313,13 @@ static struct teleporter_files { .filename = "blacklist.exact.json", .table_name = "domainlist", .listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "blacklist.regex.json", .table_name = "domainlist", .listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "client.json", @@ -349,13 +349,13 @@ static struct teleporter_files { .filename = "whitelist.exact.json", .table_name = "domainlist", .listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "whitelist.regex.json", .table_name = "domainlist", .listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } } }; @@ -371,11 +371,6 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) // Check if the JSON array is empty, if so, we can return early const int num_entries = cJSON_GetArraySize(json); - if(num_entries == 0) - { - log_info("import_json_table(%s): JSON array is empty", file->filename); - return true; - } // Check if all the JSON entries contain all the expected columns cJSON *json_object = NULL; @@ -437,6 +432,7 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) if(file->listtype < 0) { // Delete all entries in the table + log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\"", file->filename, file->table_name); if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK) { log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", @@ -448,6 +444,7 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) else { // Delete all entries in the table of the same type + log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\" of type %d", file->filename, file->table_name, file->listtype); if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK) { log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", @@ -540,9 +537,20 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) return false; } } + else if(cJSON_IsNull(json_value)) + { + // Bind NULL value + if(sqlite3_bind_null(stmt, i + 1) != SQLITE_OK) + { + log_err("Unable to bind NULL value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } else { - log_err("Unable to bind value to SQL statement: %s", sqlite3_errmsg(db)); + log_err("Unable to bind value to SQL statement: type = %X", (unsigned int)json_value->type & 0xFF); sqlite3_finalize(stmt); sqlite3_close(db); return false; @@ -604,12 +612,11 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat "The uploaded file does not appear to be a valid gzip archive - decompression failed"); } - // Check if the decompressed data is a valid TAR archive - cJSON *json_files = list_files_in_tar(archive, archive_size); - // Print all files in the TAR archive if in debug mode if(config.debug.api.v.b) { + cJSON *json_files = list_files_in_tar(archive, archive_size); + cJSON *file = NULL; cJSON_ArrayForEach(file, json_files) { @@ -661,6 +668,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(file != NULL && fileSize > 0u) { // Write file to disk + log_debug(DEBUG_API, "Writing file \"%s\" (%zu bytes) to \"%s\"", + extract_files[i].archive_name, fileSize, extract_files[i].destination); FILE *fp = fopen(extract_files[i].destination, "wb"); if(fp == NULL) { diff --git a/src/zip/tar.c b/src/zip/tar.c index be9a9e51..5e497622 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -118,7 +118,6 @@ cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, c newOffset += TAR_BLOCK_SIZE; // Add file name to cJSON array - log_info("Found file '%s' with size %zu", name, size); cJSON *file = cJSON_CreateObject(); cJSON_AddItemToObject(file, "name", cJSON_CreateString(name)); cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size)); From 2c765c94bb8a8dd06ac87310d86438ee43228dd4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 12:04:48 +0100 Subject: [PATCH 11/55] Add WEB_PORTS to setupVars.conf when importing v5 Teleporter files Signed-off-by: DL6ER --- src/api/teleporter.c | 10 ++++++++ src/config/config.c | 59 +++++++++++++++++++++++--------------------- src/config/config.h | 1 + src/main.c | 2 +- src/setupVars.c | 4 +++ 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index f726318b..2350eb28 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -687,6 +687,16 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat } } + // Append WEB_PORTS to setupVars.conf + FILE *fp = fopen(config.files.setupVars.v.s, "a"); + if(fp == NULL) + log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno)); + else + { + fprintf(fp, "WEB_PORT=%s\n", config.webserver.port.v.s); + fclose(fp); + } + // Remove pihole.toml to prevent it from being imported on restart if(remove(GLOBALTOMLPATH) != 0) log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); diff --git a/src/config/config.c b/src/config/config.c index 3d1cf57c..d2a133e8 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1385,36 +1385,39 @@ void readFTLconf(struct config *conf, const bool rewrite) rename(GLOBALTOMLPATH, new_name); } - // Determine default webserver ports - // Check if ports 80/TCP and 443/TCP are already in use - const in_port_t http_port = port_in_use(80) ? 8080 : 80; - const in_port_t https_port = port_in_use(443) ? 8443 : 443; - - // Create a string with the default ports - // Allocate memory for the string - char *ports = calloc(32, sizeof(char)); - if(ports == NULL) + // Determine default webserver ports if not imported from setupVars.conf + if(!(config.webserver.port.f & FLAG_CONF_IMPORTED)) { - log_err("Unable to allocate memory for default ports string"); - return; + // Check if ports 80/TCP and 443/TCP are already in use + const in_port_t http_port = port_in_use(80) ? 8080 : 80; + const in_port_t https_port = port_in_use(443) ? 8443 : 443; + + // Create a string with the default ports + // Allocate memory for the string + char *ports = calloc(32, sizeof(char)); + if(ports == NULL) + { + log_err("Unable to allocate memory for default ports string"); + return; + } + // Create the string + snprintf(ports, 32, "%d,%ds", http_port, https_port); + + // Append IPv6 ports if IPv6 is enabled + const bool have_ipv6 = ipv6_enabled(); + if(have_ipv6) + snprintf(ports + strlen(ports), 32 - strlen(ports), + ",[::]:%d,[::]:%ds", http_port, https_port); + + // Set default values for webserver ports + if(conf->webserver.port.t == CONF_STRING_ALLOCATED) + free(conf->webserver.port.v.s); + conf->webserver.port.v.s = ports; + conf->webserver.port.t = CONF_STRING_ALLOCATED; + + log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", + http_port, https_port, have_ipv6 ? "enabled" : "disabled"); } - // Create the string - snprintf(ports, 32, "%d,%ds", http_port, https_port); - - // Append IPv6 ports if IPv6 is enabled - const bool have_ipv6 = ipv6_enabled(); - if(have_ipv6) - snprintf(ports + strlen(ports), 32 - strlen(ports), - ",[::]:%d,[::]:%ds", http_port, https_port); - - // Set default values for webserver ports - if(conf->webserver.port.t == CONF_STRING_ALLOCATED) - free(conf->webserver.port.v.s); - conf->webserver.port.v.s = ports; - conf->webserver.port.t = CONF_STRING_ALLOCATED; - - log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", - http_port, https_port, have_ipv6 ? "enabled" : "disabled"); // Initialize the TOML config file writeFTLtoml(true); diff --git a/src/config/config.h b/src/config/config.h index 18fa6e14..9cee391a 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -97,6 +97,7 @@ enum conf_type { #define FLAG_INVALIDATE_SESSIONS (1 << 3) #define FLAG_WRITE_ONLY (1 << 4) #define FLAG_ENV_VAR (1 << 5) +#define FLAG_CONF_IMPORTED (1 << 6) struct conf_item { const char *k; // item Key diff --git a/src/main.c b/src/main.c index 22e57fe5..88d7caee 100644 --- a/src/main.c +++ b/src/main.c @@ -185,7 +185,7 @@ int main (int argc, char *argv[]) cleanup(exit_code); if(exit_code == RESTART_FTL_CODE) - execv(argv[0], argv); + execvp(argv[0], argv); return exit_code; } diff --git a/src/setupVars.c b/src/setupVars.c index ca069d5f..a4069c76 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -35,6 +35,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co free(conf_item->v.s); conf_item->v.s = strdup(setupVarsValue); conf_item->t = CONF_STRING_ALLOCATED; + conf_item->f |= FLAG_CONF_IMPORTED; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -380,6 +381,9 @@ void importsetupVarsConf(void) get_conf_bool_from_setupVars("DHCP_RAPID_COMMIT", &config.dhcp.rapidCommit); get_conf_bool_from_setupVars("queryLogging", &config.dns.queryLogging); + + // Ports may be temporarily stored when importing a legacy Teleporter v5 file + get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); } char* __attribute__((pure)) find_equals(char *s) From 268146d9c7701bd2b5e43870accb0cc11a07ce2e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 18:14:10 +0100 Subject: [PATCH 12/55] Move the setupVars.conf file to setupVars.conf.old Signed-off-by: DL6ER --- src/api/teleporter.c | 6 +++--- src/config/legacy_reader.c | 5 ++++- src/setupVars.c | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 2350eb28..12eb2ad1 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -668,8 +668,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(file != NULL && fileSize > 0u) { // Write file to disk - log_debug(DEBUG_API, "Writing file \"%s\" (%zu bytes) to \"%s\"", - extract_files[i].archive_name, fileSize, extract_files[i].destination); + log_info("Writing file \"%s\" (%zu bytes) to \"%s\"", + extract_files[i].archive_name, fileSize, extract_files[i].destination); FILE *fp = fopen(extract_files[i].destination, "wb"); if(fp == NULL) { @@ -693,7 +693,7 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno)); else { - fprintf(fp, "WEB_PORT=%s\n", config.webserver.port.v.s); + fprintf(fp, "WEB_PORTS=%s\n", config.webserver.port.v.s); fclose(fp); } diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index ce09f3df..f796131c 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -113,9 +113,12 @@ const char *readFTLlegacy(struct config *conf) const char *path = NULL; FILE *fp = openFTLconf(&path); if(fp == NULL) + { + log_warn("No readable FTL config file found, using default settings"); return NULL; + } - log_notice("Reading legacy config file"); + log_info("Reading legacy config files from %s", path); // MAXDBDAYS // defaults to: 365 days diff --git a/src/setupVars.c b/src/setupVars.c index a4069c76..92fd7104 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -312,6 +312,8 @@ static void get_conf_listeningMode_from_setupVars(void) void importsetupVarsConf(void) { + log_info("Migrating config from %s", config.files.setupVars.v.s); + // Try to obtain password hash from setupVars.conf get_conf_string_from_setupVars("WEBPASSWORD", &config.webserver.api.pwhash); @@ -384,6 +386,21 @@ void importsetupVarsConf(void) // Ports may be temporarily stored when importing a legacy Teleporter v5 file get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); + + // Move the setupVars.conf file to setupVars.conf.old + char *old_setupVars = calloc(strlen(config.files.setupVars.v.s) + 5, sizeof(char)); + if(old_setupVars == NULL) + { + log_warn("Could not allocate memory for old_setupVars"); + return; + } + strcpy(old_setupVars, config.files.setupVars.v.s); + strcat(old_setupVars, ".old"); + if(rename(config.files.setupVars.v.s, old_setupVars) != 0) + log_warn("Could not move %s to %s", config.files.setupVars.v.s, old_setupVars); + else + log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars); + free(old_setupVars); } char* __attribute__((pure)) find_equals(char *s) From 0ef89ab3eba3b3509b7f40843cafadefccf1cc8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Dec 2023 23:27:20 +0100 Subject: [PATCH 13/55] Implement special POST :batchDelete callbacks for /api/groups, /api/domains/, /api/clients, and /api/lists Signed-off-by: DL6ER --- src/api/api.c | 140 +++++----- src/api/docs/content/specs/clients.yaml | 55 +++- src/api/docs/content/specs/domains.yaml | 58 ++++ src/api/docs/content/specs/groups.yaml | 61 +++++ src/api/docs/content/specs/lists.yaml | 50 ++++ src/api/docs/content/specs/main.yaml | 12 + src/api/list.c | 213 ++++++++++++++- src/api/stats_database.c | 2 +- src/database/gravity-db.c | 337 +++++++++++++++++------- src/database/gravity-db.h | 2 +- src/webserver/http-common.h | 10 +- 11 files changed, 771 insertions(+), 169 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 98eedf32..bb26c2a4 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -30,74 +30,78 @@ static struct { bool require_auth; enum http_method methods; } api_request[] = { - // URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS - // domains json fifo + // URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS + // flags fifo ID // Note: The order of appearance matters here, more specific URIs have to // appear *before* less specific URIs: 1. "/a/b/c", 2. "/a/b", 3. "/a" - { "/api/auth/sessions", "", api_auth_sessions, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth/session", "/{id}", api_auth_session_delete, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/auth/app", "", generateAppPw, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth/totp", "", generateTOTP, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth", "", api_auth, { false, true, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE }, - { "/api/dns/blocking", "", api_dns_blocking, { false, true, 0 }, true, HTTP_GET | HTTP_POST }, - { "/api/clients/_suggestions", "", api_client_suggestions, { false, true, 0 }, true, HTTP_GET }, - { "/api/clients", "/{client}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/clients", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/domains", "/{type}/{kind}/{domain}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/domains", "/{type}/{kind}", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/search", "/{domain}", api_search, { false, true, 0 }, true, HTTP_GET }, - { "/api/groups", "/{name}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/groups", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/lists", "/{list}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/lists", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/info/client", "", api_info_client, { false, true, 0 }, false, HTTP_GET }, - { "/api/info/login", "", api_info_login, { false, true, 0 }, false, HTTP_GET }, - { "/api/info/system", "", api_info_system, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/database", "", api_info_database, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/sensors", "", api_info_sensors, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/host", "", api_info_host, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/ftl", "", api_info_ftl, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/version", "", api_info_version, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/messages/count", "", api_info_messages_count, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/messages", "/{message_id}", api_info_messages, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/info/messages", "", api_info_messages, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/metrics", "", api_info_metrics, { false, true, 0 }, true, HTTP_GET }, - { "/api/logs/dnsmasq", "", api_logs, { false, true, FIFO_DNSMASQ }, true, HTTP_GET }, - { "/api/logs/ftl", "", api_logs, { false, true, FIFO_FTL }, true, HTTP_GET }, - { "/api/logs/webserver", "", api_logs, { false, true, FIFO_WEBSERVER }, true, HTTP_GET }, - { "/api/history/clients", "", api_history_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/history/database/clients", "", api_history_database_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/history/database", "", api_history_database, { false, true, 0 }, true, HTTP_GET }, - { "/api/history", "", api_history, { false, true, 0 }, true, HTTP_GET }, - { "/api/queries/suggestions", "", api_queries_suggestions, { false, true, 0 }, true, HTTP_GET }, - { "/api/queries", "", api_queries, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/summary", "", api_stats_summary, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/query_types", "", api_stats_query_types, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/upstreams", "", api_stats_upstreams, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/top_domains", "", api_stats_top_domains, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/top_clients", "", api_stats_top_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/recent_blocked", "", api_stats_recentblocked, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/top_domains", "", api_stats_database_top_items, { true, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/top_clients", "", api_stats_database_top_items, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/summary", "", api_stats_database_summary, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/query_types", "", api_stats_database_query_types, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/upstreams", "", api_stats_database_upstreams, { false, true, 0 }, true, HTTP_GET }, - { "/api/config", "", api_config, { false, true, 0 }, true, HTTP_GET | HTTP_PATCH }, - { "/api/config", "/{element}", api_config, { false, true, 0 }, true, HTTP_GET }, - { "/api/config", "/{element}/{value}", api_config, { false, true, 0 }, true, HTTP_DELETE | HTTP_PUT }, - { "/api/network/gateway", "", api_network_gateway, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/interfaces", "", api_network_interfaces, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/devices", "", api_network_devices, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/devices", "/{device_id}", api_network_devices, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/endpoints", "", api_endpoints, { false, true, 0 }, true, HTTP_GET }, - { "/api/teleporter", "", api_teleporter, { false, false, 0 }, true, HTTP_GET | HTTP_POST }, - { "/api/dhcp/leases", "", api_dhcp_leases_GET, { false, true, 0 }, true, HTTP_GET }, - { "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/action/gravity", "", api_action_gravity, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/restartdns", "", api_action_restartDNS, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/flush/logs", "", api_action_flush_logs, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/flush/arp", "", api_action_flush_arp, { false, true, 0 }, true, HTTP_POST }, - { "/api/docs", "", api_docs, { false, true, 0 }, false, HTTP_GET }, + { "/api/auth/sessions", "", api_auth_sessions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth/session", "/{id}", api_auth_session_delete, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/auth/app", "", generateAppPw, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth/totp", "", generateTOTP, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth", "", api_auth, { API_PARSE_JSON, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE }, + { "/api/dns/blocking", "", api_dns_blocking, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_POST }, + { "/api/clients/_suggestions", "", api_client_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/clients", "/{client}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/clients", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/clients:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/domains", "/{type}/{kind}/{domain}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/domains", "/{type}/{kind}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/domains:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/search", "/{domain}", api_search, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/groups", "/{name}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/groups", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/groups:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/lists", "/{list}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/lists", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/lists:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/info/client", "", api_info_client, { API_PARSE_JSON, 0 }, false, HTTP_GET }, + { "/api/info/login", "", api_info_login, { API_PARSE_JSON, 0 }, false, HTTP_GET }, + { "/api/info/system", "", api_info_system, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/database", "", api_info_database, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/sensors", "", api_info_sensors, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/host", "", api_info_host, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/ftl", "", api_info_ftl, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/version", "", api_info_version, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/messages/count", "", api_info_messages_count, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/messages", "/{message_id}", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/info/messages", "", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/metrics", "", api_info_metrics, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/logs/dnsmasq", "", api_logs, { API_PARSE_JSON, FIFO_DNSMASQ }, true, HTTP_GET }, + { "/api/logs/ftl", "", api_logs, { API_PARSE_JSON, FIFO_FTL }, true, HTTP_GET }, + { "/api/logs/webserver", "", api_logs, { API_PARSE_JSON, FIFO_WEBSERVER }, true, HTTP_GET }, + { "/api/history/clients", "", api_history_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history/database/clients", "", api_history_database_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history/database", "", api_history_database, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history", "", api_history, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/queries/suggestions", "", api_queries_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/queries", "", api_queries, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/summary", "", api_stats_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/query_types", "", api_stats_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/upstreams", "", api_stats_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/top_domains", "", api_stats_top_domains, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/top_clients", "", api_stats_top_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/recent_blocked", "", api_stats_recentblocked, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/top_domains", "", api_stats_database_top_items, { API_DOMAINS | API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/top_clients", "", api_stats_database_top_items, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/summary", "", api_stats_database_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/query_types", "", api_stats_database_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/upstreams", "", api_stats_database_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/config", "", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PATCH }, + { "/api/config", "/{element}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/config", "/{element}/{value}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_DELETE | HTTP_PUT }, + { "/api/network/gateway", "", api_network_gateway, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/interfaces", "", api_network_interfaces, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/devices", "", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/devices", "/{device_id}", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/endpoints", "", api_endpoints, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/teleporter", "", api_teleporter, { API_FLAG_NONE, 0 }, true, HTTP_GET | HTTP_POST }, + { "/api/dhcp/leases", "", api_dhcp_leases_GET, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/action/gravity", "", api_action_gravity, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/restartdns", "", api_action_restartDNS, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/flush/logs", "", api_action_flush_logs, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/flush/arp", "", api_action_flush_arp, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/docs", "", api_docs, { API_PARSE_JSON, 0 }, false, HTTP_GET }, }; int api_handler(struct mg_connection *conn, void *ignored) @@ -113,7 +117,7 @@ int api_handler(struct mg_connection *conn, void *ignored) double_time(), { false, NULL, NULL, NULL, 0u }, { false }, - { false, false, 0 } + { API_FLAG_NONE, 0 } }; log_debug(DEBUG_API, "Requested API URI: %s -> %s %s ? %s (Content-Type %s)", @@ -149,7 +153,7 @@ int api_handler(struct mg_connection *conn, void *ignored) continue; } - if(api_request[i].opts.parse_json) + if(api_request[i].opts.flags & API_PARSE_JSON) { // Allocate memory for the payload api.payload.raw = calloc(MAX_PAYLOAD_BYTES, sizeof(char)); diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 557a68a2..d7619c5b 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -149,7 +149,7 @@ components: Creates a new client in the `clients` object. The `{client}` itself is specified in the request body (POST JSON). Clients may be described either by their IP addresses (IPv4 and IPv6 are supported), - IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).

+ IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`). Note that client recognition by IP addresses (incl. subnet ranges) is preferred over MAC address, host name or interface recognition as the two latter will only be available after some time. Furthermore, MAC address recognition only works for devices at most one networking hop away from your Pi-hole. @@ -199,6 +199,59 @@ components: allOf: - $ref: 'common.yaml#/components/schemas/took' - $ref: 'common.yaml#/components/errors/unauthorized' + batchDelete: + post: + summary: Delete multiple clients + tags: + - "Client management" + operationId: "batchDelete_clients" + description: | + Deletes multiple clients in the `clients` object. The `{client}`s themselves are specified in the request body (POST JSON). + + Clients may be described either by their IP addresses (IPv4 and IPv6 are supported), + IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).

+ + *Note:* There will be no content on success. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: client IP / MAC / hostname / interface + example: + - "item": "192.168.2.5" + - "item": "::1" + - "item": "12:34:56:78:9A:BC" + - "item": "localhost" + - "item": ":eth0" + responses: + '204': + description: Items deleted + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: clients: get: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index 773cbeba..b67cf0e6 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -212,6 +212,64 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + summary: Delete multiple domains + post: + summary: Delete multiple domains + tags: + - "Domain management" + operationId: "batchDelete_domains" + description: | + *Note:* There will be no content on success. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: Domain to delete + example: "example.com" + type: + type: string + description: Type of domain to delete + enum: + - "allow" + - "deny" + example: "allow" + kind: + type: string + description: Kind of domain to delete + enum: + - "exact" + - "regex" + example: "exact" + responses: + '204': + description: Items deleted + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: domains: get: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index c70571c6..499a238c 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -165,6 +165,67 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + post: + summary: Delete multiple groups + tags: + - "Group management" + operationId: "batchDelete_groups" + description: | + Deletes multiple groups in the `groups` object. The `{groups}` themselves are specified in the request body (POST JSON). + + On success, a new resource is created at `/groups/{name}`. + + The `database_error` with message `UNIQUE constraint failed` error indicates that a group with the same name already exists. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: group name + example: + - "item": "test1" + - "item": "test2" + responses: + '201': + description: Created item + content: + application/json: + schema: + allOf: + - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET + - $ref: 'groups.yaml#/components/schemas/lists_processed' + - $ref: 'common.yaml#/components/schemas/took' + headers: + Location: + $ref: 'common.yaml#/components/headers/Location' + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload' + duplicate: + $ref: 'groups.yaml#/components/examples/errors/database_error/duplicate' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: groups: get: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index f6536a76..55a95425 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -170,6 +170,56 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + post: + summary: Delete lists + tags: + - "List management" + operationId: "batchDelete_lists" + description: | + Deletes multiple lists in the `lists` object. The `{list}`s themselves are specified in the request body (POST JSON). + + On success, a new resource is created at `/lists/{list}`. + + The `database_error` with message `UNIQUE constraint failed` error indicates that this list already exists. + requestBody: + description: Callback payload + content: + application/json: + schema: + $ref: 'lists.yaml#/components/schemas/lists/post' + responses: + '201': + description: Created item + content: + application/json: + schema: + allOf: + - $ref: 'lists.yaml#/components/schemas/lists/get' + - $ref: 'lists.yaml#/components/schemas/lists_processed' + - $ref: 'common.yaml#/components/schemas/took' + headers: + Location: + $ref: 'common.yaml#/components/headers/Location' + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: lists: get: diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index 3e8e1b0d..1762aed4 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -142,18 +142,27 @@ paths: /domains/{type}/{kind}: $ref: 'domains.yaml#/components/paths/type_kind' + /domains:batchDelete: + $ref: 'domains.yaml#/components/paths/batchDelete' + /groups/{name}: $ref: 'groups.yaml#/components/paths/name' /groups: $ref: 'groups.yaml#/components/paths/direct' + /groups:batchDelete: + $ref: 'groups.yaml#/components/paths/batchDelete' + /clients/{client}: $ref: 'clients.yaml#/components/paths/client' /clients: $ref: 'clients.yaml#/components/paths/direct' + /clients:batchDelete: + $ref: 'clients.yaml#/components/paths/batchDelete' + /clients/_suggestions: $ref: 'clients.yaml#/components/paths/suggestions' @@ -163,6 +172,9 @@ paths: /lists: $ref: 'lists.yaml#/components/paths/direct' + /lists:batchDelete: + $ref: 'lists.yaml#/components/paths/batchDelete' + /info/client: $ref: 'info.yaml#/components/paths/client' diff --git a/src/api/list.c b/src/api/list.c index f0185846..a7cef124 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -503,7 +503,7 @@ static int api_list_write(struct ftl_conn *api, cJSON_AddItemToArray(okay ? success : errors, details); } - // Inform the resolver that it needs to reload the domainlists + // Inform the resolver that it needs to reload gravity set_event(RELOAD_GRAVITY); int response_code = 201; // 201 - Created @@ -525,21 +525,195 @@ static int api_list_remove(struct ftl_conn *api, const char *item) { const char *sql_msg = NULL; - if(gravityDB_delFromTable(listtype, item, &sql_msg)) + cJSON *array = api->payload.json; + bool allocated_json = false; + + // If this is not a :batchDelete call, then the item is specified in the + // URI, not in the payload. Create a JSON array with the item and use + // that instead + const bool isBatchDelete = api->opts.flags & API_BATCHDELETE; + + // If this is a domain callback, we need to translate type/kind into an + // integer for use in the database + if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT || + listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX || + listtype == GRAVITY_DOMAINLIST_DENY_REGEX) { - // Inform the resolver that it needs to reload the domainlists + int type = -1; + switch (listtype) + { + case GRAVITY_DOMAINLIST_ALLOW_EXACT: + type = 0; + break; + case GRAVITY_DOMAINLIST_DENY_EXACT: + type = 1; + break; + case GRAVITY_DOMAINLIST_ALLOW_REGEX: + type = 2; + break; + case GRAVITY_DOMAINLIST_DENY_REGEX: + type = 3; + case GRAVITY_GROUPS: + case GRAVITY_ADLISTS: + case GRAVITY_CLIENTS: + // No type required for these tables + break; + // Aggregate types cannot be handled by this routine + case GRAVITY_GRAVITY: + case GRAVITY_ANTIGRAVITY: + case GRAVITY_DOMAINLIST_ALLOW_ALL: + case GRAVITY_DOMAINLIST_DENY_ALL: + case GRAVITY_DOMAINLIST_ALL_EXACT: + case GRAVITY_DOMAINLIST_ALL_REGEX: + case GRAVITY_DOMAINLIST_ALL_ALL: + default: + return false; + } + + // Create new JSON array with the item and type: + // array = [{"item": "example.com", "type": 0}] + array = cJSON_CreateArray(); + cJSON *obj = cJSON_CreateObject(); + cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item)); + cJSON_AddItemToObject(obj, "type", cJSON_CreateNumber(type)); + cJSON_AddItemToArray(array, obj); + allocated_json = true; + } + else if(isBatchDelete && listtype == GRAVITY_DOMAINLIST_ALL_ALL) + { + // Loop over all items and parse type/kind for each item + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + if(!cJSON_IsObject(it)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects", + NULL); + } + + // Check if item is a string + cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item"); + if(!cJSON_IsString(json_item)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"item\" as string", + NULL); + } + + // Check if type and kind are both present and strings + cJSON *json_type = cJSON_GetObjectItemCaseSensitive(it, "type"); + cJSON *json_kind = cJSON_GetObjectItemCaseSensitive(it, "kind"); + if(!cJSON_IsString(json_type) || !cJSON_IsString(json_kind)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"type\" and \"kind\" as string", + NULL); + } + + // Parse type and kind + // 0 = allow exact + // 1 = deny exact + // 2 = allow regex + // 3 = deny regex + int type = -1; + if(strcasecmp(json_type->valuestring, "allow") == 0) + { + if(strcasecmp(json_kind->valuestring, "exact") == 0) + type = 0; + else if(strcasecmp(json_kind->valuestring, "regex") == 0) + type = 2; + } + else if(strcasecmp(json_type->valuestring, "deny") == 0) + { + if(strcasecmp(json_kind->valuestring, "exact") == 0) + type = 1; + else if(strcasecmp(json_kind->valuestring, "regex") == 0) + type = 3; + } + + // Check if type/kind combination is valid + if(type == -1) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an valid combination of \"type\" and \"kind\" for each object", + NULL); + } + + // Replace type/kind with integer type + // array = [{"item": "example.com", "type": 0}] + cJSON_DeleteItemFromObject(it, "type"); + cJSON_DeleteItemFromObject(it, "kind"); + cJSON_AddNumberToObject(it, "type", type); + } + } + else if(!isBatchDelete) + { + // Create array with object (used for clients, groups, lists) + // array = [{"item": }] + array = cJSON_CreateArray(); + cJSON *obj = cJSON_CreateObject(); + cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item)); + cJSON_AddItemToArray(array, obj); + allocated_json = true; + } + + // Verify that the payload is an array of objects each containing an + // item + if(isBatchDelete) + { + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + if(!cJSON_IsObject(it)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects", + NULL); + } + + // Check if item is a string + cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item"); + if(!cJSON_IsString(json_item)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"item\" as string", + NULL); + } + } + } + + // From here on, we can assume the JSON payload is valid + if(gravityDB_delFromTable(listtype, array, &sql_msg)) + { + // Inform the resolver that it needs to reload gravity set_event(RELOAD_GRAVITY); + // Free memory allocated above + if(allocated_json) + cJSON_free(array); + // Send empty reply with code 204 No Content cJSON *json = JSON_NEW_OBJECT(); JSON_SEND_OBJECT_CODE(json, 204); } else { + // Free memory allocated above + if(allocated_json) + cJSON_free(array); + // Send error reply return send_json_error(api, 400, "database_error", - "Could not remove domain from database table", + "Could not remove entries from table", sql_msg); } } @@ -548,21 +722,40 @@ int api_list(struct ftl_conn *api) { enum gravity_list_type listtype; bool can_modify = false; + bool batchDelete = false; if((api->item = startsWith("/api/groups", api)) != NULL) { listtype = GRAVITY_GROUPS; can_modify = true; } + else if((api->item = startsWith("/api/groups:batchDelete", api)) != NULL) + { + listtype = GRAVITY_GROUPS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/lists", api)) != NULL) { listtype = GRAVITY_ADLISTS; can_modify = true; } + else if((api->item = startsWith("/api/lists:batchDelete", api)) != NULL) + { + listtype = GRAVITY_ADLISTS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/clients", api)) != NULL) { listtype = GRAVITY_CLIENTS; can_modify = true; } + else if((api->item = startsWith("/api/clients:batchDelete", api)) != NULL) + { + listtype = GRAVITY_CLIENTS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/domains/allow/exact", api)) != NULL) { listtype = GRAVITY_DOMAINLIST_ALLOW_EXACT; @@ -603,6 +796,12 @@ int api_list(struct ftl_conn *api) { listtype = GRAVITY_DOMAINLIST_ALL_ALL; } + else if((api->item = startsWith("/api/domains:batchDelete", api)) != NULL) + { + listtype = GRAVITY_DOMAINLIST_ALL_ALL; + can_modify = true; + batchDelete = true; + } else { return send_json_error(api, 400, @@ -643,7 +842,7 @@ int api_list(struct ftl_conn *api) return ret; } } - else if(can_modify && api->method == HTTP_POST) + else if(can_modify && api->method == HTTP_POST && !batchDelete) { // Add item to list identified by payload if(api->item != NULL && strlen(api->item) != 0) @@ -651,7 +850,7 @@ int api_list(struct ftl_conn *api) return send_json_error(api, 400, "uri_error", "Invalid request: Specify item in payload, not as URI parameter", - NULL); + api->item); } else { @@ -664,7 +863,7 @@ int api_list(struct ftl_conn *api) return ret; } } - else if(can_modify && api->method == HTTP_DELETE) + else if(can_modify && (api->method == HTTP_DELETE || (api->method == HTTP_POST && batchDelete))) { // Delete item from list // We would not actually need the SHM lock here, however, we do diff --git a/src/api/stats_database.c b/src/api/stats_database.c index aef2fed6..f6fdcf17 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -181,7 +181,7 @@ int api_stats_database_top_items(struct ftl_conn *api) // Get options from API struct bool blocked = false; // Can be overwritten by query string - const bool domains = api->opts.domains; + const bool domains = api->opts.flags & API_DOMAINS; // Get parameters from query string if(api->request->query_string != NULL) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 12558f0b..ed508fd7 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1746,15 +1746,15 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { if(strcasecmp("allow", row->type) == 0 && strcasecmp("exact", row->kind) == 0) - oldtype = 0; + oldtype = 0; else if(strcasecmp("deny", row->type) == 0 && strcasecmp("exact", row->kind) == 0) - oldtype = 1; + oldtype = 1; else if(strcasecmp("allow", row->type) == 0 && strcasecmp("regex", row->kind) == 0) - oldtype = 2; + oldtype = 2; else if(strcasecmp("deny", row->type) == 0 && - strcasecmp("regex", row->kind) == 0) + strcasecmp("regex", row->kind) == 0) oldtype = 3; else { @@ -1838,7 +1838,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, return okay; } -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* argument, const char **message) +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message) { if(gravity_db == NULL) { @@ -1846,123 +1846,282 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* a return false; } - int type = -1; - switch (listtype) + // Return early if passed JSON argument is not an array + if(!cJSON_IsArray(array)) { - case GRAVITY_DOMAINLIST_ALLOW_EXACT: - type = 0; - break; - case GRAVITY_DOMAINLIST_DENY_EXACT: - type = 1; - break; - case GRAVITY_DOMAINLIST_ALLOW_REGEX: - type = 2; - break; - case GRAVITY_DOMAINLIST_DENY_REGEX: - type = 3; - break; - - case GRAVITY_GROUPS: - case GRAVITY_ADLISTS: - case GRAVITY_CLIENTS: - // No type required for these tables - break; - - // Aggregate types cannot be handled by this routine - case GRAVITY_GRAVITY: - case GRAVITY_ANTIGRAVITY: - case GRAVITY_DOMAINLIST_ALLOW_ALL: - case GRAVITY_DOMAINLIST_DENY_ALL: - case GRAVITY_DOMAINLIST_ALL_EXACT: - case GRAVITY_DOMAINLIST_ALL_REGEX: - case GRAVITY_DOMAINLIST_ALL_ALL: - default: - return false; + *message = "Argument is not an array"; + log_err("gravityDB_delFromTable(%d): %s", + listtype, *message); + return false; } - // Prepare SQLite statement + const bool isDomain = listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT || + listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX || + listtype == GRAVITY_DOMAINLIST_DENY_REGEX || + listtype == GRAVITY_DOMAINLIST_ALL_ALL; // batch delete + + // Begin transaction + const char *querystr = "BEGIN TRANSACTION;"; + int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + return false; + } + + // Create temporary table for JSON argument + if(isDomain) + // Create temporary table for domains to be deleted + querystr = "CREATE TEMPORARY TABLE deltable (type INT, item TEXT);"; + else + querystr = "CREATE TEMPORARY TABLE deltable (item TEXT);"; + sqlite3_stmt* stmt = NULL; - const char *querystr[3] = {NULL, NULL, NULL}; - if(listtype == GRAVITY_GROUPS) - querystr[0] = "DELETE FROM \"group\" WHERE name = :argument;"; - else if(listtype == GRAVITY_ADLISTS) + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) { - // This is actually a three-step deletion to satisfy foreign-key constraints - querystr[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);"; - querystr[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);"; - querystr[2] = "DELETE FROM adlist WHERE address = :argument;"; + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", + listtype, querystr, *message); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; } - else if(listtype == GRAVITY_CLIENTS) - querystr[0] = "DELETE FROM client WHERE ip = :argument;"; - else // domainlist - querystr[0] = "DELETE FROM domainlist WHERE domain = :argument AND type = :type;"; - bool okay = true; - for(unsigned int i = 0; i < ArraySize(querystr); i++) + // Execute statement + if((rc = sqlite3_step(stmt)) != SQLITE_DONE) { - // Finish if no more queries - if(querystr[i] == NULL) - break; - - // We need to perform a second SQL request - int rc = sqlite3_prepare_v2(gravity_db, querystr[i], -1, &stmt, NULL); - if( rc != SQLITE_OK ) + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s", + listtype, querystr, *message); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s) - SQL error prepare %u (%i): %s", - type, argument, i, rc, *message); - return false; + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); } + return false; + } - // Bind domain to prepared statement (if requested) - const int arg_idx = sqlite3_bind_parameter_index(stmt, ":argument"); - if(arg_idx > 0 && (rc = sqlite3_bind_text(stmt, arg_idx, argument, -1, SQLITE_STATIC)) != SQLITE_OK) + // Finalize statement + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + + // Prepare statement for inserting items into virtual table + if(isDomain) + querystr = "INSERT INTO deltable (type, item) VALUES (:type, :item);"; + else + querystr = "INSERT INTO deltable (item) VALUES (:item);"; + + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", + listtype, querystr, *message); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s): Failed to bind argument %u (error %d) - %s", - type, argument, i, rc, *message); - sqlite3_reset(stmt); - sqlite3_finalize(stmt); - return false; + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); } + return false; + } - // Bind type to prepared statement (if requested) + // Loop over all domains in the JSON array + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + // Bind type to prepared statement + cJSON *type = cJSON_GetObjectItemCaseSensitive(it, "type"); const int type_idx = sqlite3_bind_parameter_index(stmt, ":type"); - if(type_idx > 0 && (rc = sqlite3_bind_int(stmt, type_idx, type)) != SQLITE_OK) + if(type_idx > 0 && (!cJSON_IsNumber(type) || (rc = sqlite3_bind_int(stmt, type_idx, type->valueint)) != SQLITE_OK)) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s): Failed to bind type (2) (error %d) - %s", - type, argument, rc, *message); + log_err("gravityDB_delFromTable(%d): Failed to bind type (error %d) - %s", + type->valueint, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + type->valueint, querystr, *message); + } return false; } + // Bind item to prepared statement + cJSON *item = cJSON_GetObjectItemCaseSensitive(it, "item"); + const int item_idx = sqlite3_bind_parameter_index(stmt, ":item"); + if(item_idx > 0 && (!cJSON_IsString(item) || (rc = sqlite3_bind_text(stmt, item_idx, item->valuestring, -1, SQLITE_STATIC)) != SQLITE_OK)) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): Failed to bind item (error %d) - %s", + listtype, rc, *message); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; + } + + // Execute statement + if((rc = sqlite3_step(stmt)) != SQLITE_DONE) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s", + listtype, querystr, *message); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; + } + + // Reset statement + sqlite3_reset(stmt); + // Debug output if(config.debug.api.v.b) { - log_debug(DEBUG_API, "SQL: %s", querystr[i]); - if(arg_idx > 0) - log_debug(DEBUG_API, " :argument = \"%s\"", argument); + log_debug(DEBUG_API, "SQL: %s", querystr); + if(item_idx > 0) + log_debug(DEBUG_API, " :item = \"%s\"", item->valuestring); if(type_idx > 0) - log_debug(DEBUG_API, " :type = \"%i\"", type); + log_debug(DEBUG_API, " :type = %i", cJSON_IsNumber(type) ? type->valueint : -1); } + } - // Perform step - okay = false; - if((rc = sqlite3_step(stmt)) == SQLITE_DONE) - { - // Item removed - okay = true; - } - else + // Finalize statement + sqlite3_finalize(stmt); + + // Prepare SQL for deleting items from the requested table + const char *querystrs[4] = {NULL, NULL, NULL, NULL}; + if(listtype == GRAVITY_GROUPS) + querystrs[0] = "DELETE FROM \"group\" WHERE name IN (SELECT item FROM deltable);"; + else if(listtype == GRAVITY_ADLISTS) + { + // This is actually a three-step deletion to satisfy foreign-key constraints + querystrs[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));"; + querystrs[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));"; + querystrs[2] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable);"; + } + else if(listtype == GRAVITY_CLIENTS) + querystrs[0] = "DELETE FROM client WHERE ip IN (SELECT item FROM deltable);"; + else // domainlist + { + querystrs[0] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 0) AND type = 0;"; + querystrs[1] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 1) AND type = 1;"; + querystrs[2] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 2) AND type = 2;"; + querystrs[3] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 3) AND type = 3;"; + } + + bool okay = true; + for(unsigned int i = 0; i < ArraySize(querystrs); i++) + { + // Finish if no more queries + if(querystrs[i] == NULL) + break; + + // Execute statement + rc = sqlite3_exec(gravity_db, querystrs[i], NULL, NULL, NULL); + if(rc != SQLITE_OK) { *message = sqlite3_errmsg(gravity_db); - } + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystrs[i], *message); + okay = false; - // Finalize statement - sqlite3_reset(stmt); - sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec: %s", + listtype, *message); + } + + break; + } + } + + // Drop temporary table + querystr = "DROP TABLE deltable;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + okay = false; + + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + } + + // Commit transaction + querystr = "COMMIT TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + okay = false; + + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } } return okay; diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 8b77fdec..7024cdec 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -70,7 +70,7 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow * void gravityDB_readTableFinalize(void); bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, const char **message, const enum http_method method); -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* domain_name, const char **message); +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message); bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups, const tablerow *row, const char **message); diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index d10ce72b..a710dba2 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -33,9 +33,15 @@ enum http_method { HTTP_OPTIONS = 1 << 5, }; +enum api_flags { + API_FLAG_NONE = 0, + API_DOMAINS = 1 << 0, + API_PARSE_JSON = 1 << 1, + API_BATCHDELETE = 1 << 2, +}; + struct api_options { - bool domains :1; - bool parse_json :1; + enum api_flags flags; enum fifo_logs which; }; From e9a55f8836433304c839f338050bf0adc9a5ad5c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 10:08:30 +0100 Subject: [PATCH 14/55] Only include as many domains as we have in the sorted array Signed-off-by: DL6ER --- src/api/stats.c | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index e68be488..d83a9461 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -139,16 +139,6 @@ int api_stats_summary(struct ftl_conn *api) int api_stats_top_domains(struct ftl_conn *api) { - int count = 10; - bool audit = false; - const int domains = counters->domains; - int *temparray = calloc(2*domains, sizeof(int*)); - if(temparray == NULL) - { - log_err("Memory allocation failed in %s()", __FUNCTION__); - return 0; - } - // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS) { @@ -160,10 +150,23 @@ int api_stats_top_domains(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); cJSON *top_domains = JSON_NEW_ARRAY(); JSON_ADD_ITEM_TO_OBJECT(json, "top_domains", top_domains); - free(temparray); JSON_SEND_OBJECT(json); } + // Lock shared memory + lock_shm(); + + // Allocate memory + int count = 10; + bool audit = false; + const int domains = counters->domains; + int *temparray = calloc(2*domains, sizeof(int*)); + if(temparray == NULL) + { + log_err("Memory allocation failed in %s()", __FUNCTION__); + return 0; + } + bool blocked = false; // Can be overwritten by query string // /api/stats/top_domains?blocked=true if(api->request->query_string != NULL) @@ -179,10 +182,8 @@ int api_stats_top_domains(struct ftl_conn *api) get_bool_var(api->request->query_string, "audit", &audit); } - // Lock shared memory - lock_shm(); - - for(int domainID=0; domainID < domains; domainID++) + unsigned int added_domains = 0u; + for(int domainID = 0; domainID < domains; domainID++) { // Get domain pointer const domainsData* domain = getDomain(domainID, true); @@ -195,10 +196,12 @@ int api_stats_top_domains(struct ftl_conn *api) else // Count only permitted queries temparray[2*domainID + 1] = (domain->count - domain->blockedcount); + + added_domains++; } // Sort temporary array - qsort(temparray, domains, sizeof(int[2]), cmpdesc); + qsort(temparray, added_domains, sizeof(int[2]), cmpdesc); // Get filter const char* filter = read_setupVarsconf("API_QUERY_LOG_SHOW"); @@ -222,7 +225,7 @@ int api_stats_top_domains(struct ftl_conn *api) int n = 0; cJSON *top_domains = JSON_NEW_ARRAY(); - for(int i = 0; i < domains; i++) + for(unsigned int i = 0; i < added_domains; i++) { // Get sorted index const int domainID = temparray[2*i + 0]; From bc48f63ed0d1b5959d9746e8beca58fc7ac08cde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:18:38 +0000 Subject: [PATCH 15/55] Bump actions/upload-artifact from 3.1.3 to 4.0.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.0.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3.1.3...v4.0.0) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f4adf2f..766931f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,7 +119,7 @@ jobs: - name: Store binary artifacts for later deployoment if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v3.1.3 + uses: actions/upload-artifact@v4.0.0 with: name: tmp-storage path: '${{ matrix.bin_name }}*' @@ -131,7 +131,7 @@ jobs: - name: Upload documentation artifacts for deployoment if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64' - uses: actions/upload-artifact@v3.1.3 + uses: actions/upload-artifact@v4.0.0 with: name: tmp-storage path: 'api-docs.tar.gz' From a1cf6e4ff05fe3dd0006daaff9bd8d3cd986f2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 16 Dec 2023 23:14:42 +0100 Subject: [PATCH 16/55] Adjust workflow for upload/download v4 changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 766931f5..b6a2d586 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,7 @@ jobs: if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4.0.0 with: - name: tmp-storage + name: ${{ matrix.bin_name }}-binary path: '${{ matrix.bin_name }}*' - name: Extract documentation files from container @@ -133,7 +133,7 @@ jobs: if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64' uses: actions/upload-artifact@v4.0.0 with: - name: tmp-storage + name: api-docs path: 'api-docs.tar.gz' deploy: @@ -146,15 +146,21 @@ jobs: uses: actions/checkout@v4.1.1 - name: Get Binaries and documentation built in previous jobs - uses: actions/download-artifact@v3.0.2 + uses: actions/download-artifact@v4.0.0 id: download with: - name: tmp-storage - path: ftl-builds/ + path: download/ - name: Display structure of downloaded files run: ls -R working-directory: ${{steps.download.outputs.download-path}} + + - + name: Copy all artifacts from sub-directories to ftl_builds/ + run: | + mkdir ftl_builds/ + cp ${{steps.download.outputs.download-path}}/**/* ftl_builds/ + - name: Install SSH Key uses: benoitchantre/setup-ssh-authentication-action@1.0.1 @@ -163,14 +169,14 @@ jobs: known-hosts: ${{ secrets.KNOWN_HOSTS }} - name: Untar documentation files - working-directory: ${{steps.download.outputs.download-path}} + working-directory: ftl_builds/ run: | mkdir docs/ tar xzvf api-docs.tar.gz -C docs/ - name: Display structure of files ready for upload run: ls -R - working-directory: ${{steps.download.outputs.download-path}} + working-directory: ftl_builds/ - name: Transfer Builds to Pi-hole server for pihole checkout if: github.actor != 'dependabot[bot]' @@ -178,7 +184,7 @@ jobs: USER: ${{ secrets.SSH_USER }} HOST: ${{ secrets.SSH_HOST }} TARGET_DIR: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }} - SOURCE_DIR: ${{ steps.download.outputs.download-path }} + SOURCE_DIR: ftl_builds/ run: | bash ./deploy.sh - @@ -187,4 +193,4 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - ${{ steps.download.outputs.download-path }}/* + ftl_builds/* From 7915b0a4a0b01ec2a4138b822d152ece28476acd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 20 Dec 2023 17:15:41 +0100 Subject: [PATCH 17/55] Fix one-of definition in clients, domains, groups, and lists POST request payloads Signed-off-by: DL6ER --- src/api/docs/content/specs/clients.yaml | 34 ++++++++++++----------- src/api/docs/content/specs/domains.yaml | 34 ++++++++++++----------- src/api/docs/content/specs/groups.yaml | 37 +++++++++++++------------ src/api/docs/content/specs/lists.yaml | 36 ++++++++++++------------ 4 files changed, 75 insertions(+), 66 deletions(-) diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index d7619c5b..0228696c 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -262,7 +262,7 @@ components: description: Array of clients items: allOf: - - $ref: 'clients.yaml#/components/schemas/client' + - $ref: 'clients.yaml#/components/schemas/client_object' - $ref: 'clients.yaml#/components/schemas/comment' - $ref: 'clients.yaml#/components/schemas/groups' - $ref: 'clients.yaml#/components/schemas/readonly' @@ -305,25 +305,27 @@ components: description: Comma-separated list of hostnames (if available) example: "localhost,ip6-localhost" client: - type: object - properties: - client: - description: client IP / MAC / hostname / interface - type: string - example: 127.0.0.1 + description: client IP / MAC / hostname / interface + type: string + example: 127.0.0.1 client_array: + description: array of client IPs / MACs / hostnames / interfaces + type: array + items: + type: string + example: ["127.0.0.1", "192.168.2.12"] + client_maybe_array: type: object properties: client: - description: array of client IPs / MACs / hostnames / interfaces - type: array - items: - type: string - example: ["127.0.0.1", "192.168.2.12"] - client_maybe_array: - oneOf: - - $ref: 'clients.yaml#/components/schemas/client' - - $ref: 'clients.yaml#/components/schemas/client_array' + oneOf: + - $ref: 'clients.yaml#/components/schemas/client' + - $ref: 'clients.yaml#/components/schemas/client_array' + client_object: + type: object + properties: + client: + $ref: 'clients.yaml#/components/schemas/client' comment: type: object properties: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index b67cf0e6..115232dc 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -280,7 +280,7 @@ components: description: Array of domains items: allOf: - - $ref: 'domains.yaml#/components/schemas/domain' + - $ref: 'domains.yaml#/components/schemas/domain_object' - $ref: 'domains.yaml#/components/schemas/unicode' - $ref: 'domains.yaml#/components/schemas/type' - $ref: 'domains.yaml#/components/schemas/kind' @@ -302,12 +302,9 @@ components: - $ref: 'domains.yaml#/components/schemas/groups' - $ref: 'domains.yaml#/components/schemas/enabled' domain: - type: object - properties: - domain: - description: Domain - type: string - example: testdomain.com + description: Domain + type: string + example: testdomain.com unicode: type: object properties: @@ -316,18 +313,23 @@ components: type: string example: "äbc.com" domain_array: + description: array of domains + type: array + items: + type: string + example: ["testdomain.com", "otherdomain.de"] + domain_maybe_array: type: object properties: domain: - description: array of domains - type: array - items: - type: string - example: ["testdomain.com", "otherdomain.de"] - domain_maybe_array: - oneOf: - - $ref: 'domains.yaml#/components/schemas/domain' - - $ref: 'domains.yaml#/components/schemas/domain_array' + oneOf: + - $ref: 'domains.yaml#/components/schemas/domain' + - $ref: 'domains.yaml#/components/schemas/domain_array' + domain_object: + type: object + properties: + domain: + $ref: 'domains.yaml#/components/schemas/domain' type: type: object properties: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index 499a238c..20e61ae6 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -235,13 +235,14 @@ components: type: array items: allOf: - - $ref: 'groups.yaml#/components/schemas/name' + - $ref: 'groups.yaml#/components/schemas/name_object' - $ref: 'groups.yaml#/components/schemas/comment' - $ref: 'groups.yaml#/components/schemas/enabled' - $ref: 'groups.yaml#/components/schemas/readonly' put: allOf: - - $ref: 'groups.yaml#/components/schemas/name' + # Can rename group + - $ref: 'groups.yaml#/components/schemas/name_object' - $ref: 'groups.yaml#/components/schemas/comment' - $ref: 'groups.yaml#/components/schemas/enabled' post: @@ -250,25 +251,27 @@ components: - $ref: 'groups.yaml#/components/schemas/comment' - $ref: 'groups.yaml#/components/schemas/enabled' name: - type: object - properties: - name: - description: Group name - type: string - example: test_group + description: Group name + type: string + example: test_group name_array: + description: array of group names + type: array + items: + type: string + example: ["test1", "test2", "test3"] + name_maybe_array: type: object properties: name: - description: array of group names - type: array - items: - type: string - example: ["test1", "test2", "test3"] - name_maybe_array: - oneOf: - - $ref: 'groups.yaml#/components/schemas/name' - - $ref: 'groups.yaml#/components/schemas/name_array' + oneOf: + - $ref: 'groups.yaml#/components/schemas/name' + - $ref: 'groups.yaml#/components/schemas/name_array' + name_object: + type: object + properties: + name: + $ref: 'groups.yaml#/components/schemas/name' comment: type: object properties: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index 55a95425..b5019702 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -230,7 +230,7 @@ components: description: Array of lists items: allOf: - - $ref: 'lists.yaml#/components/schemas/list' + - $ref: 'lists.yaml#/components/schemas/address_object' - $ref: 'lists.yaml#/components/schemas/type' - $ref: 'lists.yaml#/components/schemas/comment' - $ref: 'lists.yaml#/components/schemas/groups' @@ -244,31 +244,33 @@ components: - $ref: 'lists.yaml#/components/schemas/enabled' post: allOf: - - $ref: 'lists.yaml#/components/schemas/list_maybe_array' + - $ref: 'lists.yaml#/components/schemas/address_maybe_array' - $ref: 'lists.yaml#/components/schemas/type' - $ref: 'lists.yaml#/components/schemas/comment' - $ref: 'lists.yaml#/components/schemas/groups' - $ref: 'lists.yaml#/components/schemas/enabled' - list: + address: + description: Address of the list + type: string + example: https://hosts-file.net/ad_servers.txt + address_array: + description: array of list addresses + type: array + items: + type: string + example: ["https://hosts-file.net/ad_servers.txt"] + address_maybe_array: type: object properties: address: - description: Address of the list - type: string - example: https://hosts-file.net/ad_servers.txt - list_array: + oneOf: + - $ref: 'lists.yaml#/components/schemas/address' + - $ref: 'lists.yaml#/components/schemas/address_array' + address_object: type: object properties: - list: - description: array of list addresses - type: array - items: - type: string - example: ["https://hosts-file.net/ad_servers.txt"] - list_maybe_array: - oneOf: - - $ref: 'lists.yaml#/components/schemas/list' - - $ref: 'lists.yaml#/components/schemas/list_array' + address: + $ref: 'lists.yaml#/components/schemas/address' type: type: object properties: From 2872ffc161ffec8df2256218868acaf053068610 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Dec 2023 10:48:12 +0000 Subject: [PATCH 18/55] Bump actions/download-artifact from 4.0.0 to 4.1.0 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6a2d586..3e386994 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -146,7 +146,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Get Binaries and documentation built in previous jobs - uses: actions/download-artifact@v4.0.0 + uses: actions/download-artifact@v4.1.0 id: download with: path: download/ From 6c921e75ae9fd5245613f8ac2b64c7bd8a87fbe2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Dec 2023 00:20:33 +0100 Subject: [PATCH 19/55] Use WAL, remove (and strip) SQLite3 shared-cache support Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- src/database/query-table.c | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35c5c332..3883bc4d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,6 +24,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster. # SQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do. # SQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster. +# SQLITE_OMIT_SHARED_CACHE: This option builds SQLite without support for shared cache mode. The sqlite3_enable_shared_cache() is omitted along with a fair amount of logic within the B-Tree subsystem associated with shared cache management. This compile-time option is recommended most applications as it results in improved performance and reduced library footprint. # SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections. # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption. @@ -31,7 +32,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection. # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384") +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/database/query-table.c b/src/database/query-table.c index 8a4fa654..d3f6fc29 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -60,7 +60,7 @@ void db_counts(unsigned long *last_idx, unsigned long *mem_num, unsigned long *d bool init_memory_database(void) { int rc; - const char *uri = "file:memdb?mode=memory&cache=shared"; + const char *uri = "file:memdb?mode=memory"; // Try to open in-memory database rc = sqlite3_open_v2(uri, &memdb, SQLITE_OPEN_READWRITE, NULL); @@ -81,6 +81,21 @@ bool init_memory_database(void) return false; } + // Change journal mode to WAL + // - WAL is significantly faster in most scenarios. + // - WAL provides more concurrency as readers do not block writers and a + // writer does not block readers. Reading and writing can proceed + // concurrently. + // - Disk I/O operations tends to be more sequential using WAL. + rc = sqlite3_exec(memdb, "PRAGMA journal_mode=WAL", NULL, NULL, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(): Step error while trying to set journal mode: %s", + sqlite3_errstr(rc)); + sqlite3_close(memdb); + return false; + } + // Create query_storage table in the database for(unsigned int i = 0; i < ArraySize(table_creation); i++) { From f6832444446525f27796ec7cc5bc8bdf84c09b3a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Dec 2023 00:38:27 +0100 Subject: [PATCH 20/55] Attach disk database once when initializing memory database and don't bother detaching it - it will finally be detached when FTL terminates Signed-off-by: DL6ER --- src/api/info.c | 2 +- src/api/queries.c | 50 --------------------------- src/database/query-table.c | 70 ++++++-------------------------------- src/database/query-table.h | 6 ++-- 4 files changed, 14 insertions(+), 114 deletions(-) diff --git a/src/api/info.c b/src/api/info.c index 1c2505b7..61398084 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -147,7 +147,7 @@ int api_info_database(struct ftl_conn *api) JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner); // Add number of queries in on-disk database - const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", true); + const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage"); JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database); // Add SQLite library version diff --git a/src/api/queries.c b/src/api/queries.c index d35c56a7..528b905e 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -444,23 +444,11 @@ int api_queries(struct ftl_conn *api) // Finish preparing query string querystr_finish(querystr, sort_col, sort_dir); - // Attach disk database if necessary - const char *message = ""; - if(disk && !attach_disk_database(&message)) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, cannot attach disk database", - message); - } - // Prepare SQLite3 statement sqlite3_stmt *read_stmt = NULL; int rc = sqlite3_prepare_v2(db, querystr, -1, &read_stmt, NULL); if( rc != SQLITE_OK ) { - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to prepare read SQL query", @@ -484,8 +472,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind timestamp:from to SQL query", @@ -501,8 +487,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind timestamp:until to SQL query", @@ -518,8 +502,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind domain to SQL query", @@ -535,8 +517,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind cip to SQL query", @@ -552,8 +532,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind client to SQL query", @@ -569,8 +547,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind upstream to SQL query", @@ -595,8 +571,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind type to SQL query", @@ -605,8 +579,6 @@ int api_queries(struct ftl_conn *api) } else { - if(disk) - detach_disk_database(NULL); return send_json_error(api, 400, "bad_request", "Requested type is invalid", @@ -631,8 +603,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind status to SQL query", @@ -641,8 +611,6 @@ int api_queries(struct ftl_conn *api) } else { - if(disk) - detach_disk_database(NULL); return send_json_error(api, 400, "bad_request", "Requested status is invalid", @@ -667,8 +635,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind reply to SQL query", @@ -677,8 +643,6 @@ int api_queries(struct ftl_conn *api) } else { - if(disk) - detach_disk_database(NULL); return send_json_error(api, 400, "bad_request", "Requested reply is invalid", @@ -703,8 +667,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind dnssec to SQL query", @@ -713,8 +675,6 @@ int api_queries(struct ftl_conn *api) } else { - if(disk) - detach_disk_database(NULL); return send_json_error(api, 400, "bad_request", "Requested dnssec is invalid", @@ -731,8 +691,6 @@ int api_queries(struct ftl_conn *api) { sqlite3_reset(read_stmt); sqlite3_finalize(read_stmt); - if(disk) - detach_disk_database(NULL); return send_json_error(api, 500, "internal_error", "Internal server error, failed to bind count to SQL query", @@ -901,13 +859,5 @@ int api_queries(struct ftl_conn *api) // Finalize statements sqlite3_finalize(read_stmt); - if(disk && !detach_disk_database(&message)) - { - return send_json_error(api, 500, - "internal_error", - "Internal server error, cannot detach disk database", - message); - } - JSON_SEND_OBJECT(json); } diff --git a/src/database/query-table.c b/src/database/query-table.c index d3f6fc29..0f66db38 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -123,6 +123,10 @@ bool init_memory_database(void) } } + // Attach disk database + if(!attach_database(memdb, NULL, config.files.database.v.s, "disk")) + return false; + // Everything went well return true; } @@ -203,7 +207,7 @@ static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries) *memsize = page_count * page_size; // Get number of queries in the memory table - if((*queries = get_number_of_queries_in_DB(db, "query_storage", false)) == DB_FAILED) + if((*queries = get_number_of_queries_in_DB(db, "query_storage")) == DB_FAILED) return false; return true; @@ -227,11 +231,6 @@ static void log_in_memory_usage(void) } } -// Attach disk database to in-memory database -bool attach_disk_database(const char **message) -{ - return attach_database(memdb, message, config.files.database.v.s, "disk"); -} // Attach database using specified path and alias bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias) @@ -292,12 +291,6 @@ bool attach_database(sqlite3* db, const char **message, const char *path, const return okay; } -// Detach disk database to in-memory database -bool detach_disk_database(const char **message) -{ - return detach_database(memdb, message, "disk"); -} - // Detach a previously attached database by its alias bool detach_database(sqlite3* db, const char **message, const char *alias) { @@ -348,13 +341,10 @@ bool detach_database(sqlite3* db, const char **message, const char *alias) // Get number of queries either in the temp or in the on-diks database // This routine is used by the API routines. -int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool do_attach) +int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) { int rc = 0, num = 0; sqlite3_stmt *stmt = NULL; - // Attach disk database if required - if(do_attach && !attach_disk_database(NULL)) - return DB_FAILED; // Count number of rows const size_t buflen = 42 + strlen(tablename); @@ -373,8 +363,6 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool d log_err("get_number_of_queries_in_DB(%s): Prepare error: %s", tablename, sqlite3_errstr(rc)); free(querystr); - if(do_attach) - detach_disk_database(NULL); return false; } rc = sqlite3_step(stmt); @@ -386,17 +374,11 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool d tablename, sqlite3_errstr(rc)); free(querystr); sqlite3_finalize(stmt); - if(do_attach) - detach_disk_database(NULL); return false; } sqlite3_finalize(stmt); free(querystr); - // Detach only if attached herein - if(do_attach && !detach_disk_database(NULL)) - return DB_FAILED; - return num; } @@ -410,16 +392,11 @@ bool import_queries_from_disk(void) const double mintime = now - config.webserver.api.maxHistory.v.ui; const char *querystr = "INSERT INTO query_storage SELECT * FROM disk.query_storage WHERE timestamp >= ?"; - // Attach disk database - if(!attach_disk_database(NULL)) - return false; - // Begin transaction int rc; if((rc = sqlite3_exec(memdb, "BEGIN TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) { log_err("import_queries_from_disk(): Cannot start transaction: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } @@ -427,7 +404,6 @@ bool import_queries_from_disk(void) sqlite3_stmt *stmt = NULL; if((rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL)) != SQLITE_OK){ log_err("import_queries_from_disk(): SQL error prepare: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } @@ -436,7 +412,6 @@ bool import_queries_from_disk(void) { log_err("import_queries_from_disk(): Failed to bind type mintime: %s", sqlite3_errstr(rc)); sqlite3_finalize(stmt); - detach_disk_database(NULL); return false; } @@ -479,16 +454,12 @@ bool import_queries_from_disk(void) if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) { log_err("import_queries_from_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } // Get number of queries on disk before detaching - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", false); - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", false); - - if(!detach_disk_database(NULL)) - return false; + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); + mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); @@ -509,10 +480,6 @@ bool export_queries_to_disk(bool final) // Start database timer timer_start(DATABASE_WRITE_TIMER); - // Attach disk database - if(!attach_disk_database(NULL)) - return false; - // Start transaction SQL_bool(memdb, "BEGIN TRANSACTION"); @@ -521,7 +488,6 @@ bool export_queries_to_disk(bool final) int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ){ log_err("export_queries_to_disk(): SQL error prepare: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } @@ -529,7 +495,6 @@ bool export_queries_to_disk(bool final) if((rc = sqlite3_bind_int64(stmt, 1, last_disk_db_idx)) != SQLITE_OK) { log_err("export_queries_to_disk(): Failed to bind id: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } @@ -540,7 +505,6 @@ bool export_queries_to_disk(bool final) if((rc = sqlite3_bind_double(stmt, 2, time)) != SQLITE_OK) { log_err("export_queries_to_disk(): Failed to bind time: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } @@ -604,16 +568,11 @@ bool export_queries_to_disk(bool final) if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) { log_err("export_queries_to_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); - detach_disk_database(NULL); return false; } // Update number of queries in the disk database - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", false); - - // Detach disk database - if(!detach_disk_database(NULL)) - return false; + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); // All temp queries were stored to disk, update the IDs last_disk_db_idx += insertions; @@ -672,7 +631,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) mintime, sqlite3_errstr(rc)); // Update number of queries in in-memory database - const int new_num = get_number_of_queries_in_DB(memdb, "query_storage", false); + const int new_num = get_number_of_queries_in_DB(memdb, "query_storage"); log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i", sqlite3_changes(db), (mem_db_num - new_num), new_num); mem_db_num = new_num; @@ -1231,10 +1190,6 @@ void update_disk_db_idx(void) // starting counting from zero (would result in a UNIQUE constraint violation) const char *querystr = "SELECT MAX(id) FROM disk.query_storage"; - // Attach disk database - if(!attach_disk_database(NULL)) - return; - // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); @@ -1251,9 +1206,6 @@ void update_disk_db_idx(void) log_debug(DEBUG_DATABASE, "Last long-term idx is %lu", last_disk_db_idx); - if(!detach_disk_database(NULL)) - return; - // Update indices so that the next call to DB_save_queries() skips the // queries that we just imported from the database last_mem_db_idx = last_disk_db_idx; @@ -1591,7 +1543,7 @@ bool queries_to_database(void) } // Update number of queries in in-memory database - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", false); + mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); if(config.debug.database.v.b && updated + added > 0) { diff --git a/src/database/query-table.h b/src/database/query-table.h index e9b29a54..bd6e831b 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -11,7 +11,7 @@ #define QUERY_TABLE_PRIVATE_H // struct queriesData -#include "../datastructure.h" +#include "datastructure.h" #define CREATE_FTL_TABLE "CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );" @@ -111,11 +111,9 @@ bool init_memory_database(void); sqlite3 *get_memdb(void) __attribute__((pure)); void close_memory_database(void); bool import_queries_from_disk(void); -bool attach_disk_database(const char **msg); bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias); -bool detach_disk_database(const char **msg); bool detach_database(sqlite3* db, const char **message, const char *alias); -int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool do_attach); +int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename); bool export_queries_to_disk(bool final); bool delete_old_queries_from_db(const bool use_memdb, const double mintime); bool add_additional_info_column(sqlite3 *db); From 2665da72f483032b233be80be78824e1b2ad80a8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 8 Dec 2023 18:59:02 +0100 Subject: [PATCH 21/55] Remove SQLite3 URI feature - we do not need it any longer. It is not part of the regular SQLite3 shell builds. Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 +- src/api/queries.c | 17 +++- src/database/message-table.c | 19 +--- src/database/query-table.c | 176 ++++++++++++++++++++++++++++------- src/database/session-table.c | 25 ++--- 5 files changed, 171 insertions(+), 69 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3883bc4d..fdc7b7f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,10 +29,9 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption. # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime). -# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection. # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384") +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/api/queries.c b/src/api/queries.c index 528b905e..dcdb8507 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -34,8 +34,8 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char * "Could not read from in-memory database", NULL); } - sqlite3_stmt *stmt; + sqlite3_stmt *stmt = NULL; int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ) { @@ -438,15 +438,22 @@ int api_queries(struct ftl_conn *api) } } - // Get connection to in-memory database - sqlite3 *db = get_memdb(); - // Finish preparing query string querystr_finish(querystr, sort_col, sort_dir); + // Get connection to in-memory database + sqlite3 *memdb = get_memdb(); + if(memdb == NULL) + { + return send_json_error(api, 500, // 500 Internal error + "database_error", + "Could not read from in-memory database", + NULL); + } + // Prepare SQLite3 statement sqlite3_stmt *read_stmt = NULL; - int rc = sqlite3_prepare_v2(db, querystr, -1, &read_stmt, NULL); + int rc = sqlite3_prepare_v2(memdb, querystr, -1, &read_stmt, NULL); if( rc != SQLITE_OK ) { return send_json_error(api, 500, diff --git a/src/database/message-table.c b/src/database/message-table.c index 6696fcd7..3461b200 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -27,6 +27,8 @@ #include "gc.h" // get_filesystem_details() #include "files.h" +// get_memdb() +#include "database/query-table.h" static const char *get_message_type_str(const enum message_type type) { @@ -214,23 +216,10 @@ bool create_message_table(sqlite3 *db) // Flush message table bool flush_message_table(void) { - // Return early if database is known to be broken - if(FTLDBerror()) - return false; - - sqlite3 *db; - // Open database connection - if((db = dbopen(false, false)) == NULL) - { - log_err("flush_message_table() - Failed to open DB"); - return false; - } + sqlite3 *memdb = get_memdb(); // Flush message table - SQL_bool(db, "DELETE FROM message;"); - - // Close database connection - dbclose(&db); + SQL_bool(memdb, "DELETE FROM disk.message;"); return true; } diff --git a/src/database/query-table.c b/src/database/query-table.c index 0f66db38..8b0ed937 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -22,7 +22,7 @@ #include "database/common.h" #include "timers.h" -static sqlite3 *memdb = NULL; +static sqlite3 *_memdb = NULL; static double new_last_timestamp = 0; static unsigned int new_total = 0, new_blocked = 0; static unsigned long last_mem_db_idx = 0, last_disk_db_idx = 0; @@ -60,10 +60,8 @@ void db_counts(unsigned long *last_idx, unsigned long *mem_num, unsigned long *d bool init_memory_database(void) { int rc; - const char *uri = "file:memdb?mode=memory"; - // Try to open in-memory database - rc = sqlite3_open_v2(uri, &memdb, SQLITE_OPEN_READWRITE, NULL); + rc = sqlite3_open_v2(":memory:", &_memdb, SQLITE_OPEN_READWRITE, NULL); if( rc != SQLITE_OK ) { log_err("init_memory_database(): Step error while trying to open database: %s", @@ -72,27 +70,12 @@ bool init_memory_database(void) } // Explicitly set busy handler to value defined in FTL.h - rc = sqlite3_busy_timeout(memdb, DATABASE_BUSY_TIMEOUT); + rc = sqlite3_busy_timeout(_memdb, DATABASE_BUSY_TIMEOUT); if( rc != SQLITE_OK ) { log_err("init_memory_database(): Step error while trying to set busy timeout (%d ms): %s", DATABASE_BUSY_TIMEOUT, sqlite3_errstr(rc)); - sqlite3_close(memdb); - return false; - } - - // Change journal mode to WAL - // - WAL is significantly faster in most scenarios. - // - WAL provides more concurrency as readers do not block writers and a - // writer does not block readers. Reading and writing can proceed - // concurrently. - // - Disk I/O operations tends to be more sequential using WAL. - rc = sqlite3_exec(memdb, "PRAGMA journal_mode=WAL", NULL, NULL, NULL); - if( rc != SQLITE_OK ) - { - log_err("init_memory_database(): Step error while trying to set journal mode: %s", - sqlite3_errstr(rc)); - sqlite3_close(memdb); + sqlite3_close(_memdb); return false; } @@ -100,11 +83,11 @@ bool init_memory_database(void) for(unsigned int i = 0; i < ArraySize(table_creation); i++) { log_debug(DEBUG_DATABASE, "init_memory_database(): Executing %s", table_creation[i]); - rc = sqlite3_exec(memdb, table_creation[i], NULL, NULL, NULL); + rc = sqlite3_exec(_memdb, table_creation[i], NULL, NULL, NULL); if( rc != SQLITE_OK ){ log_err("init_memory_database(\"%s\") failed: %s", table_creation[i], sqlite3_errstr(rc)); - sqlite3_close(memdb); + sqlite3_close(_memdb); return false; } } @@ -114,18 +97,132 @@ bool init_memory_database(void) for(unsigned int i = 0; i < ArraySize(index_creation); i++) { log_debug(DEBUG_DATABASE, "init_memory_database(): Executing %s", index_creation[i]); - rc = sqlite3_exec(memdb, index_creation[i], NULL, NULL, NULL); + rc = sqlite3_exec(_memdb, index_creation[i], NULL, NULL, NULL); if( rc != SQLITE_OK ){ log_err("init_memory_database(\"%s\") failed: %s", index_creation[i], sqlite3_errstr(rc)); - sqlite3_close(memdb); + sqlite3_close(_memdb); return false; } } // Attach disk database - if(!attach_database(memdb, NULL, config.files.database.v.s, "disk")) + if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk")) return false; +/* + // Change journal mode to WAL + // - WAL is significantly faster in most scenarios. + // - WAL provides more concurrency as readers do not block writers and a + // writer does not block readers. Reading and writing can proceed + // concurrently. + // - Disk I/O operations tends to be more sequential using WAL. + rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(): Step error while trying to set journal mode: %s", + sqlite3_errstr(rc)); + sqlite3_close(_memdb); + return false; + } +*/ +/* + // Change synchronous mode to NORMAL + // - NORMAL is the fastest synchronous mode + // - NORMAL still provides full ACID (atomicity, consistency, isolation, + // and durability) properties. + rc = sqlite3_exec(_memdb, "PRAGMA disk.synchronous=NORMAL", NULL, NULL, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(): Step error while trying to set synchronous mode: %s", + sqlite3_errstr(rc)); + sqlite3_close(_memdb); + return false; + } +*/ + // Get result of PRAGMA journal_mode + const unsigned char *journal_mode = NULL; + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(_memdb, "PRAGMA journal_mode", -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("init_memory_database(PRAGMA journal_mode): Prepare error: %s", + sqlite3_errstr(rc)); + return false; + } + rc = sqlite3_step(stmt); + if( rc == SQLITE_ROW ) + journal_mode = sqlite3_column_text(stmt, 0); + else + { + log_err("init_memory_database(PRAGMA journal_mode): Step error: %s", + sqlite3_errstr(rc)); + return false; + } + log_info("Using %s journal mode for memory database", journal_mode); + sqlite3_finalize(stmt); + + // Get result of PRAGMA journal_mode + rc = sqlite3_prepare_v2(_memdb, "PRAGMA disk.journal_mode", -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("init_memory_database(PRAGMA journal_mode): Prepare error: %s", + sqlite3_errstr(rc)); + return false; + } + rc = sqlite3_step(stmt); + if( rc == SQLITE_ROW ) + journal_mode = sqlite3_column_text(stmt, 0); + else + { + log_err("init_memory_database(PRAGMA journal_mode): Step error: %s", + sqlite3_errstr(rc)); + return false; + } + log_info("Using %s journal mode for disk database", journal_mode); + sqlite3_finalize(stmt); + + // Get result of PRAGMA synchronous + const unsigned char *synchronous = NULL; + rc = sqlite3_prepare_v2(_memdb, "PRAGMA synchronous", -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("init_memory_database(PRAGMA synchronous): Prepare error: %s", + sqlite3_errstr(rc)); + return false; + } + rc = sqlite3_step(stmt); + if( rc == SQLITE_ROW ) + synchronous = sqlite3_column_text(stmt, 0); + else + { + log_err("init_memory_database(PRAGMA synchronous): Step error: %s", + sqlite3_errstr(rc)); + return false; + } + log_info("Using %s synchronous mode for in-memory database", synchronous); + sqlite3_finalize(stmt); + rc = sqlite3_prepare_v2(_memdb, "PRAGMA disk.synchronous", -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("init_memory_database(PRAGMA synchronous): Prepare error: %s", + sqlite3_errstr(rc)); + return false; + } + rc = sqlite3_step(stmt); + if( rc == SQLITE_ROW ) + synchronous = sqlite3_column_text(stmt, 0); + else + { + log_err("init_memory_database(PRAGMA synchronous): Step error: %s", + sqlite3_errstr(rc)); + return false; + } + log_info("Using %s synchronous mode for disk database", synchronous); + sqlite3_finalize(stmt); // Everything went well return true; @@ -135,11 +232,15 @@ bool init_memory_database(void) void close_memory_database(void) { // Return early if there is no memory database to be closed - if(memdb == NULL) + if(_memdb == NULL) return; + // Detach disk database + if(!detach_database(_memdb, NULL, "disk")) + log_err("close_memory_database(): Failed to detach disk database"); + // Close SQLite3 memory database - int ret = sqlite3_close(memdb); + int ret = sqlite3_close(_memdb); if(ret != SQLITE_OK) log_err("Finalizing memory database failed: %s", sqlite3_errstr(ret)); @@ -147,12 +248,13 @@ void close_memory_database(void) log_debug(DEBUG_DATABASE, "Closed memory database"); // Set global pointer to NULL - memdb = NULL; + _memdb = NULL; } sqlite3 *__attribute__((pure)) get_memdb(void) { - return memdb; + log_debug(DEBUG_DATABASE, "Accessing in-memory database"); + return _memdb; } // Get memory usage and size of in-memory tables @@ -221,6 +323,7 @@ static void log_in_memory_usage(void) size_t memsize = 0; int queries = 0; + sqlite3 *memdb = get_memdb(); if(get_memdb_size(memdb, &memsize, &queries)) { char prefix[2] = { 0 }; @@ -353,7 +456,7 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) // The database pointer may be NULL, meaning we want the memdb if(db == NULL) - db = memdb; + db = get_memdb(); // PRAGMA page_size rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); @@ -394,6 +497,7 @@ bool import_queries_from_disk(void) // Begin transaction int rc; + sqlite3 *memdb = get_memdb(); if((rc = sqlite3_exec(memdb, "BEGIN TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) { log_err("import_queries_from_disk(): Cannot start transaction: %s", sqlite3_errstr(rc)); @@ -402,6 +506,7 @@ bool import_queries_from_disk(void) // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; + log_debug(DEBUG_DATABASE, "Accessing in-memory database"); if((rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL)) != SQLITE_OK){ log_err("import_queries_from_disk(): SQL error prepare: %s", sqlite3_errstr(rc)); return false; @@ -481,10 +586,12 @@ bool export_queries_to_disk(bool final) timer_start(DATABASE_WRITE_TIMER); // Start transaction + sqlite3 *memdb = get_memdb(); SQL_bool(memdb, "BEGIN TRANSACTION"); // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; + log_debug(DEBUG_DATABASE, "Accessing in-memory database"); int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ){ log_err("export_queries_to_disk(): SQL error prepare: %s", sqlite3_errstr(rc)); @@ -527,6 +634,7 @@ bool export_queries_to_disk(bool final) // Update last_disk_db_idx // Prepare SQLite3 statement + log_debug(DEBUG_DATABASE, "Accessing in-memory database"); rc = sqlite3_prepare_v2(memdb, "SELECT MAX(id) FROM disk.query_storage;", -1, &stmt, NULL); // Perform step @@ -603,7 +711,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) sqlite3 *db = NULL; if(use_memdb) - db = memdb; + db = get_memdb(); else db = dbopen(false, false); @@ -631,6 +739,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) mintime, sqlite3_errstr(rc)); // Update number of queries in in-memory database + sqlite3 *memdb = get_memdb(); const int new_num = get_number_of_queries_in_DB(memdb, "query_storage"); log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i", sqlite3_changes(db), (mem_db_num - new_num), new_num); @@ -877,6 +986,7 @@ void DB_read_queries(void) // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; + sqlite3 *memdb = get_memdb(); int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ) { @@ -1192,6 +1302,7 @@ void update_disk_db_idx(void) // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; + sqlite3 *memdb = get_memdb(); int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL); // Perform step @@ -1241,6 +1352,7 @@ bool queries_to_database(void) } // Start preparing query + sqlite3 *memdb = get_memdb(); rc = sqlite3_prepare_v3(memdb, "REPLACE INTO query_storage VALUES "\ "(?1," \ "?2," \ diff --git a/src/database/session-table.c b/src/database/session-table.c index d8b7d190..0147263a 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -12,6 +12,8 @@ #include "database/session-table.h" #include "database/common.h" #include "config/config.h" +// get_memdb() +#include "database/query-table.h" bool create_session_table(sqlite3 *db) { @@ -216,22 +218,17 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) return true; } - sqlite3 *db = dbopen(false, false); - if(db == NULL) - { - log_warn("Failed to open database in restore_db_sessions()"); - return false; - } + sqlite3 *memdb = get_memdb(); // Remove expired sessions from database - SQL_bool(db, "DELETE FROM session WHERE valid_until < strftime('%%s', 'now');"); + SQL_bool(memdb, "DELETE FROM disk.session WHERE valid_until < strftime('%%s', 'now');"); // Get all sessions from database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(db, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM session;", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in restore_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(memdb), sqlite3_errcode(memdb)); return false; } @@ -303,7 +300,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_finalize(stmt) != SQLITE_OK) { log_err("SQL error in restore_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(memdb), sqlite3_errcode(memdb)); return false; } @@ -311,11 +308,9 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // We use secure_delete to make sure the sessions are really gone // In this mode, SQLite overwrites the deleted content with zeros // (https://www.sqlite.org/pragma.html#pragma_secure_delete) - SQL_bool(db, "PRAGMA secure_delete = ON;"); - SQL_bool(db, "DELETE FROM session;"); - - // Close database connection - dbclose(&db); + SQL_bool(memdb, "PRAGMA secure_delete = ON;"); + SQL_bool(memdb, "DELETE FROM disk.session;"); + SQL_bool(memdb, "PRAGMA secure_delete = OFF;"); return true; } From c52d20bdc5edc48b654054876ab569f5ff498566 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 12 Dec 2023 20:43:12 +0100 Subject: [PATCH 22/55] Initialize database only after forking Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 23 +++++++++++++++++++++++ src/main.c | 33 ++------------------------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 1e1ab3b1..d898b1f2 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2823,6 +2823,29 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) else savepid(); + // Initialize query database (pihole-FTL.db) + db_init(); + + // Initialize in-memory databases + if(!init_memory_database()) + log_crit("Cannot initialize in-memory database."); + + // Flush messages stored in the long-term database + flush_message_table(); + + // Try to import queries from long-term database if available + if(config.database.DBimport.v.b) + { + import_queries_from_disk(); + DB_read_queries(); + } + + // Initialize in-memory database starting index + update_disk_db_idx(); + + // Log some information about the imported queries (if any) + log_counter_info(); + // Handle real-time signals in this process (and its children) // Helper processes are already split from the main instance // so they will not listen to real-time signals diff --git a/src/main.c b/src/main.c index bc3a0bab..1f8d1dd5 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,6 @@ #include "config/setupVars.h" #include "args.h" #include "config/config.h" -#include "database/common.h" #include "main.h" // exit_code #include "signals.h" @@ -24,12 +23,10 @@ #include "capabilities.h" #include "timers.h" #include "procps.h" -// init_memory_database(), import_queries_from_disk() -#include "database/query-table.h" // init_overtime() #include "overTime.h" -// flush_message_table() -#include "database/message-table.h" +// export_queries_to_disk() +#include "database/query-table.h" #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) #pragma message "Minimum GLIBC version: " xstr(__GLIBC__) "." xstr(__GLIBC_MINOR__) @@ -110,32 +107,6 @@ int main (int argc, char *argv[]) // Initialize overTime datastructure initOverTime(); - // Initialize query database (pihole-FTL.db) - db_init(); - - // Initialize in-memory databases - if(!init_memory_database()) - { - log_crit("FATAL: Cannot initialize in-memory database."); - return EXIT_FAILURE; - } - - // Flush messages stored in the long-term database - flush_message_table(); - - // Try to import queries from long-term database if available - if(config.database.DBimport.v.b) - { - import_queries_from_disk(); - DB_read_queries(); - } - - // Initialize in-memory database starting index - update_disk_db_idx(); - - // Log some information about the imported queries (if any) - log_counter_info(); - // Check for availability of capabilities in debug mode if(config.debug.caps.v.b) check_capabilities(); From cbec12a657e72ef775ad3dfc9266a338d7931c8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 13 Dec 2023 09:30:10 +0100 Subject: [PATCH 23/55] Set disk.synchronous=NORMAL Signed-off-by: DL6ER --- src/database/query-table.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 8b0ed937..17e3d91a 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -61,6 +61,8 @@ bool init_memory_database(void) { int rc; // Try to open in-memory database + // The :memory: database always has synchronous=OFF since the content of + // it is ephemeral and is not expected to survive a power outage. rc = sqlite3_open_v2(":memory:", &_memdb, SQLITE_OPEN_READWRITE, NULL); if( rc != SQLITE_OK ) { @@ -109,7 +111,7 @@ bool init_memory_database(void) // Attach disk database if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk")) return false; -/* + // Change journal mode to WAL // - WAL is significantly faster in most scenarios. // - WAL provides more concurrency as readers do not block writers and a @@ -124,12 +126,22 @@ bool init_memory_database(void) sqlite3_close(_memdb); return false; } -*/ -/* + // Change synchronous mode to NORMAL - // - NORMAL is the fastest synchronous mode - // - NORMAL still provides full ACID (atomicity, consistency, isolation, - // and durability) properties. + // When synchronous is NORMAL (1), the SQLite database engine will still + // sync at the most critical moments, but less often than in FULL mode. + // There is a very small (though non-zero) chance that a power failure + // at just the wrong time could corrupt the database in + // journal_mode=DELETE on an older filesystem. WAL mode is safe from + // corruption with synchronous=NORMAL, and probably DELETE mode is safe + // too on modern filesystems. WAL mode is always consistent with + // synchronous=NORMAL, but WAL mode does lose durability. A transaction + // committed in WAL mode with synchronous=NORMAL might roll back + // following a power loss or system crash. Transactions are durable + // across application crashes regardless of the synchronous setting or + // journal mode. The synchronous=NORMAL setting is a good choice for + // most applications running in WAL mode. + // https://www.sqlite.org/pragma.html#pragma_synchronous rc = sqlite3_exec(_memdb, "PRAGMA disk.synchronous=NORMAL", NULL, NULL, NULL); if( rc != SQLITE_OK ) { @@ -138,7 +150,7 @@ bool init_memory_database(void) sqlite3_close(_memdb); return false; } -*/ + // Get result of PRAGMA journal_mode const unsigned char *journal_mode = NULL; sqlite3_stmt *stmt = NULL; From 96c2f6cb216504d4884953f1b97645d2fe18de39 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 12:23:14 +0100 Subject: [PATCH 24/55] Greatly simplify memory db initialization by defining normal sync mode globally as compile-time option Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 +- src/database/query-table.c | 109 ------------------------------------- 2 files changed, 2 insertions(+), 110 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fdc7b7f9..bd93a57a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -31,7 +31,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime). # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384") +# SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/database/query-table.c b/src/database/query-table.c index 17e3d91a..ba135af9 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -127,115 +127,6 @@ bool init_memory_database(void) return false; } - // Change synchronous mode to NORMAL - // When synchronous is NORMAL (1), the SQLite database engine will still - // sync at the most critical moments, but less often than in FULL mode. - // There is a very small (though non-zero) chance that a power failure - // at just the wrong time could corrupt the database in - // journal_mode=DELETE on an older filesystem. WAL mode is safe from - // corruption with synchronous=NORMAL, and probably DELETE mode is safe - // too on modern filesystems. WAL mode is always consistent with - // synchronous=NORMAL, but WAL mode does lose durability. A transaction - // committed in WAL mode with synchronous=NORMAL might roll back - // following a power loss or system crash. Transactions are durable - // across application crashes regardless of the synchronous setting or - // journal mode. The synchronous=NORMAL setting is a good choice for - // most applications running in WAL mode. - // https://www.sqlite.org/pragma.html#pragma_synchronous - rc = sqlite3_exec(_memdb, "PRAGMA disk.synchronous=NORMAL", NULL, NULL, NULL); - if( rc != SQLITE_OK ) - { - log_err("init_memory_database(): Step error while trying to set synchronous mode: %s", - sqlite3_errstr(rc)); - sqlite3_close(_memdb); - return false; - } - - // Get result of PRAGMA journal_mode - const unsigned char *journal_mode = NULL; - sqlite3_stmt *stmt = NULL; - rc = sqlite3_prepare_v2(_memdb, "PRAGMA journal_mode", -1, &stmt, NULL); - if( rc != SQLITE_OK ) - { - if( rc != SQLITE_BUSY ) - log_err("init_memory_database(PRAGMA journal_mode): Prepare error: %s", - sqlite3_errstr(rc)); - return false; - } - rc = sqlite3_step(stmt); - if( rc == SQLITE_ROW ) - journal_mode = sqlite3_column_text(stmt, 0); - else - { - log_err("init_memory_database(PRAGMA journal_mode): Step error: %s", - sqlite3_errstr(rc)); - return false; - } - log_info("Using %s journal mode for memory database", journal_mode); - sqlite3_finalize(stmt); - - // Get result of PRAGMA journal_mode - rc = sqlite3_prepare_v2(_memdb, "PRAGMA disk.journal_mode", -1, &stmt, NULL); - if( rc != SQLITE_OK ) - { - if( rc != SQLITE_BUSY ) - log_err("init_memory_database(PRAGMA journal_mode): Prepare error: %s", - sqlite3_errstr(rc)); - return false; - } - rc = sqlite3_step(stmt); - if( rc == SQLITE_ROW ) - journal_mode = sqlite3_column_text(stmt, 0); - else - { - log_err("init_memory_database(PRAGMA journal_mode): Step error: %s", - sqlite3_errstr(rc)); - return false; - } - log_info("Using %s journal mode for disk database", journal_mode); - sqlite3_finalize(stmt); - - // Get result of PRAGMA synchronous - const unsigned char *synchronous = NULL; - rc = sqlite3_prepare_v2(_memdb, "PRAGMA synchronous", -1, &stmt, NULL); - if( rc != SQLITE_OK ) - { - if( rc != SQLITE_BUSY ) - log_err("init_memory_database(PRAGMA synchronous): Prepare error: %s", - sqlite3_errstr(rc)); - return false; - } - rc = sqlite3_step(stmt); - if( rc == SQLITE_ROW ) - synchronous = sqlite3_column_text(stmt, 0); - else - { - log_err("init_memory_database(PRAGMA synchronous): Step error: %s", - sqlite3_errstr(rc)); - return false; - } - log_info("Using %s synchronous mode for in-memory database", synchronous); - sqlite3_finalize(stmt); - rc = sqlite3_prepare_v2(_memdb, "PRAGMA disk.synchronous", -1, &stmt, NULL); - if( rc != SQLITE_OK ) - { - if( rc != SQLITE_BUSY ) - log_err("init_memory_database(PRAGMA synchronous): Prepare error: %s", - sqlite3_errstr(rc)); - return false; - } - rc = sqlite3_step(stmt); - if( rc == SQLITE_ROW ) - synchronous = sqlite3_column_text(stmt, 0); - else - { - log_err("init_memory_database(PRAGMA synchronous): Step error: %s", - sqlite3_errstr(rc)); - return false; - } - log_info("Using %s synchronous mode for disk database", synchronous); - sqlite3_finalize(stmt); - // Everything went well return true; } From 867d1466232281c42fbc4f9353cf95ca8239e45b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 12:25:40 +0100 Subject: [PATCH 25/55] Add recommended SQLITE_LIKE_DOESNT_MATCH_BLOBS compile-time option Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bd93a57a..ed8f15ad 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,7 +32,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1") +# SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed From e34208e38f2dae885564c3c0ea26cbf655717e62 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 12:27:48 +0100 Subject: [PATCH 26/55] Add recommended HAVE_MALLOC_USABLE_SIZE compile-time option Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ed8f15ad..e5016c5f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,7 +33,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS") +# HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed From e9763c69d0d93ee01b34e6a0a7f1d787ac45793f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 12:30:09 +0100 Subject: [PATCH 27/55] Add recommended HAVE_FDATASYNC compile-time option Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5016c5f..1b4695e0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,7 +34,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE") +# HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed From 2719121b53f4da59c5afdd53f3a5e29edf5cd2e7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 12:37:52 +0100 Subject: [PATCH 28/55] Allow SQLite3 to start up to four auxiliary worker threads for work-intense prepared statements. This is most useful with complex queries as it allows parallel sorting and indexing. Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b4695e0..f4517ac4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,7 +35,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible. # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC") +# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed From c81f1a3f6a37f7223082ebb965e05df14450ff58 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Dec 2023 16:04:50 +0100 Subject: [PATCH 29/55] If someone terminates FTL, try to obtain who is the murderer and log it Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 4 +- src/signals.c | 98 ++++++++++++++++++++++++++++++++++++++++++- src/signals.h | 2 + 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index ef4ab3e9..2dd8102d 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -94,7 +94,7 @@ int main_dnsmasq (int argc, char **argv) sigaction(SIGUSR1, &sigact, NULL); sigaction(SIGUSR2, &sigact, NULL); sigaction(SIGHUP, &sigact, NULL); - sigaction(SIGTERM, &sigact, NULL); + sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification sigaction(SIGALRM, &sigact, NULL); sigaction(SIGCHLD, &sigact, NULL); sigaction(SIGINT, &sigact, NULL); @@ -1330,7 +1330,7 @@ static void sig_handler(int sig) event = EVENT_CHILD; else if (sig == SIGALRM) event = EVENT_ALARM; - else if (sig == SIGTERM) + else if (sig == SIGUSR6) // Pi-hole modified event = EVENT_TERM; else if (sig == SIGUSR1) event = EVENT_DUMP; diff --git a/src/signals.c b/src/signals.c index 9f769739..9c638445 100644 --- a/src/signals.c +++ b/src/signals.c @@ -311,11 +311,94 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // Parse neighbor cache set_event(PARSE_NEIGHBOR_CACHE); } + // else if(rtsig == 6) + // { + // // Signal internally used to signal dnsmasq it has to stop + // } // Restore errno before returning back to previous context errno = _errno; } +static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) +{ + // Ignore SIGTERM outside of the main process (TCP forks) + if(mpid != getpid()) + return; + + // Get PID and UID of the process that sent the terminating signal + const pid_t kill_pid = si->si_pid; + const uid_t kill_uid = si->si_uid; + + // Get name of the process that sent the terminating signal + char kill_name[256] = { 0 }; + char kill_exe [256] = { 0 }; + snprintf(kill_exe, sizeof(kill_exe), "/proc/%ld/cmdline", (long int)kill_pid); + FILE *fp = fopen(kill_exe, "r"); + if(fp != NULL) + { + // Successfully opened file + size_t read = 0; + // Read line from file + if((read = fread(kill_name, sizeof(char), sizeof(kill_name), fp)) > 0) + { + // Successfully read line + + // cmdline contains the command-line arguments as a set + // of strings separated by null bytes ('\0'), with a + // further null byte after the last string. Hence, we + // need to replace all null bytes with spaces for + // displaying it below + for(unsigned int i = 0; i < min((size_t)read, sizeof(kill_name)); i++) + { + if(kill_name[i] == '\0') + kill_name[i] = ' '; + } + + // Remove any trailing spaces + for(unsigned int i = read - 1; i > 0; i--) + { + if(kill_name[i] == ' ') + kill_name[i] = '\0'; + else + break; + } + } + else + { + // Failed to read line + strcpy(kill_name, "N/A"); + } + } + else + { + // Failed to open file + strcpy(kill_name, "N/A"); + } + + // Get username of the process that sent the terminating signal + char kill_user[256] = { 0 }; + struct passwd *pwd = getpwuid(kill_uid); + if(pwd != NULL) + { + // Successfully obtained username + strncpy(kill_user, pwd->pw_name, sizeof(kill_user)); + } + else + { + // Failed to obtain username + strcpy(kill_user, "N/A"); + } + + // Log who sent the signal + log_info("Asked to terminate by \"%s\" (PID %ld, user %s UID %ld)", + kill_name, (long int)kill_pid, + kill_user, (long int)kill_uid); + + // Terminate dnsmasq to stop DNS service + raise(SIGUSR6); +} + // Register ordinary signals handler void handle_signals(void) { @@ -337,6 +420,13 @@ void handle_signals(void) } } + // Also catch SIGTERM + struct sigaction SIGaction = { 0 }; + SIGaction.sa_flags = SA_SIGINFO; + sigemptyset(&SIGaction.sa_mask); + SIGaction.sa_sigaction = &SIGTERM_handler; + sigaction(SIGTERM, &SIGaction, NULL); + // Log start time of FTL FTLstarttime = time(NULL); } @@ -351,8 +441,12 @@ void handle_realtime_signals(void) // Catch all real-time signals for(int signum = SIGRTMIN; signum <= SIGRTMAX; signum++) { - struct sigaction SIGACTION; - memset(&SIGACTION, 0, sizeof(struct sigaction)); + if(signum == SIGUSR6) + // Skip SIGUSR6 as it is used internally to signify + // dnsmasq to stop + continue; + + struct sigaction SIGACTION = { 0 }; SIGACTION.sa_flags = SA_SIGINFO; sigemptyset(&SIGACTION.sa_mask); SIGACTION.sa_sigaction = &SIGRT_handler; diff --git a/src/signals.h b/src/signals.h index 78b2d282..4a08e4b9 100644 --- a/src/signals.h +++ b/src/signals.h @@ -12,6 +12,8 @@ #include "enums.h" +#define SIGUSR6 (SIGRTMIN + 6) + // defined in dnsmasq/dnsmasq.h extern volatile char FTL_terminate; From bb23ef090a78e8d1a305f6481dfe01dbc7978d3e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Dec 2023 10:37:20 +0100 Subject: [PATCH 30/55] DNS cache entries were never recycled, possibly causing incorrect blocking of certain domain/client/type combinations Signed-off-by: DL6ER --- src/database/query-table.c | 8 ++++---- src/datastructure.c | 15 ++++----------- src/datastructure.h | 1 + src/dnsmasq_interface.c | 9 ++++++--- src/gc.c | 16 +++++++++++----- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 8a4fa654..a2b2ca7a 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1071,6 +1071,7 @@ void DB_read_queries(void) query->domainID = domainID; query->clientID = clientID; query->upstreamID = upstreamID; + query->cacheID = findCacheID(domainID, clientID, query->type, true); query->id = counters->queries; query->response = 0; query->flags.response_calculated = reply_time_avail; @@ -1125,8 +1126,7 @@ void DB_read_queries(void) // Set ID of the domainlist entry that was the reason for permitting/blocking this query // We assume the value in this field is said ID when it is not a CNAME-related domain // (checked above) and the value of additional_info is not NULL (0 bytes storage size) - const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); - DNSCacheData *cache = getDNSCache(cacheID, true); + DNSCacheData *cache = getDNSCache(query->cacheID, true); // Only load if // a) we have a cache entry // b) the value of additional_info is not NULL (0 bytes storage size) @@ -1459,8 +1459,8 @@ bool queries_to_database(void) } // Get cache entry for this query - const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false); - DNSCacheData *cache = cacheID < 0 ? NULL : getDNSCache(cacheID, true); + const int cacheID = query->cacheID >= 0 ? query->cacheID : findCacheID(query->domainID, query->clientID, query->type, false); + DNSCacheData *cache = getDNSCache(cacheID, true); // ADDITIONAL_INFO if(query->status == QUERY_GRAVITY_CNAME || diff --git a/src/datastructure.c b/src/datastructure.c index 8cf2e5ae..4e37d370 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -369,7 +369,7 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim } } -static int get_next_cacheID(void) +static int get_next_free_cacheID(void) { // Compare content of cache against known cache IP addresses for(int cacheID=0; cacheID < counters->dns_cache_size; cacheID++) @@ -415,7 +415,7 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q return -1; // Get ID of new cache entry - const int cacheID = get_next_cacheID(); + const int cacheID = get_next_free_cacheID(); // Get client pointer DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file); @@ -553,15 +553,8 @@ void FTL_reset_per_client_domain_data(void) { log_debug(DEBUG_DATABASE, "Resetting per-client DNS cache, size is %i", counters->dns_cache_size); - for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) - { - // Reset all blocking yes/no fields for all domains and clients - // This forces a reprocessing of all available filters for any - // given domain and client the next time they are seen - DNSCacheData *dns_cache = getDNSCache(cacheID, true); - if(dns_cache != NULL) - dns_cache->blocking_status = UNKNOWN_BLOCKED; - } + // Set entire DNS cache to zero + memset(getDNSCache(0, false), 0, sizeof(DNSCacheData) * counters->dns_cache_size); } // Reloads all domainlists and performs a few extra tasks such as cleaning the diff --git a/src/datastructure.h b/src/datastructure.h index b94358aa..acd6317c 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -30,6 +30,7 @@ typedef struct { int domainID; int clientID; int upstreamID; + int cacheID; int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here int CNAME_domainID; // only valid if query has a CNAME blocking status int ede; diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 1e1ab3b1..26e107e0 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -783,6 +783,10 @@ bool _FTL_new_query(const unsigned int flags, const char *name, // Query extended DNS error query->ede = EDE_UNSET; + // Initialize cache ID, may be reusing an existing one if this + // (domain,client,type) tuple was already seen before + query->cacheID = findCacheID(domainID, clientID, querytype, true); + // This query is new and not yet known to the database query->db = -1; @@ -1295,8 +1299,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } // Get cache pointer - unsigned int cacheID = findCacheID(domainID, clientID, query->type, true); - DNSCacheData *dns_cache = getDNSCache(cacheID, true); + DNSCacheData *dns_cache = getDNSCache(query->cacheID, true); if(dns_cache == NULL) { log_err("No memory available, skipping query analysis"); @@ -1588,7 +1591,7 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file else if(query->status == QUERY_REGEX) { // Get parent and child DNS cache entries - const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type, false); + const int parent_cacheID = query->cacheID; const int child_cacheID = findCacheID(child_domainID, clientID, query->type, false); // Get cache pointers diff --git a/src/gc.c b/src/gc.c index 9be9fb1e..d70588aa 100644 --- a/src/gc.c +++ b/src/gc.c @@ -52,7 +52,9 @@ static void recycle(void) bool *client_used = calloc(counters->clients, sizeof(bool)); bool *domain_used = calloc(counters->domains, sizeof(bool)); bool *upstreams_used = calloc(counters->upstreams, sizeof(bool)); - if(client_used == NULL || domain_used == NULL || upstreams_used == NULL) + bool *cache_used = calloc(counters->dns_cache_size, sizeof(bool)); + if(client_used == NULL || domain_used == NULL || + upstreams_used == NULL || cache_used == NULL) { log_err("Cannot allocate memory for recycling"); return; @@ -77,6 +79,10 @@ static void recycle(void) // Mark CNAME domain as used (if any) if(query->CNAME_domainID >= 0) domain_used[query->CNAME_domainID] = true; + + // Mark cache entry as used (if any) + if(query->cacheID >= 0) + cache_used[query->cacheID] = true; } // Recycle clients @@ -128,12 +134,11 @@ static void recycle(void) unsigned int cache_recycled = 0; for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) { - DNSCacheData *cache = getDNSCache(cacheID, true); - if(cache == NULL) + if(cache_used[cacheID]) continue; - // Skip cache entries that are still in use - if(cache->magic != 0x00) + DNSCacheData *cache = getDNSCache(cacheID, true); + if(cache == NULL) continue; log_debug(DEBUG_GC, "Recycling cache entry with ID %d", cacheID); @@ -148,6 +153,7 @@ static void recycle(void) free(client_used); free(domain_used); free(upstreams_used); + free(cache_used); // Scan number of recycled clients and domains if in debug mode if(config.debug.gc.v.b) From a5d1d4477b55f97e905e9d8b600dd8c6c554e974 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Dec 2023 10:39:50 +0100 Subject: [PATCH 31/55] Move debug messages meant for debug.status from debug.gc over Signed-off-by: DL6ER --- src/database/query-table.c | 4 ++-- src/datastructure.c | 4 ++-- src/dnsmasq_interface.c | 12 ++++++------ src/gc.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index a2b2ca7a..a7c48ef7 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1065,7 +1065,7 @@ void DB_read_queries(void) query->qtype = type - 100; } counters->querytype[query->type]++; - log_debug(DEBUG_GC, "query type %d set (database), ID = %d, new count = %d", query->type, counters->queries, counters->querytype[query->type]); + log_debug(DEBUG_STATUS, "query type %d set (database), ID = %d, new count = %d", query->type, counters->queries, counters->querytype[query->type]); // Status is set below query->domainID = domainID; @@ -1078,7 +1078,7 @@ void DB_read_queries(void) query->dnssec = dnssec; query->reply = reply; counters->reply[query->reply]++; - log_debug(DEBUG_GC, "reply type %d set (database), ID = %d, new count = %d", query->reply, counters->queries, counters->reply[query->reply]); + log_debug(DEBUG_STATUS, "reply type %d set (database), ID = %d, new count = %d", query->reply, counters->queries, counters->reply[query->reply]); query->response = reply_time; query->CNAME_domainID = -1; // Initialize flags diff --git a/src/datastructure.c b/src/datastructure.c index 4e37d370..bd3fd010 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -1032,10 +1032,10 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c if(!init) { counters->status[old_status]--; - log_debug(DEBUG_GC, "status %d removed (!init), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); + log_debug(DEBUG_STATUS, "status %d removed (!init), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); } counters->status[new_status]++; - log_debug(DEBUG_GC, "status %d set, ID = %d, new count = %d", new_status, query->id, counters->status[new_status]); + log_debug(DEBUG_STATUS, "status %d set, ID = %d, new count = %d", new_status, query->id, counters->status[new_status]); // ... update overTime counters, ... const int timeidx = getOverTimeID(query->timestamp); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 26e107e0..33e77356 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -747,7 +747,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, query->timestamp = querytimestamp; query->type = querytype; counters->querytype[querytype]++; - log_debug(DEBUG_GC, "query type %d set (new query), ID = %d, new count = %d", query->type, id, counters->querytype[query->type]); + log_debug(DEBUG_STATUS, "query type %d set (new query), ID = %d, new count = %d", query->type, id, counters->querytype[query->type]); query->qtype = qtype; query->id = id; // Has to be set before calling query_set_status() @@ -764,7 +764,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, // Initialize reply type query->reply = REPLY_UNKNOWN; counters->reply[REPLY_UNKNOWN]++; - log_debug(DEBUG_GC, "reply type %d set (new query), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); + log_debug(DEBUG_STATUS, "reply type %d set (new query), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Store DNSSEC result for this domain query->dnssec = DNSSEC_UNKNOWN; query->CNAME_domainID = -1; @@ -2736,12 +2736,12 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep // Subtract from old reply counter counters->reply[query->reply]--; - log_debug(DEBUG_GC, "reply type %d removed (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); + log_debug(DEBUG_STATUS, "reply type %d removed (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Add to new reply counter counters->reply[new_reply]++; // Store reply type query->reply = new_reply; - log_debug(DEBUG_GC, "reply type %d added (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); + log_debug(DEBUG_STATUS, "reply type %d added (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Save response time // Skipped internally if already computed @@ -3358,10 +3358,10 @@ void FTL_multiple_replies(const int id, int *firstID) // Copy relevant information over counters->reply[duplicated_query->reply]--; - log_debug(DEBUG_GC, "duplicated_query reply type %d removed, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); + log_debug(DEBUG_STATUS, "duplicated_query reply type %d removed, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); duplicated_query->reply = source_query->reply; counters->reply[duplicated_query->reply]++; - log_debug(DEBUG_GC, "duplicated_query reply type %d set, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); + log_debug(DEBUG_STATUS, "duplicated_query reply type %d set, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); duplicated_query->dnssec = source_query->dnssec; duplicated_query->flags.complete = true; diff --git a/src/gc.c b/src/gc.c index d70588aa..0069c813 100644 --- a/src/gc.c +++ b/src/gc.c @@ -384,17 +384,17 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) // Update reply counters counters->reply[query->reply]--; - log_debug(DEBUG_GC, "reply type %d removed (GC), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); + log_debug(DEBUG_STATUS, "reply type %d removed (GC), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Update type counters counters->querytype[query->type]--; - log_debug(DEBUG_GC, "query type %d removed (GC), ID = %d, new count = %d", query->type, query->id, counters->querytype[query->type]); + log_debug(DEBUG_STATUS, "query type %d removed (GC), ID = %d, new count = %d", query->type, query->id, counters->querytype[query->type]); // Subtract UNKNOWN from the counters before // setting the status if different. // Minus one here and plus one below = net zero counters->status[QUERY_UNKNOWN]--; - log_debug(DEBUG_GC, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); + log_debug(DEBUG_STATUS, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); // Set query again to UNKNOWN to reset the counters query_set_status(query, QUERY_UNKNOWN); From 5c95d263051770e7308ff63f3022048d0abe6b0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Dec 2023 11:46:40 +0100 Subject: [PATCH 32/55] Simplify recycler debug summary Signed-off-by: DL6ER --- src/gc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gc.c b/src/gc.c index 0069c813..47ed2fec 100644 --- a/src/gc.c +++ b/src/gc.c @@ -187,10 +187,10 @@ static void recycle(void) free_cache++; } - log_debug(DEBUG_GC, "Recycler summary: %u/%d (max %d) clients, %u/%d (max %d) domains and %u/%d (max %d) cache records are free", - free_clients, counters->clients, counters->clients_MAX, - free_domains, counters->domains, counters->domains_MAX, - free_cache, counters->dns_cache_size, counters->dns_cache_MAX); + log_debug(DEBUG_GC, "%d/%d clients, %d/%d domains and %d/%d cache records are free", + counters->clients_MAX + (int)free_clients - counters->clients, counters->clients_MAX, + counters->domains_MAX + (int)free_domains - counters->domains_MAX, counters->domains_MAX, + counters->dns_cache_MAX + (int)free_cache - counters->dns_cache_MAX, counters->dns_cache_MAX); log_debug(DEBUG_GC, "Recycled additional %u clients, %u domains, and %u cache records (scanned %d queries)", clients_recycled, domains_recycled, cache_recycled, counters->queries); From 3084b1c5072a5a7cf8a442ed7aba2b78ebd1c06d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Dec 2023 21:38:45 +0100 Subject: [PATCH 33/55] Also log number of DNS cache records after history import Signed-off-by: DL6ER --- src/datastructure.c | 13 ++++++++++++- src/gc.c | 4 ++-- src/log.c | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/datastructure.c b/src/datastructure.c index bd3fd010..16e534ec 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -426,6 +426,9 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q return -1; } + log_debug(DEBUG_GC, "New cache entry: domainID %d, clientID %d, query_type %d (ID %d)", + domainID, clientID, query_type, cacheID); + // Initialize cache entry dns_cache->magic = MAGICBYTE; dns_cache->blocking_status = UNKNOWN_BLOCKED; @@ -554,7 +557,15 @@ void FTL_reset_per_client_domain_data(void) log_debug(DEBUG_DATABASE, "Resetting per-client DNS cache, size is %i", counters->dns_cache_size); // Set entire DNS cache to zero - memset(getDNSCache(0, false), 0, sizeof(DNSCacheData) * counters->dns_cache_size); + DNSCacheData *first_cache = getDNSCache(0, false); + if(first_cache == NULL) + { + log_err("Encountered serious memory error in FTL_reset_per_client_domain_data()"); + return; + } + + // else: Set entire DNS cache to zero + memset(first_cache, 0, sizeof(DNSCacheData) * counters->dns_cache_size); } // Reloads all domainlists and performs a few extra tasks such as cleaning the diff --git a/src/gc.c b/src/gc.c index 47ed2fec..b9748295 100644 --- a/src/gc.c +++ b/src/gc.c @@ -77,11 +77,11 @@ static void recycle(void) upstreams_used[query->upstreamID] = true; // Mark CNAME domain as used (if any) - if(query->CNAME_domainID >= 0) + if(query->CNAME_domainID > -1) domain_used[query->CNAME_domainID] = true; // Mark cache entry as used (if any) - if(query->cacheID >= 0) + if(query->cacheID > -1) cache_used[query->cacheID] = true; } diff --git a/src/log.c b/src/log.c index 4eff2910..bc0ce1c1 100644 --- a/src/log.c +++ b/src/log.c @@ -482,6 +482,7 @@ void log_counter_info(void) log_info(" -> Unknown DNS queries: %i", counters->status[QUERY_UNKNOWN]); log_info(" -> Unique domains: %i", counters->domains); log_info(" -> Unique clients: %i", counters->clients); + log_info(" -> DNS cache records: %i", counters->dns_cache_size); log_info(" -> Known forward destinations: %i", counters->upstreams); } From 8ec30408c8544fa3e8937dba65cb4ed247549f48 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Dec 2023 21:44:35 +0100 Subject: [PATCH 34/55] Remove unused upstream recycle check Signed-off-by: DL6ER --- src/gc.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/gc.c b/src/gc.c index b9748295..1d67126f 100644 --- a/src/gc.c +++ b/src/gc.c @@ -51,10 +51,8 @@ static void recycle(void) { bool *client_used = calloc(counters->clients, sizeof(bool)); bool *domain_used = calloc(counters->domains, sizeof(bool)); - bool *upstreams_used = calloc(counters->upstreams, sizeof(bool)); bool *cache_used = calloc(counters->dns_cache_size, sizeof(bool)); - if(client_used == NULL || domain_used == NULL || - upstreams_used == NULL || cache_used == NULL) + if(client_used == NULL || domain_used == NULL || cache_used == NULL) { log_err("Cannot allocate memory for recycling"); return; @@ -72,10 +70,6 @@ static void recycle(void) client_used[query->clientID] = true; domain_used[query->domainID] = true; - // Mark upstream as used (if any) - if(query->upstreamID > -1) - upstreams_used[query->upstreamID] = true; - // Mark CNAME domain as used (if any) if(query->CNAME_domainID > -1) domain_used[query->CNAME_domainID] = true; @@ -152,7 +146,6 @@ static void recycle(void) // Free memory free(client_used); free(domain_used); - free(upstreams_used); free(cache_used); // Scan number of recycled clients and domains if in debug mode From 7b4c0aa3622e9dcc528293c11c2a9884ae5afd7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 04:29:44 +0100 Subject: [PATCH 35/55] When FTL_check_blocking() is called with a different domain than the one already stored in the query, we have to re-lookup the cache ID. This can happen when a CNAME chain is followed and analyzed Signed-off-by: DL6ER --- src/datastructure.c | 34 +++++++++++++++++++--------------- src/dnsmasq_interface.c | 8 +++++++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/datastructure.c b/src/datastructure.c index 16e534ec..65b4f9c6 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -137,10 +137,10 @@ int _findUpstreamID(const char *upstreamString, const in_port_t port, int line, return upstreamID; } -static int get_next_domainID(void) +static int get_next_free_domainID(void) { // Compare content of domain against known domain IP addresses - for(int domainID=0; domainID < counters->domains; domainID++) + for(int domainID = 0; domainID < counters->domains; domainID++) { // Get domain pointer domainsData* domain = getDomain(domainID, false); @@ -188,7 +188,7 @@ int _findDomainID(const char *domainString, const bool count, int line, const ch // If we did not return until here, then this domain is not known // Store ID - const int domainID = get_next_domainID(); + const int domainID = get_next_free_domainID(); // Get domain pointer domainsData* domain = _getDomain(domainID, false, line, func, file); @@ -218,10 +218,10 @@ int _findDomainID(const char *domainString, const bool count, int line, const ch return domainID; } -static int get_next_clientID(void) +static int get_next_free_clientID(void) { // Compare content of client against known client IP addresses - for(int clientID=0; clientID < counters->clients; clientID++) + for(int clientID = 0; clientID < counters->clients; clientID++) { // Get client pointer clientsData* client = getClient(clientID, false); @@ -271,7 +271,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient // If we did not return until here, then this client is definitely new // Store ID - const int clientID = get_next_clientID(); + const int clientID = get_next_free_clientID(); // Get client pointer clientsData* client = _getClient(clientID, false, line, func, file); @@ -372,7 +372,7 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim static int get_next_free_cacheID(void) { // Compare content of cache against known cache IP addresses - for(int cacheID=0; cacheID < counters->dns_cache_size; cacheID++) + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) { // Get cache pointer DNSCacheData* cache = getDNSCache(cacheID, false); @@ -556,16 +556,20 @@ void FTL_reset_per_client_domain_data(void) { log_debug(DEBUG_DATABASE, "Resetting per-client DNS cache, size is %i", counters->dns_cache_size); - // Set entire DNS cache to zero - DNSCacheData *first_cache = getDNSCache(0, false); - if(first_cache == NULL) + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) { - log_err("Encountered serious memory error in FTL_reset_per_client_domain_data()"); - return; - } + // Get cache pointer + DNSCacheData* dns_cache = getDNSCache(cacheID, true); - // else: Set entire DNS cache to zero - memset(first_cache, 0, sizeof(DNSCacheData) * counters->dns_cache_size); + // Check if the returned pointer is valid before trying to access it + if(dns_cache == NULL) + continue; + + // Reset blocking status + dns_cache->blocking_status = UNKNOWN_BLOCKED; + // Reset domainlist ID + dns_cache->domainlist_id = -1; + } } // Reloads all domainlists and performs a few extra tasks such as cleaning the diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 33e77356..83c4de41 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1299,7 +1299,13 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } // Get cache pointer - DNSCacheData *dns_cache = getDNSCache(query->cacheID, true); + // When this function is called with a different domain than the one + // already stored in the query, we have to re-lookup the cache ID. + // This can happen when a CNAME chain is followed and analyzed + const int cacheID = query->domainID == domainID && query->clientID == clientID ? + query->cacheID : + findCacheID(domainID, clientID, query->type, true); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache == NULL) { log_err("No memory available, skipping query analysis"); From d92e1a056c1b2994dad35ccd24a85ceb0a2c1092 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 22:19:03 +0100 Subject: [PATCH 36/55] Do not check errors on ROLLBACK TRANSACTION when gravityDB_delFromTable() fails. We remove this to avoid overwriting the initial cause of the error. Signed-off-by: DL6ER --- src/database/gravity-db.c | 84 ++++++++++----------------------------- 1 file changed, 21 insertions(+), 63 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 10a0d934..92c320b9 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1840,15 +1840,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", listtype, querystr, *message); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -1860,15 +1856,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* listtype, querystr, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -1888,15 +1880,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", listtype, querystr, *message); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -1914,15 +1902,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* type->valueint, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - type->valueint, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -1936,15 +1920,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* listtype, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -1956,15 +1936,11 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* listtype, querystr, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + return false; } @@ -2024,13 +2000,7 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec: %s", - listtype, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); break; } @@ -2048,13 +2018,7 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); } // Commit transaction @@ -2069,13 +2033,7 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* // Rollback transaction querystr = "ROLLBACK TRANSACTION;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - { - *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", - listtype, querystr, *message); - } + sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); } return okay; From 5f0e405d82f90c4c53bfe9dd558e858c52061327 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Dec 2023 09:36:06 +0100 Subject: [PATCH 37/55] Update bundled cJSON from 1.7.15 -> 1.7.17 released yesterday Signed-off-by: DL6ER --- src/webserver/cJSON/cJSON.c | 43 ++++++++++++++++++++++++++----------- src/webserver/cJSON/cJSON.h | 11 ++++++++-- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/webserver/cJSON/cJSON.c b/src/webserver/cJSON/cJSON.c index ebd48668..4e4979e9 100644 --- a/src/webserver/cJSON/cJSON.c +++ b/src/webserver/cJSON/cJSON.c @@ -96,9 +96,9 @@ CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) return (const char*) (global_error.json + global_error.position); } -CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) +CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) { - if (!cJSON_IsString(item)) + if (!cJSON_IsString(item)) { return NULL; } @@ -106,9 +106,9 @@ CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) return item->valuestring; } -CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) +CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) { - if (!cJSON_IsNumber(item)) + if (!cJSON_IsNumber(item)) { return (double) NAN; } @@ -117,7 +117,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 15) +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 17) #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif @@ -401,7 +401,12 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */ - if (!(object->type & cJSON_String) || (object->type & cJSON_IsReference)) + if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference)) + { + return NULL; + } + /* return NULL if the object is corrupted */ + if (object->valuestring == NULL) { return NULL; } @@ -511,7 +516,7 @@ static unsigned char* ensure(printbuffer * const p, size_t needed) return NULL; } - + memcpy(newbuffer, p->buffer, p->offset + 1); p->hooks.deallocate(p->buffer); } @@ -562,6 +567,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out { length = sprintf((char*)number_buffer, "null"); } + else if(d == (double)item->valueint) + { + length = sprintf((char*)number_buffer, "%d", item->valueint); + } else { /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ @@ -1103,7 +1112,7 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer } buffer.content = (const unsigned char*)value; - buffer.length = buffer_length; + buffer.length = buffer_length; buffer.offset = 0; buffer.hooks = global_hooks; @@ -2260,7 +2269,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON { cJSON *after_inserted = NULL; - if (which < 0) + if (which < 0 || newitem == NULL) { return false; } @@ -2271,6 +2280,11 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON return add_item_to_array(array, newitem); } + if (after_inserted != array->child && after_inserted->prev == NULL) { + /* return false if after_inserted is a corrupted array item */ + return false; + } + newitem->next = after_inserted; newitem->prev = after_inserted->prev; after_inserted->prev = newitem; @@ -2287,7 +2301,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) { - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) + if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || (item == NULL)) { return false; } @@ -2357,6 +2371,11 @@ static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSO cJSON_free(replacement->string); } replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if (replacement->string == NULL) + { + return false; + } + replacement->type &= ~cJSON_StringIsConst; return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); @@ -2689,7 +2708,7 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int co if (a && a->child) { a->child->prev = n; } - + return a; } @@ -3107,4 +3126,4 @@ CJSON_PUBLIC(void *) cJSON_malloc(size_t size) CJSON_PUBLIC(void) cJSON_free(void *object) { global_hooks.deallocate(object); -} \ No newline at end of file +} diff --git a/src/webserver/cJSON/cJSON.h b/src/webserver/cJSON/cJSON.h index 8ff711b8..218cc9ea 100644 --- a/src/webserver/cJSON/cJSON.h +++ b/src/webserver/cJSON/cJSON.h @@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 15 +#define CJSON_VERSION_PATCH 17 #include @@ -279,6 +279,13 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); /* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring); +/* If the object is not a boolean type this does nothing and returns cJSON_Invalid else it returns the new type*/ +#define cJSON_SetBoolValue(object, boolValue) ( \ + (object != NULL && ((object)->type & (cJSON_False|cJSON_True))) ? \ + (object)->type=((object)->type &(~(cJSON_False|cJSON_True)))|((boolValue)?cJSON_True:cJSON_False) : \ + cJSON_Invalid\ +) + /* Macro for iterating over an array or object */ #define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) @@ -290,4 +297,4 @@ CJSON_PUBLIC(void) cJSON_free(void *object); } #endif -#endif \ No newline at end of file +#endif From 975c46817bffe1e39e376816fd00ad1799a7559f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jan 2024 08:41:37 +0100 Subject: [PATCH 38/55] Optimize database on close (gravity) or frequently (queries) Signed-off-by: DL6ER --- src/database/database-thread.c | 26 +++++++++++++++++++++++++- src/database/gravity-db.c | 16 ++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 3eaf9935..40c10853 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -56,6 +56,14 @@ static bool delete_old_queries_in_DB(sqlite3 *db) return true; } +static bool optimize_database(sqlite3 *db) +{ + // Optimize the database by running PRAGMA optimize + SQL_bool(db, "PRAGMA optimize;"); + + return true; +} + #define DBOPEN_OR_AGAIN() { if(!db) db = dbopen(false, false); if(!db) { thread_sleepms(DB, 5000); continue; } } #define BREAK_IF_KILLED() { if(killed) break; } #define DBCLOSE_OR_BREAK() { dbclose(&db); BREAK_IF_KILLED(); } @@ -72,6 +80,10 @@ void *DB_thread(void *val) time_t before = time(NULL); time_t lastDBsave = before - before%config.database.DBinterval.v.ui; + // Other timestamps + time_t lastOptimize = before; + time_t lastMACVendor = before; + // This thread runs until shutdown of the process. We keep this thread // running when pihole-FTL.db is corrupted because reloading of privacy // level, and the gravity database (initially and after gravity) @@ -135,13 +147,25 @@ void *DB_thread(void *val) set_event(PARSE_NEIGHBOR_CACHE); } + // Intermediate cancellation-point + if(killed) + break; + + // Optimize database every 24 hours + if(now - lastOptimize >= 86400) + { + DBOPEN_OR_AGAIN(); + optimize_database(db); + DBCLOSE_OR_BREAK(); + } + // Intermediate cancellation-point if(killed) break; // Update MAC vendor strings once a month (the MAC vendor // database is not updated very often) - if(now % 2592000L == 0) + if(now - lastMACVendor >= 2592000) { DBOPEN_OR_AGAIN(); updateMACVendorRecords(db); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 10a0d934..a47d5db7 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -953,6 +953,22 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&gravity_stmt); free_sqlite3_stmt_vec(&antigravity_stmt); + // Run PRAMGMA optimize to optimize database file + // It is recommended to run this command on a regular basis + // when closing the database connection + // See https://www.sqlite.org/pragma.html#pragma_optimize + // We set a small value of analysis_limit=1000 to ensure the + // command returns quickly (it limits the number of rows + // analyzed by the query planner to 1000) + const char *querystr = "PRAGMA analysis_limit = 1000;"; + int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); + querystr = "PRAGMA optimize;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); + // Close table sqlite3_close(gravity_db); gravity_db = NULL; From 136982c9dc83e8cc0a0c48aa98022de358a6f906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 3 Jan 2024 22:28:33 +0100 Subject: [PATCH 39/55] Simplify artifacts download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e386994..2a30d6e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,18 +149,14 @@ jobs: uses: actions/download-artifact@v4.1.0 id: download with: - path: download/ + path: ftl_builds/ + pattern: pihole-* + merge-multiple: true - name: Display structure of downloaded files run: ls -R working-directory: ${{steps.download.outputs.download-path}} - - - name: Copy all artifacts from sub-directories to ftl_builds/ - run: | - mkdir ftl_builds/ - cp ${{steps.download.outputs.download-path}}/**/* ftl_builds/ - - name: Install SSH Key uses: benoitchantre/setup-ssh-authentication-action@1.0.1 From 54041665b6b5bd68a73ab8a3b0f6395cbb2a9352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 3 Jan 2024 22:33:38 +0100 Subject: [PATCH 40/55] Account for api-docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a30d6e4..b93df030 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,7 +133,7 @@ jobs: if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64' uses: actions/upload-artifact@v4.0.0 with: - name: api-docs + name: pihole-api-docs path: 'api-docs.tar.gz' deploy: From da7c3a7735adc6cc4865f49bcd6493cdb00a2fd0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 4 Jan 2024 23:30:14 +0100 Subject: [PATCH 41/55] Run ANALYZE instead of PRAMGA optimize after some discussion with the SQlite3 developers. Also ensure notices and mere messages are not always logged as errors in FTL's log. Furhtermore, reduce the frequency of running ANALYZE from once per day to once per week. Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- src/database/common.c | 10 ++++++++-- src/database/database-thread.c | 29 ++++++++++++++++++++++------- src/database/gravity-db.c | 16 ---------------- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f4517ac4..d7722c86 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -36,7 +36,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible. # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. # SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4") +# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL. +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed diff --git a/src/database/common.c b/src/database/common.c index 4fd30b6c..a081fc2e 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -251,9 +251,15 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) generate_backtrace(); if(iErrCode == SQLITE_WARNING) - log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode); + log_warn("SQLite3: %s (%d)", zMsg, iErrCode); + else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) + // SQLITE_SCHEMA is returned when the database schema has changed + // This is not necessarily an error, as sqlite3_step() will re-prepare + // the statement and try again. If it cannot, it will return an error + // and this will be handled over there. + log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode); else - log_err("SQLite3 message: %s (%d)", zMsg, iErrCode); + log_err("SQLite3: %s (%d)", zMsg, iErrCode); } void db_init(void) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 40c10853..d4e39267 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -56,10 +56,23 @@ static bool delete_old_queries_in_DB(sqlite3 *db) return true; } -static bool optimize_database(sqlite3 *db) +static bool analyze_database(sqlite3 *db) { - // Optimize the database by running PRAGMA optimize - SQL_bool(db, "PRAGMA optimize;"); + // Optimize the database by running ANALYZE + // The ANALYZE command gathers statistics about tables and indices and + // stores the collected information in internal tables of the database + // where the query optimizer can access the information and use it to + // help make better query planning choices. + + // Measure time + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); + SQL_bool(db, "ANALYZE;"); + clock_gettime(CLOCK_MONOTONIC, &end); + + // Print final message + log_info("Optimized database in %.3f seconds", + (double)(end.tv_sec - start.tv_sec) + 1e-9*(end.tv_nsec - start.tv_nsec)); return true; } @@ -81,7 +94,7 @@ void *DB_thread(void *val) time_t lastDBsave = before - before%config.database.DBinterval.v.ui; // Other timestamps - time_t lastOptimize = before; + time_t lastAnalyze = before; time_t lastMACVendor = before; // This thread runs until shutdown of the process. We keep this thread @@ -151,11 +164,12 @@ void *DB_thread(void *val) if(killed) break; - // Optimize database every 24 hours - if(now - lastOptimize >= 86400) + // Optimize database once per week + if(now - lastAnalyze >= 604800) { DBOPEN_OR_AGAIN(); - optimize_database(db); + analyze_database(db); + lastAnalyze = now; DBCLOSE_OR_BREAK(); } @@ -169,6 +183,7 @@ void *DB_thread(void *val) { DBOPEN_OR_AGAIN(); updateMACVendorRecords(db); + lastMACVendor = now; DBCLOSE_OR_BREAK(); } diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index a47d5db7..10a0d934 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -953,22 +953,6 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&gravity_stmt); free_sqlite3_stmt_vec(&antigravity_stmt); - // Run PRAMGMA optimize to optimize database file - // It is recommended to run this command on a regular basis - // when closing the database connection - // See https://www.sqlite.org/pragma.html#pragma_optimize - // We set a small value of analysis_limit=1000 to ensure the - // command returns quickly (it limits the number of rows - // analyzed by the query planner to 1000) - const char *querystr = "PRAGMA analysis_limit = 1000;"; - int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); - querystr = "PRAGMA optimize;"; - rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); - if(rc != SQLITE_OK) - log_err("gravityDB_close(\"%s\") - SQL error: %s", querystr, sqlite3_errstr(rc)); - // Close table sqlite3_close(gravity_db); gravity_db = NULL; From 98127f12b921a3005e0c242dd38a06a21649af69 Mon Sep 17 00:00:00 2001 From: Artur Kordowski <9746197+akordowski@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:45:26 +0100 Subject: [PATCH 42/55] Fix API dns/blocking documentation Signed-off-by: Artur Kordowski <9746197+akordowski@users.noreply.github.com> --- src/api/docs/content/specs/dns.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/api/docs/content/specs/dns.yaml b/src/api/docs/content/specs/dns.yaml index 6b8f015b..1b678c2c 100644 --- a/src/api/docs/content/specs/dns.yaml +++ b/src/api/docs/content/specs/dns.yaml @@ -29,7 +29,7 @@ components: description: | Change the current blocking mode by setting `blocking` to the desired value. The optional `timer` object may used to set a timer. Once this timer elapsed, the opposite blocking mode is automatically set. - For instance, you can request `{blocking: true, timer: 60}` to disable Pi-hole for one minute. + For instance, you can request `{blocking: false, timer: 60}` to disable Pi-hole for one minute. Blocking will be automatically resumed afterwards. You can terminate a possibly running timer by setting `timer` to `null` (the set mode becomes permanent). @@ -39,9 +39,8 @@ components: 'application/json': schema: allOf: - - $ref: 'dns.yaml#/components/schemas/blocking' + - $ref: 'dns.yaml#/components/schemas/blocking_bool' - $ref: 'dns.yaml#/components/schemas/timer' - - $ref: 'common.yaml#/components/schemas/took' responses: '200': description: OK @@ -83,6 +82,14 @@ components: - "failed" - "unknown" example: "enabled" + blocking_bool: + type: object + properties: + blocking: + type: boolean + description: Blocking status + default: true + example: true timer: type: object properties: From d77547d4d7d42de15667d3cbc6af642f297fb11a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jan 2024 09:13:29 +0100 Subject: [PATCH 43/55] Ensure database analysis / MAC vendor update is running also when FTL is frequently restarted (e.g. during development or for users joining a special branch, e.g. during extended bug fixing or a beta release period) Signed-off-by: DL6ER --- src/FTL.h | 9 +++++++++ src/database/database-thread.c | 17 ++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 14f726d2..9edd2034 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -50,6 +50,7 @@ // Number of elements in an array #define ArraySize(X) (sizeof(X)/sizeof(X[0])) +// Constant socket buffer length #define SOCKETBUFFERLEN 1024 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds] @@ -133,6 +134,14 @@ // Special exit code used to signal that FTL wants to restart #define RESTART_FTL_CODE 22 +// How often should the database be analyzed? +// Default: 604800 (once per week) +#define DATABASE_ANALYZE_INTERVAL 604800 + +// How often should we update client vendor's from the MAC vendor database? +// Default: 2592000 (once per month) +#define DATABASE_MACVENDOR_INTERVAL 2592000 + // Use out own syscalls handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crash // caused by insufficient memory or by code bugs (not properly dealing diff --git a/src/database/database-thread.c b/src/database/database-thread.c index d4e39267..250c6f05 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -93,9 +93,16 @@ void *DB_thread(void *val) time_t before = time(NULL); time_t lastDBsave = before - before%config.database.DBinterval.v.ui; - // Other timestamps - time_t lastAnalyze = before; - time_t lastMACVendor = before; + // Other timestamps, made independent from the exact time FTL was + // started + time_t lastAnalyze = before - before % DATABASE_ANALYZE_INTERVAL; + time_t lastMACVendor = before - before % DATABASE_MACVENDOR_INTERVAL; + + // Add some randomness (up to ome hour) to these timestamps to avoid + // them running at the same time. This is not a security feature, so + // using rand() is fine. + lastAnalyze += rand() % 3600; + lastMACVendor += rand() % 3600; // This thread runs until shutdown of the process. We keep this thread // running when pihole-FTL.db is corrupted because reloading of privacy @@ -165,7 +172,7 @@ void *DB_thread(void *val) break; // Optimize database once per week - if(now - lastAnalyze >= 604800) + if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL) { DBOPEN_OR_AGAIN(); analyze_database(db); @@ -179,7 +186,7 @@ void *DB_thread(void *val) // Update MAC vendor strings once a month (the MAC vendor // database is not updated very often) - if(now - lastMACVendor >= 2592000) + if(now - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL) { DBOPEN_OR_AGAIN(); updateMACVendorRecords(db); From 8806f6427fcc66cf9282c2a140cda0d2809ba538 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jan 2024 16:57:36 +0100 Subject: [PATCH 44/55] Add Location header for newly created groups/clients/domains/lists Signed-off-by: DL6ER --- src/api/list.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/api/list.c b/src/api/list.c index e9233e8d..8b371e95 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -510,6 +510,20 @@ static int api_list_write(struct ftl_conn *api, if(api->method == HTTP_PUT) response_code = 200; // 200 - OK + // Add "Location" header to response + if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers), "Location: %s/%s", api->action_path, row.item) >= (int)sizeof(pi_hole_extra_headers)) + { + // This may happen for *extremely* long URLs but is not issue in + // itself. Merely add a warning to the log file + log_warn("Could not add Location header to response: URL too long"); + + // Truncate location by replacing the last characters with "...\0" + pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-4] = '.'; + pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-3] = '.'; + pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-2] = '.'; + pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-1] = '\0'; + } + // Send GET style reply const int ret = api_list_read(api, response_code, listtype, row.item, processed); From e5e9d11211921482aaa23b3439d14ab0e66298fe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 20 Dec 2023 23:09:05 +0100 Subject: [PATCH 45/55] Fix DELETE API endpoints. They should return 204 when something was deleted and 404 is nothing was found at this resource Signed-off-by: DL6ER --- src/api/auth.c | 31 +++++++++++-------- src/api/docs/content/specs/auth.yaml | 41 ++++++++++--------------- src/api/docs/content/specs/clients.yaml | 2 ++ src/api/docs/content/specs/domains.yaml | 2 ++ src/api/docs/content/specs/groups.yaml | 2 ++ src/api/docs/content/specs/lists.yaml | 2 ++ src/api/list.c | 9 ++++-- src/database/gravity-db.c | 6 +++- src/database/gravity-db.h | 2 +- 9 files changed, 55 insertions(+), 42 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index d85811c9..5b535b72 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -336,14 +336,18 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ return 0; } -static void delete_session(const int user_id) +static bool delete_session(const int user_id) { // Skip if nothing to be done here if(user_id < 0 || user_id >= max_sessions) - return; + return false; + + const bool was_valid = auth_data[user_id].used; // Zero out this session (also sets valid to false == 0) memset(&auth_data[user_id], 0, sizeof(auth_data[user_id])); + + return was_valid; } void delete_all_sessions(void) @@ -392,13 +396,14 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t { log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie"); - // Revoke client authentication. This slot can be used by a new client afterwards. - delete_session(user_id); - strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers)); - cJSON *json = JSON_NEW_OBJECT(); - get_session_object(api, json, user_id, now); - JSON_SEND_OBJECT_CODE(json, 410); // 410 Gone + + // Revoke client authentication. This slot can be used by a new client afterwards. + const int code = delete_session(user_id) ? 204 : 404; + + // Send empty reply with appropriate HTTP status code + send_http_code(api, "application/json; charset=utf-8", code, ""); + return code; } else { @@ -563,7 +568,7 @@ int api_auth(struct ftl_conn *api) { // Expired slow, mark as unused if(auth_data[i].used && - auth_data[i].valid_until < now) + auth_data[i].valid_until < now) { log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...", i, auth_data[i].remote_addr); @@ -667,9 +672,9 @@ int api_auth_session_delete(struct ftl_conn *api) return send_json_error(api, 400, "bad_request", "Session ID not in use", NULL); // Delete session - delete_session(uid); + const int code = delete_session(uid) ? 204 : 404; - // Send empty reply with code 204 No Content - send_http_code(api, "application/json; charset=utf-8", 204, ""); - return 204; + // Send empty reply with appropriate HTTP status code + send_http_code(api, "application/json; charset=utf-8", code, ""); + return code; } diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 1a665f43..15bf051e 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -118,21 +118,23 @@ components: - Authentication operationId: "delete_groups" description: | - A logout attempt without a valid session will result in a `401 Unauthorized` error. + This endpoint can be used to delete the current session. It will + invalidate the session token and the CSRF token. The session can be + extended before its expiration by performing any authenticated action. + By default, the session lasts for 5 minutes. It can be invalidated by + either logging out or deleting the session. Additionally, the session + becomes invalid when the password is altered or a new application + password is created. - A session that was not created due to a login cannot be deleted (e.g., empty API password). + You can also delete a session by its ID using the `DELETE /auth/session/{id}` endpoint. + + Note that you cannot delete the current session if you have not + authenticated (e.g., no password has been set on your Pi-hole). responses: - '200': - description: OK (session not deletable) - content: - application/json: - schema: - allOf: - - $ref: 'auth.yaml#/components/schemas/session' - - $ref: 'common.yaml#/components/schemas/took' - examples: - no_login_required: - $ref: 'auth.yaml#/components/examples/no_login_required' + '204': + description: No Content (deleted) + '404': + description: Not Found (no session active) '401': description: Unauthorized content: @@ -141,17 +143,6 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' - '410': - description: Gone - content: - application/json: - schema: - allOf: - - $ref: 'auth.yaml#/components/schemas/session' - - $ref: 'common.yaml#/components/schemas/took' - examples: - login_failed: - $ref: 'auth.yaml#/components/examples/login_failed' session_list: get: summary: List of all current sessions @@ -213,6 +204,8 @@ components: responses: '204': description: No Content (deleted) + '404': + description: Not Found (session not found) '400': description: Bad Request content: diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 0228696c..4bf6318d 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -95,6 +95,8 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found '400': description: Bad request content: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index 115232dc..48f1e534 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -128,6 +128,8 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found '400': description: Bad request content: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index 20e61ae6..16899960 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -94,6 +94,8 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found '400': description: Bad request content: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index b5019702..c205a5ca 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -93,6 +93,8 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found '400': description: Bad request content: diff --git a/src/api/list.c b/src/api/list.c index 8b371e95..467ea03b 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -705,7 +705,8 @@ static int api_list_remove(struct ftl_conn *api, } // From here on, we can assume the JSON payload is valid - if(gravityDB_delFromTable(listtype, array, &sql_msg)) + unsigned int deleted = 0u; + if(gravityDB_delFromTable(listtype, array, &deleted, &sql_msg)) { // Inform the resolver that it needs to reload gravity set_event(RELOAD_GRAVITY); @@ -714,9 +715,11 @@ static int api_list_remove(struct ftl_conn *api, if(allocated_json) cJSON_free(array); - // Send empty reply with code 204 No Content + // Send empty reply with codes: + // - 204 No Content (if any items were deleted) + // - 404 Not Found (if no items were deleted) cJSON *json = JSON_NEW_OBJECT(); - JSON_SEND_OBJECT_CODE(json, 204); + JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404); } else { diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 92c320b9..f137039a 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1792,8 +1792,9 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, return okay; } -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message) +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message) { + // Return early if database is not available if(gravity_db == NULL) { *message = "Database not available"; @@ -2004,6 +2005,9 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* break; } + + // Add number of deleted rows + *deleted += sqlite3_changes(gravity_db); } // Drop temporary table diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 7d47892d..a77deb72 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -69,7 +69,7 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow * void gravityDB_readTableFinalize(void); bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, const char **message, const enum http_method method); -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message); +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message); bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups, const tablerow *row, const char **message); From bb50a106d10429f298811f594a4a253e7b449746 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:20:28 +0100 Subject: [PATCH 46/55] A few fixed for the response code documentation of the :batchDelete elements Signed-off-by: DL6ER --- src/api/docs/content/specs/auth.yaml | 16 ++++++++++++++ src/api/docs/content/specs/clients.yaml | 18 +++++++++++++++ src/api/docs/content/specs/config.yaml | 4 ++++ src/api/docs/content/specs/dhcp.yaml | 4 ++++ src/api/docs/content/specs/domains.yaml | 18 +++++++++++++++ src/api/docs/content/specs/groups.yaml | 29 +++++++++++++++++-------- src/api/docs/content/specs/info.yaml | 4 ++++ src/api/docs/content/specs/lists.yaml | 26 ++++++++++++++-------- src/api/docs/content/specs/network.yaml | 4 ++++ 9 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 15bf051e..d32ed174 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -133,8 +133,16 @@ components: responses: '204': description: No Content (deleted) + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Not Found (no session active) + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '401': description: Unauthorized content: @@ -204,8 +212,16 @@ components: responses: '204': description: No Content (deleted) + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Not Found (session not found) + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad Request content: diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 4bf6318d..1eaf385f 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -95,8 +95,16 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -235,6 +243,16 @@ components: responses: '204': description: Items deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 90b107dd..d53b5560 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -144,6 +144,10 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/dhcp.yaml b/src/api/docs/content/specs/dhcp.yaml index bde0ef54..86a0bdc6 100644 --- a/src/api/docs/content/specs/dhcp.yaml +++ b/src/api/docs/content/specs/dhcp.yaml @@ -40,6 +40,10 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index 48f1e534..5fa86d2a 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -128,8 +128,16 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -253,6 +261,16 @@ components: responses: '204': description: Items deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index 16899960..5bf6fe59 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -63,6 +63,9 @@ components: - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET - $ref: 'groups.yaml#/components/schemas/lists_processed' - $ref: 'common.yaml#/components/schemas/took' + headers: + Location: + $ref: 'common.yaml#/components/headers/Location' '400': description: Bad request content: @@ -94,8 +97,16 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -195,18 +206,18 @@ components: - "item": "test1" - "item": "test2" responses: - '201': - description: Created item + '204': + description: Items deleted content: application/json: schema: - allOf: - - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET - - $ref: 'groups.yaml#/components/schemas/lists_processed' - - $ref: 'common.yaml#/components/schemas/took' - headers: - Location: - $ref: 'common.yaml#/components/headers/Location' + $ref: 'common.yaml#/components/schemas/took' + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index 65f118ea..e98b6925 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -222,6 +222,10 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index c205a5ca..1260e8bb 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -93,8 +93,16 @@ components: responses: '204': description: Item deleted + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -191,18 +199,18 @@ components: schema: $ref: 'lists.yaml#/components/schemas/lists/post' responses: - '201': - description: Created item + '204': + description: Items deleted content: application/json: schema: - allOf: - - $ref: 'lists.yaml#/components/schemas/lists/get' - - $ref: 'lists.yaml#/components/schemas/lists_processed' - - $ref: 'common.yaml#/components/schemas/took' - headers: - Location: - $ref: 'common.yaml#/components/headers/Location' + $ref: 'common.yaml#/components/schemas/took' + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 00f74afb..90c7644e 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -93,6 +93,10 @@ components: responses: '204': description: No Content (deleted) + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '401': description: Unauthorized content: From c0172130fd857599698862273f09f6dab63510d3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:25:25 +0100 Subject: [PATCH 47/55] Extend 204/404 logic to /info/messages/{message_id} Signed-off-by: DL6ER --- src/api/docs/content/specs/info.yaml | 16 +++++++++++++++- src/api/info.c | 9 ++++++--- src/database/message-table.c | 6 +++++- src/database/message-table.h | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index e98b6925..0513db60 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -218,7 +218,7 @@ components: parameters: - $ref: 'info.yaml#/components/parameters/message_id' description: | - *Note:* There will be no content on success. You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`) + You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`) responses: '204': description: Item deleted @@ -226,6 +226,12 @@ components: application/json: schema: $ref: 'common.yaml#/components/schemas/took' + '404': + description: Not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -239,6 +245,14 @@ components: $ref: 'info.yaml#/components/examples/errors/messages/uri_error' bad_request: $ref: 'info.yaml#/components/examples/errors/messages/bad_request' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' messages_count: get: summary: Get count of Pi-hole diagnosis messages diff --git a/src/api/info.c b/src/api/info.c index 61398084..e1397f7c 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -940,15 +940,18 @@ static int api_info_messages_DELETE(struct ftl_conn *api) } // Delete message with this ID from the database - delete_message(ids); + int deleted = 0; + delete_message(ids, &deleted); // Free memory free(id); cJSON_free(ids); - // Send empty reply with code 204 No Content + // Send empty reply with codes: + // - 204 No Content (if any items were deleted) + // - 404 Not Found (if no items were deleted) cJSON *json = JSON_NEW_OBJECT(); - JSON_SEND_OBJECT_CODE(json, 204); + JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404); } int api_info_messages(struct ftl_conn *api) diff --git a/src/database/message-table.c b/src/database/message-table.c index 3461b200..55c36a24 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -378,7 +378,7 @@ end_of_add_message: // Close database connection return rowid; } -bool delete_message(cJSON *ids) +bool delete_message(cJSON *ids, int *deleted) { // Return early if database is known to be broken if(FTLDBerror()) @@ -413,6 +413,10 @@ bool delete_message(cJSON *ids) log_err("SQL error (%i): %s", sqlite3_errcode(db), sqlite3_errmsg(db)); return false; } + + // Add to deleted count + *deleted += sqlite3_changes(db); + sqlite3_reset(res); sqlite3_clear_bindings(res); } diff --git a/src/database/message-table.h b/src/database/message-table.h index 196b406a..14956bf5 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -16,7 +16,7 @@ int count_messages(const bool filter_dnsmasq_warnings); bool format_messages(cJSON *array); bool create_message_table(sqlite3 *db); -bool delete_message(cJSON *ids); +bool delete_message(cJSON *ids, int *deleted); bool flush_message_table(void); void logg_regex_warning(const char *type, const char *warning, const int dbindex, const char *regex); void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids, From 55e0e84a927103ba41f33b43b7b92206aa191116 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:28:27 +0100 Subject: [PATCH 48/55] Extend 204/404 logic to /network/devices/{device_id} Signed-off-by: DL6ER --- src/api/docs/content/specs/network.yaml | 14 ++++++++++++++ src/api/info.c | 2 +- src/api/network.c | 9 ++++++--- src/database/network-table.c | 8 +++++++- src/database/network-table.h | 2 +- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 90c7644e..0c83dd1f 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -97,6 +97,20 @@ components: application/json: schema: $ref: 'common.yaml#/components/schemas/took' + '404': + description: Not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' '401': description: Unauthorized content: diff --git a/src/api/info.c b/src/api/info.c index e1397f7c..7158cd01 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -951,7 +951,7 @@ static int api_info_messages_DELETE(struct ftl_conn *api) // - 204 No Content (if any items were deleted) // - 404 Not Found (if no items were deleted) cJSON *json = JSON_NEW_OBJECT(); - JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404); + JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404); } int api_info_messages(struct ftl_conn *api) diff --git a/src/api/network.c b/src/api/network.c index 9d26ddc2..532a985e 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -440,7 +440,8 @@ static int api_network_devices_DELETE(struct ftl_conn *api) // Delete row from network table by ID const char *sql_msg = NULL; - if(!networkTable_deleteDevice(db, device_id, &sql_msg)) + int deleted = 0; + if(!networkTable_deleteDevice(db, device_id, &deleted, &sql_msg)) { // Add SQL message (may be NULL = not available) return send_json_error(api, 500, @@ -452,9 +453,11 @@ static int api_network_devices_DELETE(struct ftl_conn *api) // Close database dbclose(&db); - // Send empty reply with code 204 No Content + // Send empty reply with codes: + // - 204 No Content (if any items were deleted) + // - 404 Not Found (if no items were deleted) cJSON *json = JSON_NEW_OBJECT(); - JSON_SEND_OBJECT_CODE(json, 204); + JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404); } int api_network_devices(struct ftl_conn *api) diff --git a/src/database/network-table.c b/src/database/network-table.c index 8c0e9a1f..c76de35c 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -2425,7 +2425,7 @@ void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt) sqlite3_finalize(read_stmt); } -bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message) +bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message) { // First step: Delete all associated IPs of this device // Prepare SQLite statement @@ -2462,6 +2462,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message) return false; } + // Check if we deleted any rows + *deleted += sqlite3_changes(db); + // Finalize statement sqlite3_finalize(stmt); @@ -2498,6 +2501,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message) return false; } + // Check if we deleted any rows + *deleted += sqlite3_changes(db); + // Finalize statement sqlite3_finalize(stmt); diff --git a/src/database/network-table.h b/src/database/network-table.h index 1cb4697d..8da0fa3f 100644 --- a/src/database/network-table.h +++ b/src/database/network-table.h @@ -52,6 +52,6 @@ bool networkTable_readIPs(sqlite3 *db, sqlite3_stmt **read_stmt, const int id, c bool networkTable_readIPsGetRecord(sqlite3_stmt *read_stmt, network_addresses_record *network_addresses, const char **message); void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt); -bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message); +bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message); #endif //NETWORKTABLE_H From 06147abd5d1b52618fef6ac04aa983be3870da27 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:32:05 +0100 Subject: [PATCH 49/55] A 204 response must not contain a body (https://tools.ietf.org/html/rfc7231#section-6.3.5) Signed-off-by: DL6ER --- src/api/docs/content/specs/auth.yaml | 8 -------- src/api/docs/content/specs/clients.yaml | 8 -------- src/api/docs/content/specs/config.yaml | 4 ---- src/api/docs/content/specs/dhcp.yaml | 4 ---- src/api/docs/content/specs/domains.yaml | 8 -------- src/api/docs/content/specs/groups.yaml | 8 -------- src/api/docs/content/specs/info.yaml | 4 ---- src/api/docs/content/specs/lists.yaml | 8 -------- src/api/docs/content/specs/network.yaml | 4 ---- src/webserver/json_macros.h | 5 ++++- 10 files changed, 4 insertions(+), 57 deletions(-) diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index d32ed174..dabe9e5d 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -133,10 +133,6 @@ components: responses: '204': description: No Content (deleted) - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Not Found (no session active) content: @@ -212,10 +208,6 @@ components: responses: '204': description: No Content (deleted) - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Not Found (session not found) content: diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 1eaf385f..72f0a471 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -95,10 +95,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: @@ -243,10 +239,6 @@ components: responses: '204': description: Items deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index d53b5560..90b107dd 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -144,10 +144,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/dhcp.yaml b/src/api/docs/content/specs/dhcp.yaml index 86a0bdc6..bde0ef54 100644 --- a/src/api/docs/content/specs/dhcp.yaml +++ b/src/api/docs/content/specs/dhcp.yaml @@ -40,10 +40,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index 5fa86d2a..1ce7213f 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -128,10 +128,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: @@ -261,10 +257,6 @@ components: responses: '204': description: Items deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index 5bf6fe59..6712c599 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -97,10 +97,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: @@ -208,10 +204,6 @@ components: responses: '204': description: Items deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index 0513db60..78391afe 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -222,10 +222,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Not found content: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index 1260e8bb..90df09ac 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -93,10 +93,6 @@ components: responses: '204': description: Item deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: @@ -201,10 +197,6 @@ components: responses: '204': description: Items deleted - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Item not found content: diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 0c83dd1f..1c497a65 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -93,10 +93,6 @@ components: responses: '204': description: No Content (deleted) - content: - application/json: - schema: - $ref: 'common.yaml#/components/schemas/took' '404': description: Not found content: diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index e1304a37..f3966f08 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -200,7 +200,10 @@ }) #define JSON_SEND_OBJECT_CODE(object, code)({ \ - cJSON_AddNumberToObject(object, "took", double_time() - api->now);\ + if((code) != 204) \ + { \ + cJSON_AddNumberToObject(object, "took", double_time() - api->now); \ + } \ char *json_string = json_formatter(object); \ if(json_string == NULL) \ { \ From 85e3d4dd086acb3f72f00ea698ca0cd0dc741f96 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:41:25 +0100 Subject: [PATCH 50/55] Extend 204/404 logic to /config/{element}/{value} Signed-off-by: DL6ER --- src/api/config.c | 20 +++++++++++--------- src/api/docs/content/specs/config.yaml | 19 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index ac01cfb2..29e516a3 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -904,7 +904,7 @@ static int api_config_put_delete(struct ftl_conn *api) key, true); } - // Check if this entry does already exist in the array + // Check if this entry exists in the array int idx = 0; for(; idx < cJSON_GetArraySize(new_item->v.json); idx++) { @@ -938,13 +938,12 @@ static int api_config_put_delete(struct ftl_conn *api) if(found) { // Remove item from array + found = true; cJSON_DeleteItemFromArray(new_item->v.json, idx); } else { // Item not found - message = "Item not found"; - hint = "Can only delete existing items"; break; } } @@ -964,13 +963,16 @@ static int api_config_put_delete(struct ftl_conn *api) // Release allocated memory free_config_path(requested_path); - // Error 404 if not found - if(!found || message != NULL) + // Error 404 if config element not found + if(!found) + { + cJSON *json = JSON_NEW_OBJECT(); + JSON_SEND_OBJECT_CODE(json, 404); + } + + // Error 400 if unique item already present + if(message != NULL) { - // For any other error, a more specific message will have been added - // above - if(!message) - message = "No item specified"; return send_json_error(api, 400, "bad_request", message, diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 90b107dd..91ea040b 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -121,7 +121,7 @@ components: examples: invalid_path_depth: $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth' - item_not_found: + item_already_present: $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present' '401': description: Unauthorized @@ -144,6 +144,12 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: @@ -155,8 +161,8 @@ components: examples: invalid_path_depth: $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth' - item_not_found: - $ref: 'config.yaml#/components/examples/errors/bad_request/item_not_found' + item_already_present: + $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present' '401': description: Unauthorized content: @@ -795,13 +801,6 @@ components: key: "bad_request" message: "Invalid path depth" hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams" - item_not_found: - summary: Item to be deleted does not exist - value: - error: - key: "bad_request" - message: "Item not found" - hint: "Can only delete existing items" item_already_present: summary: Item to be added exists already value: From 1d83e394211e2c7d0e45a19970c7d54bfac4620d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 09:44:55 +0100 Subject: [PATCH 51/55] Extend 204/404 logic to /dhcp/leases/{ip} Signed-off-by: DL6ER --- src/api/dhcp.c | 12 +++++++----- src/api/docs/content/specs/dhcp.yaml | 6 ++++++ src/dnsmasq_interface.c | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/api/dhcp.c b/src/api/dhcp.c index 2d94a3ee..91df5129 100644 --- a/src/api/dhcp.c +++ b/src/api/dhcp.c @@ -85,16 +85,18 @@ int api_dhcp_leases_DELETE(struct ftl_conn *api) // Send empty reply with code 204 No Content return send_json_error(api, 400, - "bad_request", + "bad_request", "The provided IPv4 address is invalid", - api->item); + api->item); } // Delete lease log_debug(DEBUG_API, "Deleting DHCP lease for address %s", api->item); - FTL_unlink_DHCP_lease(api->item); + const bool found = FTL_unlink_DHCP_lease(api->item); - // Send empty reply with code 204 No Content + // Send empty reply with codes: + // - 204 No Content (if a lease was deleted) + // - 404 Not Found (if no lease was found) cJSON *json = JSON_NEW_OBJECT(); - JSON_SEND_OBJECT_CODE(json, 204); + JSON_SEND_OBJECT_CODE(json, found ? 204 : 404); } \ No newline at end of file diff --git a/src/api/docs/content/specs/dhcp.yaml b/src/api/docs/content/specs/dhcp.yaml index bde0ef54..80cae62e 100644 --- a/src/api/docs/content/specs/dhcp.yaml +++ b/src/api/docs/content/specs/dhcp.yaml @@ -40,6 +40,12 @@ components: responses: '204': description: Item deleted + '404': + description: Item not found + content: + application/json: + schema: + $ref: 'common.yaml#/components/schemas/took' '400': description: Bad request content: diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index a110af4c..311ce2c8 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3263,6 +3263,7 @@ bool FTL_unlink_DHCP_lease(const char *ipaddr) #endif else { + // Invalid IP address or no lease found return false; } From a961a4d14fed2fb9205c7dc8637bca0d4d35b0fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 15:20:07 +0100 Subject: [PATCH 52/55] Do not accept DELETE session if no session is used (this also applies to password-less or localhost-no-auth mode) Signed-off-by: DL6ER --- src/api/auth.c | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index 5b535b72..dbb21b6c 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -358,24 +358,6 @@ void delete_all_sessions(void) static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now) { - if(user_id == API_AUTH_LOCALHOST) - { - log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)"); - - cJSON *json = JSON_NEW_OBJECT(); - get_session_object(api, json, user_id, now); - JSON_SEND_OBJECT(json); - } - - if(user_id == API_AUTH_EMPTYPASS) - { - log_debug(DEBUG_API, "API Auth status: OK (empty password)"); - - cJSON *json = JSON_NEW_OBJECT(); - get_session_object(api, json, user_id, now); - JSON_SEND_OBJECT(json); - } - if(user_id > API_AUTH_UNAUTHORIZED && (api->method == HTTP_GET || api->method == HTTP_POST)) { log_debug(DEBUG_API, "API Auth status: OK"); @@ -392,18 +374,45 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t get_session_object(api, json, user_id, now); JSON_SEND_OBJECT(json); } - else if(user_id > API_AUTH_UNAUTHORIZED && api->method == HTTP_DELETE) + else if(api->method == HTTP_DELETE) { - log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie"); + if(user_id > API_AUTH_UNAUTHORIZED) + { + log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie"); - strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers)); + strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers)); - // Revoke client authentication. This slot can be used by a new client afterwards. - const int code = delete_session(user_id) ? 204 : 404; + // Revoke client authentication. This slot can be used by a new client afterwards. + const int code = delete_session(user_id) ? 204 : 404; - // Send empty reply with appropriate HTTP status code - send_http_code(api, "application/json; charset=utf-8", code, ""); - return code; + // Send empty reply with appropriate HTTP status code + send_http_code(api, "application/json; charset=utf-8", code, ""); + return code; + } + else + { + log_debug(DEBUG_API, "API Auth status: Logout, but not authenticated"); + + cJSON *json = JSON_NEW_OBJECT(); + get_session_object(api, json, user_id, now); + JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized + } + } + else if(user_id == API_AUTH_LOCALHOST) + { + log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)"); + + cJSON *json = JSON_NEW_OBJECT(); + get_session_object(api, json, user_id, now); + JSON_SEND_OBJECT(json); + } + else if(user_id == API_AUTH_EMPTYPASS) + { + log_debug(DEBUG_API, "API Auth status: OK (empty password)"); + + cJSON *json = JSON_NEW_OBJECT(); + get_session_object(api, json, user_id, now); + JSON_SEND_OBJECT(json); } else { From 35e1acb533bd2bf78bb298de96c33c18cb40e3da Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 15:26:49 +0100 Subject: [PATCH 53/55] Do not accept password login when the system is configured to not require a password Signed-off-by: DL6ER --- src/api/auth.c | 5 +++++ src/config/password.c | 3 ++- src/config/password.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/api/auth.c b/src/api/auth.c index dbb21b6c..4fd0015d 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -648,6 +648,11 @@ int api_auth(struct ftl_conn *api) "Rate-limiting login attempts", NULL); } + else if(result == NO_PASSWORD_SET) + { + // No password set + log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password); + } else { log_debug(DEBUG_API, "API: Password incorrect: '%s'", password); diff --git a/src/config/password.c b/src/config/password.c index 727d1908..7a650678 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -328,6 +328,7 @@ enum password_result verify_login(const char *password) log_debug(DEBUG_API, "App password correct"); return APPPASSWORD_CORRECT; } + // Return result return pw; } @@ -336,7 +337,7 @@ enum password_result verify_password(const char *password, const char *pwhash, c { // No password set if(pwhash == NULL || pwhash[0] == '\0') - return PASSWORD_CORRECT; + return NO_PASSWORD_SET; // No password supplied if(password == NULL || password[0] == '\0') diff --git a/src/config/password.h b/src/config/password.h index a4cc96b6..063e5dcf 100644 --- a/src/config/password.h +++ b/src/config/password.h @@ -26,6 +26,7 @@ enum password_result { PASSWORD_INCORRECT = 0, PASSWORD_CORRECT = 1, APPPASSWORD_CORRECT = 2, + NO_PASSWORD_SET = 3, PASSWORD_RATE_LIMITED = -1 } __attribute__((packed)); From 14af354979ec16caf0af4949910b99d53b8ad982 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jan 2024 21:16:30 +0100 Subject: [PATCH 54/55] Do not rely on the old behavior of empty password is always correct when no password is set when changing the latter Signed-off-by: DL6ER --- src/api/config.c | 2 +- src/config/cli.c | 5 +++-- src/config/password.c | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 29e516a3..ade2896f 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -294,7 +294,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct } if(!set_and_check_password(conf_item, elem->valuestring)) - return "Failed to create password hash (verification failed), password remains unchanged"; + return "password hash verification failed"; break; } diff --git a/src/config/cli.c b/src/config/cli.c index a7868342..effaae96 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -160,8 +160,9 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru // Get password hash as allocated string (an empty string is hashed to an empty string) char *pwhash = strlen(value) > 0 ? create_password(value) : strdup(""); - // Verify that the password hash is valid - if(verify_password(value, pwhash, false) != PASSWORD_CORRECT) + // Verify that the password hash is either valid or empty + const enum password_result status = verify_password(value, pwhash, false); + if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET) { log_err("Failed to create password hash (verification failed), password remains unchanged"); free(pwhash); diff --git a/src/config/password.c b/src/config/password.c index 7a650678..d97df6d2 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -607,8 +607,9 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password) // Get password hash as allocated string (an empty string is hashed to an empty string) char *pwhash = strlen(password) > 0 ? create_password(password) : strdup(""); - // Verify that the password hash is valid - if(verify_password(password, pwhash, false) != PASSWORD_CORRECT) + // Verify that the password hash is valid or that no password is set + const enum password_result status = verify_password(password, pwhash, false); + if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET) { free(pwhash); log_warn("Failed to create password hash (verification failed), password remains unchanged"); From 4174fe3b098f6392f03cbd4e661309d5a8007843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 Jan 2024 22:23:19 +0100 Subject: [PATCH 55/55] Don't print double newlines after invalid domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/tools/gravity-parseList.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 61856134..fc73dafa 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -542,8 +542,6 @@ end_of_parseList: // Print newline puts(""); } - // Print final newline - puts(""); } // Free memory